Java tutorial
/* * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 * (the "License"). You may not use this work except in compliance with the License, which is * available at www.apache.org/licenses/LICENSE-2.0 * * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, * either express or implied, as more fully set forth in the License. * * See the NOTICE file distributed with this work for information regarding copyright ownership. */ package alluxio.cli.fs.command; import alluxio.AlluxioURI; import alluxio.Configuration; import alluxio.Constants; import alluxio.PropertyKey; import alluxio.cli.CommandUtils; import alluxio.client.file.FileInStream; import alluxio.client.file.FileOutStream; import alluxio.client.file.FileSystem; import alluxio.client.file.URIStatus; import alluxio.client.file.options.CreateFileOptions; import alluxio.client.file.options.OpenFileOptions; import alluxio.client.file.policy.FileWriteLocationPolicy; import alluxio.exception.AlluxioException; import alluxio.exception.ExceptionMessage; import alluxio.exception.FileAlreadyExistsException; import alluxio.exception.FileDoesNotExistException; import alluxio.exception.InvalidPathException; import alluxio.cli.fs.FileSystemShellUtils; import alluxio.exception.status.InvalidArgumentException; import alluxio.util.CommonUtils; import alluxio.util.io.PathUtils; import com.google.common.base.Joiner; import com.google.common.io.Closer; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.RandomStringUtils; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.util.ArrayList; import java.util.List; import javax.annotation.concurrent.ThreadSafe; /** * Copies a file or a directory in the Alluxio filesystem. */ @ThreadSafe public final class CpCommand extends AbstractFileSystemCommand { private static final Option RECURSIVE_OPTION = Option.builder("R").required(false).hasArg(false) .desc("copy files in subdirectories recursively").build(); /** * @param fs the filesystem of Alluxio */ public CpCommand(FileSystem fs) { super(fs); } @Override public String getCommandName() { return "cp"; } @Override public void validateArgs(CommandLine cl) throws InvalidArgumentException { CommandUtils.checkNumOfArgsEquals(this, cl, 2); } @Override public Options getOptions() { return new Options().addOption(RECURSIVE_OPTION); } @Override public int run(CommandLine cl) throws AlluxioException, IOException { String[] args = cl.getArgs(); AlluxioURI srcPath = new AlluxioURI(args[0]); AlluxioURI dstPath = new AlluxioURI(args[1]); if ((dstPath.getScheme() == null || isAlluxio(dstPath.getScheme())) && isFile(srcPath.getScheme())) { List<File> srcFiles = FileSystemShellUtils.getFiles(srcPath.getPath()); if (srcFiles.size() == 0) { throw new IOException(ExceptionMessage.PATH_DOES_NOT_EXIST.getMessage(srcPath)); } if (srcPath.containsWildcard()) { List<AlluxioURI> srcPaths = new ArrayList<>(); for (File srcFile : srcFiles) { srcPaths.add(new AlluxioURI(srcPath.getScheme(), srcPath.getAuthority(), srcFile.getPath())); } copyFromLocalWildcard(srcPaths, dstPath); } else { copyFromLocal(srcPath, dstPath); } } else if ((srcPath.getScheme() == null || isAlluxio(srcPath.getScheme())) && isFile(dstPath.getScheme())) { List<AlluxioURI> srcPaths = FileSystemShellUtils.getAlluxioURIs(mFileSystem, srcPath); if (srcPaths.size() == 0) { throw new IOException(ExceptionMessage.PATH_DOES_NOT_EXIST.getMessage(srcPath)); } if (srcPath.containsWildcard()) { copyWildcardToLocal(srcPaths, dstPath); } else { copyToLocal(srcPath, dstPath); } } else if ((srcPath.getScheme() == null || isAlluxio(srcPath.getScheme())) && (dstPath.getScheme() == null || isAlluxio(dstPath.getScheme()))) { List<AlluxioURI> srcPaths = FileSystemShellUtils.getAlluxioURIs(mFileSystem, srcPath); if (srcPaths.size() == 0) { throw new FileDoesNotExistException( ExceptionMessage.PATH_DOES_NOT_EXIST.getMessage(srcPath.getPath())); } if (srcPath.containsWildcard()) { copyWildcard(srcPaths, dstPath, cl.hasOption("R")); } else { copy(srcPath, dstPath, cl.hasOption("R")); } } else { throw new InvalidPathException( "Schemes must be either file or alluxio, and at most one file scheme is allowed."); } return 0; } /** * Copies a list of files or directories specified by srcPaths to the destination specified by * dstPath. This method is used when the original source path contains wildcards. * * @param srcPaths a list of files or directories in the Alluxio filesystem * @param dstPath the destination in the Alluxio filesystem * @param recursive indicates whether directories should be copied recursively */ private void copyWildcard(List<AlluxioURI> srcPaths, AlluxioURI dstPath, boolean recursive) throws AlluxioException, IOException { URIStatus dstStatus = null; try { dstStatus = mFileSystem.getStatus(dstPath); } catch (FileDoesNotExistException e) { // if the destination does not exist, it will be created } if (dstStatus != null && !dstStatus.isFolder()) { throw new InvalidPathException(ExceptionMessage.DESTINATION_CANNOT_BE_FILE.getMessage()); } if (dstStatus == null) { mFileSystem.createDirectory(dstPath); System.out.println("Created directory: " + dstPath); } List<String> errorMessages = new ArrayList<>(); for (AlluxioURI srcPath : srcPaths) { try { copy(srcPath, new AlluxioURI(dstPath.getScheme(), dstPath.getAuthority(), PathUtils.concatPath(dstPath.getPath(), srcPath.getName())), recursive); } catch (AlluxioException | IOException e) { errorMessages.add(e.getMessage()); } } if (errorMessages.size() != 0) { throw new IOException(Joiner.on('\n').join(errorMessages)); } } /** * Copies a file or a directory in the Alluxio filesystem. * * @param srcPath the source {@link AlluxioURI} (could be a file or a directory) * @param dstPath the {@link AlluxioURI} of the destination path in the Alluxio filesystem * @param recursive indicates whether directories should be copied recursively */ private void copy(AlluxioURI srcPath, AlluxioURI dstPath, boolean recursive) throws AlluxioException, IOException { URIStatus srcStatus = mFileSystem.getStatus(srcPath); URIStatus dstStatus = null; try { dstStatus = mFileSystem.getStatus(dstPath); } catch (FileDoesNotExistException e) { // if the destination does not exist, it will be created } if (!srcStatus.isFolder()) { if (dstStatus != null && dstStatus.isFolder()) { dstPath = new AlluxioURI(PathUtils.concatPath(dstPath.getPath(), srcPath.getName())); } copyFile(srcPath, dstPath); } else { if (!recursive) { throw new IOException( srcPath.getPath() + " is a directory, to copy it please use \"cp -R <src> <dst>\""); } List<URIStatus> statuses; statuses = mFileSystem.listStatus(srcPath); if (dstStatus != null) { if (!dstStatus.isFolder()) { throw new InvalidPathException(ExceptionMessage.DESTINATION_CANNOT_BE_FILE.getMessage()); } // if copying a directory to an existing directory, the copied directory will become a // subdirectory of the destination if (srcStatus.isFolder()) { dstPath = new AlluxioURI(PathUtils.concatPath(dstPath.getPath(), srcPath.getName())); mFileSystem.createDirectory(dstPath); System.out.println("Created directory: " + dstPath); } } if (dstStatus == null) { mFileSystem.createDirectory(dstPath); System.out.println("Created directory: " + dstPath); } List<String> errorMessages = new ArrayList<>(); for (URIStatus status : statuses) { try { copy(new AlluxioURI(srcPath.getScheme(), srcPath.getAuthority(), status.getPath()), new AlluxioURI(dstPath.getScheme(), dstPath.getAuthority(), PathUtils.concatPath(dstPath.getPath(), status.getName())), recursive); } catch (IOException e) { errorMessages.add(e.getMessage()); } } if (errorMessages.size() != 0) { throw new IOException(Joiner.on('\n').join(errorMessages)); } } } /** * Copies a file in the Alluxio filesystem. * * @param srcPath the source {@link AlluxioURI} (has to be a file) * @param dstPath the destination path in the Alluxio filesystem */ private void copyFile(AlluxioURI srcPath, AlluxioURI dstPath) throws AlluxioException, IOException { try (Closer closer = Closer.create()) { OpenFileOptions openFileOptions = OpenFileOptions.defaults(); FileInStream is = closer.register(mFileSystem.openFile(srcPath, openFileOptions)); CreateFileOptions createFileOptions = CreateFileOptions.defaults(); FileOutStream os = closer.register(mFileSystem.createFile(dstPath, createFileOptions)); try { IOUtils.copy(is, os); } catch (Exception e) { os.cancel(); throw e; } System.out.println("Copied " + srcPath + " to " + dstPath); } } /** * Copies a directory from local to Alluxio filesystem. The destination directory structure * maintained as local directory. This method is used when input path is a directory. * * @param srcPath the {@link AlluxioURI} of the source directory in the local filesystem * @param dstPath the {@link AlluxioURI} of the destination */ private void copyFromLocalDir(AlluxioURI srcPath, AlluxioURI dstPath) throws AlluxioException, IOException { File srcDir = new File(srcPath.getPath()); boolean dstExistedBefore = mFileSystem.exists(dstPath); createDstDir(dstPath); List<String> errorMessages = new ArrayList<>(); File[] fileList = srcDir.listFiles(); if (fileList == null) { String errMsg = String.format("Failed to list files for directory %s", srcDir); errorMessages.add(errMsg); fileList = new File[0]; } int misFiles = 0; for (File srcFile : fileList) { AlluxioURI newURI = new AlluxioURI(dstPath, new AlluxioURI(srcFile.getName())); try { copyPath(new AlluxioURI(srcPath.getScheme(), srcPath.getAuthority(), srcFile.getPath()), newURI); } catch (AlluxioException | IOException e) { errorMessages.add(e.getMessage()); if (!mFileSystem.exists(newURI)) { misFiles++; } } } if (errorMessages.size() != 0) { if (misFiles == fileList.length) { // If the directory doesn't exist and no files were created, then delete the directory if (!dstExistedBefore && mFileSystem.exists(dstPath)) { mFileSystem.delete(dstPath); } } throw new IOException(Joiner.on('\n').join(errorMessages)); } } /** * Copies a list of files or directories specified by srcPaths from the local filesystem to * dstPath in the Alluxio filesystem space. This method is used when the input path contains * wildcards. * * @param srcPaths a list of files or directories in the local filesystem * @param dstPath the {@link AlluxioURI} of the destination */ private void copyFromLocalWildcard(List<AlluxioURI> srcPaths, AlluxioURI dstPath) throws AlluxioException, IOException { boolean dstExistedBefore = mFileSystem.exists(dstPath); createDstDir(dstPath); List<String> errorMessages = new ArrayList<>(); int misFiles = 0; for (AlluxioURI srcPath : srcPaths) { AlluxioURI newURI = new AlluxioURI(dstPath, new AlluxioURI(srcPath.getName())); try { copyPath(srcPath, newURI); System.out.println("Copied " + srcPath + " to " + dstPath); } catch (AlluxioException | IOException e) { errorMessages.add(e.getMessage()); if (!mFileSystem.exists(newURI)) { misFiles++; } } } if (errorMessages.size() != 0) { if (misFiles == srcPaths.size()) { // If the directory doesn't exist and no files were created, then delete the directory if (!dstExistedBefore && mFileSystem.exists(dstPath)) { mFileSystem.delete(dstPath); } } throw new IOException(Joiner.on('\n').join(errorMessages)); } } /** * Creates a directory in the Alluxio filesystem space. It will not throw any exception if the * destination directory already exists. * * @param dstPath the {@link AlluxioURI} of the destination directory which will be created */ private void createDstDir(AlluxioURI dstPath) throws AlluxioException, IOException { try { mFileSystem.createDirectory(dstPath); } catch (FileAlreadyExistsException e) { // it's fine if the directory already exists } URIStatus dstStatus = mFileSystem.getStatus(dstPath); if (!dstStatus.isFolder()) { throw new InvalidPathException(ExceptionMessage.DESTINATION_CANNOT_BE_FILE.getMessage()); } } /** * Copies a file or directory specified by srcPath from the local filesystem to dstPath in the * Alluxio filesystem space. * * @param srcPath the {@link AlluxioURI} of the source in the local filesystem * @param dstPath the {@link AlluxioURI} of the destination */ private void copyFromLocal(AlluxioURI srcPath, AlluxioURI dstPath) throws AlluxioException, IOException { File srcFile = new File(srcPath.getPath()); if (srcFile.isDirectory()) { copyFromLocalDir(srcPath, dstPath); } else { copyPath(srcPath, dstPath); } System.out.println("Copied " + srcPath + " to " + dstPath); } /** * Copies a file or directory specified by srcPath from the local filesystem to dstPath in the * Alluxio filesystem space. * * @param srcPath the {@link AlluxioURI} of the source file in the local filesystem * @param dstPath the {@link AlluxioURI} of the destination */ private void copyPath(AlluxioURI srcPath, AlluxioURI dstPath) throws AlluxioException, IOException { File src = new File(srcPath.getPath()); if (!src.isDirectory()) { // If the dstPath is a directory, then it should be updated to be the path of the file where // src will be copied to. if (mFileSystem.exists(dstPath) && mFileSystem.getStatus(dstPath).isFolder()) { dstPath = dstPath.join(src.getName()); } FileOutStream os = null; try (Closer closer = Closer.create()) { FileWriteLocationPolicy locationPolicy; locationPolicy = CommonUtils.createNewClassInstance( Configuration.<FileWriteLocationPolicy>getClass( PropertyKey.USER_FILE_COPY_FROM_LOCAL_WRITE_LOCATION_POLICY), new Class[] {}, new Object[] {}); os = closer.register(mFileSystem.createFile(dstPath, CreateFileOptions.defaults().setLocationPolicy(locationPolicy))); FileInputStream in = closer.register(new FileInputStream(src)); FileChannel channel = closer.register(in.getChannel()); ByteBuffer buf = ByteBuffer.allocate(8 * Constants.MB); while (channel.read(buf) != -1) { buf.flip(); os.write(buf.array(), 0, buf.limit()); } } catch (Exception e) { // Close the out stream and delete the file, so we don't have an incomplete file lying // around. if (os != null) { os.cancel(); if (mFileSystem.exists(dstPath)) { mFileSystem.delete(dstPath); } } throw e; } } else { mFileSystem.createDirectory(dstPath); List<String> errorMessages = new ArrayList<>(); File[] fileList = src.listFiles(); if (fileList == null) { String errMsg = String.format("Failed to list files for directory %s", src); errorMessages.add(errMsg); fileList = new File[0]; } int misFiles = 0; for (File srcFile : fileList) { AlluxioURI newURI = new AlluxioURI(dstPath, new AlluxioURI(srcFile.getName())); try { copyPath(new AlluxioURI(srcPath.getScheme(), srcPath.getAuthority(), srcFile.getPath()), newURI); } catch (IOException e) { errorMessages.add(e.getMessage()); if (!mFileSystem.exists(newURI)) { misFiles++; } } } if (errorMessages.size() != 0) { if (misFiles == fileList.length) { // If the directory doesn't exist and no files were created, then delete the directory if (mFileSystem.exists(dstPath)) { mFileSystem.delete(dstPath); } } throw new IOException(Joiner.on('\n').join(errorMessages)); } } } /** * Copies a list of files or directories specified by srcPaths from the Alluxio filesystem to * dstPath in the local filesystem. This method is used when the input path contains wildcards. * * @param srcPaths the list of files in the Alluxio filesystem * @param dstPath the {@link AlluxioURI} of the destination directory in the local filesystem */ private void copyWildcardToLocal(List<AlluxioURI> srcPaths, AlluxioURI dstPath) throws AlluxioException, IOException { File dstFile = new File(dstPath.getPath()); if (dstFile.exists() && !dstFile.isDirectory()) { throw new InvalidPathException(ExceptionMessage.DESTINATION_CANNOT_BE_FILE.getMessage()); } if (!dstFile.exists()) { if (!dstFile.mkdirs()) { throw new IOException("Fail to create directory: " + dstPath); } else { System.out.println("Create directory: " + dstPath); } } List<String> errorMessages = new ArrayList<>(); for (AlluxioURI srcPath : srcPaths) { try { File dstSubFile = new File(dstFile.getAbsoluteFile(), srcPath.getName()); copyToLocal(srcPath, new AlluxioURI(dstPath.getScheme(), dstPath.getAuthority(), dstSubFile.getPath())); } catch (IOException e) { errorMessages.add(e.getMessage()); } } if (errorMessages.size() != 0) { throw new IOException(Joiner.on('\n').join(errorMessages)); } } /** * Copies a file or a directory from the Alluxio filesystem to the local filesystem. * * @param srcPath the source {@link AlluxioURI} (could be a file or a directory) * @param dstPath the {@link AlluxioURI} of the destination in the local filesystem */ private void copyToLocal(AlluxioURI srcPath, AlluxioURI dstPath) throws AlluxioException, IOException { URIStatus srcStatus = mFileSystem.getStatus(srcPath); File dstFile = new File(dstPath.getPath()); if (srcStatus.isFolder()) { // make a local directory if (!dstFile.exists()) { if (!dstFile.mkdirs()) { throw new IOException("mkdir failure for directory: " + dstPath); } else { System.out.println("Create directory: " + dstPath); } } List<URIStatus> statuses; try { statuses = mFileSystem.listStatus(srcPath); } catch (AlluxioException e) { throw new IOException(e.getMessage()); } List<String> errorMessages = new ArrayList<>(); for (URIStatus status : statuses) { try { File subDstFile = new File(dstFile.getAbsolutePath(), status.getName()); copyToLocal(new AlluxioURI(srcPath.getScheme(), srcPath.getAuthority(), status.getPath()), new AlluxioURI(dstPath.getScheme(), dstPath.getAuthority(), subDstFile.getPath())); } catch (IOException e) { errorMessages.add(e.getMessage()); } } if (errorMessages.size() != 0) { throw new IOException(Joiner.on('\n').join(errorMessages)); } } else { copyFileToLocal(srcPath, dstPath); } } /** * Copies a file specified by argv from the filesystem to the local filesystem. This is the * utility function. * * @param srcPath The source {@link AlluxioURI} (has to be a file) * @param dstPath The {@link AlluxioURI} of the destination in the local filesystem */ private void copyFileToLocal(AlluxioURI srcPath, AlluxioURI dstPath) throws AlluxioException, IOException { File dstFile = new File(dstPath.getPath()); String randomSuffix = String.format(".%s_copyToLocal_", RandomStringUtils.randomAlphanumeric(8)); File outputFile; if (dstFile.isDirectory()) { outputFile = new File(PathUtils.concatPath(dstFile.getAbsolutePath(), srcPath.getName())); } else { outputFile = dstFile; } File tmpDst = new File(outputFile.getPath() + randomSuffix); try (Closer closer = Closer.create()) { OpenFileOptions options = OpenFileOptions.defaults(); FileInStream is = closer.register(mFileSystem.openFile(srcPath, options)); FileOutputStream out = closer.register(new FileOutputStream(tmpDst)); byte[] buf = new byte[64 * Constants.MB]; int t = is.read(buf); while (t != -1) { out.write(buf, 0, t); t = is.read(buf); } if (!tmpDst.renameTo(outputFile)) { throw new IOException( "Failed to rename " + tmpDst.getPath() + " to destination " + outputFile.getPath()); } System.out.println("Copied " + srcPath + " to " + "file://" + outputFile.getPath()); } finally { tmpDst.delete(); } } @Override public String getUsage() { return "cp [-R] <src> <dst>"; } @Override public String getDescription() { return "Copies a file or a directory in the Alluxio filesystem or between local filesystem " + "and Alluxio filesystem. The -R flag is needed to copy directories in the Alluxio " + "filesystem. Local Path with schema \"file\"."; } private static boolean isAlluxio(String scheme) { return Constants.SCHEME.equals(scheme); } private static boolean isFile(String scheme) { return "file".equals(scheme); } }