Java tutorial
/* * Copyright 2011-2013 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.springframework.xd.shell.hadoop; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import javax.annotation.PostConstruct; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.FsShell; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Trash; import org.springframework.shell.core.ExecutionProcessor; import org.springframework.shell.core.annotation.CliCommand; import org.springframework.shell.core.annotation.CliOption; import org.springframework.shell.event.ParseResult; import org.springframework.stereotype.Component; import org.springframework.util.ReflectionUtils; /** * HDFS shell commands * * @author Jarred Li * @author Glenn Renfro * */ @Component public class FsShellCommands extends ConfigurationAware implements ExecutionProcessor { private static final String PREFIX = "hadoop fs "; private static final String FALSE = "false"; private static final String TRUE = "true"; private static final String RECURSIVE = "recursive"; private static final String RECURSION_HELP = "whether with recursion"; private static final String PATH = "path"; private static final String FROM = "from"; private static final String SOURCE_FILE_NAMES = "source file names"; private static final String DESTINATION_PATH_NAME = "destination path name"; private FsShell shell; @PostConstruct public void init() { shell = new FsShell(getHadoopConfiguration()); } @Override protected String failedComponentName() { return "shell"; } @Override protected boolean configurationChanged() { if (shell != null) { LOG.info("Hadoop configuration changed, re-initializing shell..."); } init(); return true; } @Override public ParseResult beforeInvocation(ParseResult invocationContext) { ParseResult result = super.beforeInvocation(invocationContext); String defaultNameKey = (String) ReflectionUtils .getField(ReflectionUtils.findField(FileSystem.class, "FS_DEFAULT_NAME_KEY"), null); String fs = getHadoopConfiguration().get(defaultNameKey); if (fs != null && fs.length() > 0) { return result; } else { LOG.error( "You must set namenode URL before running fs commands. Use `hadoop config fs` to configure namenode URL."); throw new IllegalStateException("You must set fs URL before running fs commands"); } } @CliCommand(value = PREFIX + "ls", help = "List files in the directory") public void ls(@CliOption(key = { "", "dir" }, mandatory = false, unspecifiedDefaultValue = ".", help = "directory to be listed") final String path, @CliOption(key = { RECURSIVE }, mandatory = false, specifiedDefaultValue = TRUE, unspecifiedDefaultValue = FALSE, help = RECURSION_HELP) final boolean recursive) { if (recursive) { List<String> argv = new ArrayList<String>(); argv.add("-ls"); argv.add("-R"); String[] fileNames = path.split(" "); argv.addAll(Arrays.asList(fileNames)); run(argv.toArray(new String[0])); } else { runCommand("-ls", path); } } @CliCommand(value = PREFIX + "cat", help = "Copy source paths to stdout") public void cat(@CliOption(key = { "", PATH }, mandatory = true, unspecifiedDefaultValue = ".", help = "file name to be shown") final String path) { runCommand("-cat", path); } @CliCommand(value = PREFIX + "chgrp", help = "Change group association of files") public void chgrp(@CliOption(key = { RECURSIVE }, mandatory = false, specifiedDefaultValue = TRUE, unspecifiedDefaultValue = FALSE, help = RECURSION_HELP) final boolean recursive, @CliOption(key = { "group" }, mandatory = true, help = "group name") final String group, @CliOption(key = { "", PATH }, mandatory = true, help = "path of the file whose group will be changed") final String path) { List<String> argv = new ArrayList<String>(); argv.add("-chgrp"); if (recursive) { argv.add("-R"); } argv.add(group); String[] fileNames = path.split(" "); argv.addAll(Arrays.asList(fileNames)); run(argv.toArray(new String[0])); } @CliCommand(value = PREFIX + "chown", help = "Change the owner of files") public void chown(@CliOption(key = { RECURSIVE }, mandatory = false, specifiedDefaultValue = TRUE, unspecifiedDefaultValue = FALSE, help = RECURSION_HELP) final boolean recursive, @CliOption(key = { "owner" }, mandatory = true, help = "owner name") final String owner, @CliOption(key = { "", PATH }, mandatory = true, help = "path of the file whose ownership will be changed") final String path) { List<String> argv = new ArrayList<String>(); argv.add("-chown"); if (recursive) { argv.add("-R"); } argv.add(owner); String[] fileNames = path.split(" "); argv.addAll(Arrays.asList(fileNames)); run(argv.toArray(new String[0])); } @CliCommand(value = PREFIX + "chmod", help = "Change the permissions of files") public void chmod(@CliOption(key = { RECURSIVE }, mandatory = false, specifiedDefaultValue = TRUE, unspecifiedDefaultValue = FALSE, help = RECURSION_HELP) final boolean recursive, @CliOption(key = { "mode" }, mandatory = true, help = "permission mode") final String mode, @CliOption(key = { "", PATH }, mandatory = true, help = "path of the file whose permissions will be changed") final String path) { List<String> argv = new ArrayList<String>(); argv.add("-chmod"); if (recursive) { argv.add("-R"); } argv.add(mode); String[] fileNames = path.split(" "); argv.addAll(Arrays.asList(fileNames)); run(argv.toArray(new String[0])); } @CliCommand(value = PREFIX + "copyFromLocal", help = "Copy single src, or multiple srcs from local file system to the destination file system. Same as put") public void copyFromLocal( @CliOption(key = { FROM }, mandatory = true, help = SOURCE_FILE_NAMES) final String source, @CliOption(key = { "to" }, mandatory = true, help = DESTINATION_PATH_NAME) final String dest) { List<String> argv = new ArrayList<String>(); argv.add("-copyFromLocal"); String[] fileNames = source.split(" "); argv.addAll(Arrays.asList(fileNames)); argv.add(dest); run(argv.toArray(new String[0])); } @CliCommand(value = PREFIX + "put", help = "Copy single src, or multiple srcs from local file system to the destination file system") public void put(@CliOption(key = { FROM }, mandatory = true, help = SOURCE_FILE_NAMES) final String source, @CliOption(key = { "to" }, mandatory = true, help = DESTINATION_PATH_NAME) final String dest) { List<String> argv = new ArrayList<String>(); argv.add("-put"); String[] fileNames = source.split(" "); argv.addAll(Arrays.asList(fileNames)); argv.add(dest); run(argv.toArray(new String[0])); } @CliCommand(value = PREFIX + "moveFromLocal", help = "Similar to put command, except that the source localsrc is deleted after it's copied") public void moveFromLocal( @CliOption(key = { FROM }, mandatory = true, help = SOURCE_FILE_NAMES) final String source, @CliOption(key = { "to" }, mandatory = true, help = DESTINATION_PATH_NAME) final String dest) { List<String> argv = new ArrayList<String>(); argv.add("-moveFromLocal"); String[] fileNames = source.split(" "); argv.addAll(Arrays.asList(fileNames)); argv.add(dest); run(argv.toArray(new String[0])); } @CliCommand(value = PREFIX + "copyToLocal", help = "Copy files to the local file system. Same as get") public void copyToLocal( @CliOption(key = { FROM }, mandatory = true, help = SOURCE_FILE_NAMES) final String source, @CliOption(key = { "to" }, mandatory = true, help = DESTINATION_PATH_NAME) final String dest, @CliOption(key = { "ignoreCrc" }, mandatory = false, specifiedDefaultValue = TRUE, unspecifiedDefaultValue = FALSE, help = "whether ignore CRC") final boolean ignoreCrc, @CliOption(key = { "crc" }, mandatory = false, specifiedDefaultValue = TRUE, unspecifiedDefaultValue = FALSE, help = "whether copy CRC") final boolean crc) { List<String> argv = new ArrayList<String>(); argv.add("-copyToLocal"); if (ignoreCrc) { argv.add("-ignoreCrc"); } if (crc) { argv.add("-crc"); } argv.add(source); argv.add(dest); run(argv.toArray(new String[0])); } @CliCommand(value = PREFIX + "copyMergeToLocal", help = "Takes a source directory and a destination file as input and concatenates files in src into the destination local file") public void copyMergeToLocal( @CliOption(key = { FROM }, mandatory = true, help = SOURCE_FILE_NAMES) final String source, @CliOption(key = { "to" }, mandatory = true, help = DESTINATION_PATH_NAME) final String dest, @CliOption(key = { "endline" }, mandatory = false, specifiedDefaultValue = TRUE, unspecifiedDefaultValue = FALSE, help = "whether add a newline character at the end of each file") final boolean endline) { List<String> argv = new ArrayList<String>(); argv.add("-getmerge"); argv.add(source); argv.add(dest); if (endline) { argv.add(TRUE); } run(argv.toArray(new String[0])); } @CliCommand(value = PREFIX + "get", help = "Copy files to the local file system") public void get(@CliOption(key = { FROM }, mandatory = true, help = SOURCE_FILE_NAMES) final String source, @CliOption(key = { "to" }, mandatory = true, help = DESTINATION_PATH_NAME) final String dest, @CliOption(key = { "ignoreCrc" }, mandatory = false, specifiedDefaultValue = TRUE, unspecifiedDefaultValue = FALSE, help = "whether ignore CRC") final boolean ignoreCrc, @CliOption(key = { "crc" }, mandatory = false, specifiedDefaultValue = TRUE, unspecifiedDefaultValue = FALSE, help = "whether copy CRC") final boolean crc) { List<String> argv = new ArrayList<String>(); argv.add("-get"); if (ignoreCrc) { argv.add("-ignoreCrc"); } if (crc) { argv.add("-crc"); } argv.add(source); argv.add(dest); run(argv.toArray(new String[0])); } @CliCommand(value = PREFIX + "count", help = "Count the number of directories, files, bytes, quota, and remaining quota") public void count(@CliOption(key = { "quota" }, mandatory = false, specifiedDefaultValue = TRUE, unspecifiedDefaultValue = FALSE, help = "whether with quta information") final boolean quota, @CliOption(key = { PATH }, mandatory = true, help = "path name") final String path) { List<String> argv = new ArrayList<String>(); argv.add("-count"); if (quota) { argv.add("-q"); } String[] fileNames = path.split(" "); argv.addAll(Arrays.asList(fileNames)); run(argv.toArray(new String[0])); } @CliCommand(value = PREFIX + "cp", help = "Copy files from source to destination. This command allows multiple sources as well in which case the destination must be a directory") public void cp(@CliOption(key = { FROM }, mandatory = true, help = SOURCE_FILE_NAMES) final String source, @CliOption(key = { "to" }, mandatory = true, help = DESTINATION_PATH_NAME) final String dest) { List<String> argv = new ArrayList<String>(); argv.add("-cp"); String[] fileNames = source.split(" "); argv.addAll(Arrays.asList(fileNames)); argv.add(dest); run(argv.toArray(new String[0])); } @CliCommand(value = PREFIX + "mv", help = "Move source files to destination in the HDFS") public void mv(@CliOption(key = { FROM }, mandatory = true, help = SOURCE_FILE_NAMES) final String source, @CliOption(key = { "to" }, mandatory = true, help = DESTINATION_PATH_NAME) final String dest) { List<String> argv = new ArrayList<String>(); argv.add("-mv"); String[] fileNames = source.split(" "); argv.addAll(Arrays.asList(fileNames)); argv.add(dest); run(argv.toArray(new String[0])); } @CliCommand(value = PREFIX + "du", help = "Displays sizes of files and directories contained in the given directory or the length of a file in case its just a file") public void du(@CliOption(key = { "", "dir" }, mandatory = false, unspecifiedDefaultValue = ".", help = "directory to be listed") final String path, @CliOption(key = { "summary" }, mandatory = false, specifiedDefaultValue = TRUE, unspecifiedDefaultValue = FALSE, help = "whether with summary") final boolean summary) { List<String> argv = new ArrayList<String>(); if (summary) { argv.add("-dus"); } else { argv.add("-du"); } argv.add(path); run(argv.toArray(new String[0])); } @CliCommand(value = PREFIX + "expunge", help = "Empty the trash") public void expunge() { List<String> argv = new ArrayList<String>(); argv.add("-expunge"); run(argv.toArray(new String[0])); } @CliCommand(value = PREFIX + "mkdir", help = "Create a new directory") public void mkdir(@CliOption(key = { "", "dir" }, mandatory = true, help = "directory name") final String dir) { List<String> argv = new ArrayList<String>(); argv.add("-mkdir"); argv.add(dir); run(argv.toArray(new String[0])); } @CliCommand(value = PREFIX + "rm", help = "Remove files in the HDFS") public void rm(@CliOption(key = { "", PATH }, mandatory = false, unspecifiedDefaultValue = ".", help = "path to be deleted") final String path, @CliOption(key = { "skipTrash" }, mandatory = false, specifiedDefaultValue = TRUE, unspecifiedDefaultValue = FALSE, help = "whether to skip trash") final boolean skipTrash, @CliOption(key = { RECURSIVE }, mandatory = false, specifiedDefaultValue = TRUE, unspecifiedDefaultValue = FALSE, help = "whether to recurse") final boolean recursive) { try { Path file = new Path(path); FileSystem fs = file.getFileSystem(getHadoopConfiguration()); for (Path p : FileUtil.stat2Paths(fs.globStatus(file), file)) { FileStatus status = fs.getFileStatus(p); if (status.isDirectory() && !recursive) { LOG.error("To remove directory, please use 'fs rm </path/to/dir> --recursive' instead"); return; } if (!skipTrash) { Trash trash = new Trash(fs, getHadoopConfiguration()); trash.moveToTrash(p); } fs.delete(p, recursive); } } catch (Exception t) { LOG.error("Exception: run HDFS shell failed. Message is: " + t.getMessage()); } catch (Error t) { LOG.error("Error: run HDFS shell failed. Message is: " + t.getMessage()); } } @CliCommand(value = PREFIX + "setrep", help = "Change the replication factor of a file") public void setrep(@CliOption(key = { PATH }, mandatory = true, help = "path name") final String path, @CliOption(key = { "replica" }, mandatory = true, help = SOURCE_FILE_NAMES) final int replica, @CliOption(key = { RECURSIVE }, mandatory = false, specifiedDefaultValue = TRUE, unspecifiedDefaultValue = FALSE, help = RECURSION_HELP) final boolean recursive, @CliOption(key = { "waiting" }, mandatory = false, specifiedDefaultValue = TRUE, unspecifiedDefaultValue = FALSE, help = "whether wait for the replic number is eqal to the number") final boolean waiting) { List<String> argv = new ArrayList<String>(); argv.add("-setrep"); if (recursive) { argv.add("-R"); } if (waiting) { argv.add("-w"); } argv.add(String.valueOf(replica)); argv.add(path); run(argv.toArray(new String[0])); } @CliCommand(value = PREFIX + "tail", help = "Display last kilobyte of the file to stdout") public void tail( @CliOption(key = { "", "file" }, mandatory = true, help = "file to be tailed") final String path, @CliOption(key = { "follow" }, mandatory = false, specifiedDefaultValue = TRUE, unspecifiedDefaultValue = FALSE, help = "whether show content while file grow") final boolean file) { List<String> argv = new ArrayList<String>(); argv.add("-tail"); if (file) { argv.add("-f"); } argv.add(path); run(argv.toArray(new String[0])); } @CliCommand(value = PREFIX + "text", help = "Take a source file and output the file in text format") public void text( @CliOption(key = { "", "file" }, mandatory = true, help = "file to be shown") final String path) { List<String> argv = new ArrayList<String>(); argv.add("-text"); argv.add(path); run(argv.toArray(new String[0])); } @CliCommand(value = PREFIX + "touchz", help = "Create a file of zero length") public void touchz( @CliOption(key = { "", "file" }, mandatory = true, help = "file to be touched") final String path) { List<String> argv = new ArrayList<String>(); argv.add("-touchz"); argv.add(path); run(argv.toArray(new String[0])); } /** * @param value */ private void runCommand(String command, String value) { List<String> argv = new ArrayList<String>(); argv.add(command); String[] fileNames = value.split(" "); argv.addAll(Arrays.asList(fileNames)); run(argv.toArray(new String[0])); } private void run(String[] argv) { try { shell.run(argv); } catch (Error t) { LOG.error("Severe: run HDFS shell failed. Message is: " + t.getMessage()); if (t.getCause() != null) { LOG.error("root error message is:" + t.getCause().getMessage()); } } catch (Exception t) { LOG.error("Exception: run HDFS shell failed. Message is: " + t.getMessage()); if (t.getCause() != null) { LOG.error("root error message is:" + t.getCause().getMessage()); } } } }