Java tutorial
/* * Copyright [2013-2014] PayPal Software Foundation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package ml.shifu.guagua.mapreduce; import java.io.IOException; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import ml.shifu.guagua.GuaguaConstants; import ml.shifu.guagua.coordinator.zk.ZooKeeperUtils; import ml.shifu.guagua.hadoop.io.GuaguaOptionsParser; import ml.shifu.guagua.hadoop.io.GuaguaWritableSerializer; import ml.shifu.guagua.hadoop.util.HDPUtils; import ml.shifu.guagua.io.Bytable; import ml.shifu.guagua.io.HaltBytable; import ml.shifu.guagua.master.MasterComputable; import ml.shifu.guagua.util.ReflectionUtils; import ml.shifu.guagua.worker.WorkerComputable; import org.apache.commons.cli.CommandLine; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapred.JobClient; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.RunningJob; import org.apache.hadoop.mapreduce.InputFormat; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.lib.jobcontrol.ControlledJob; import org.apache.hadoop.mapreduce.lib.jobcontrol.JobControl; import org.apache.hadoop.util.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * {@link GuaguaMapReduceClient} is the entry point for guagua mapreduce implementation application. * * <p> * To use it in normal Hadoop mode. Use {@link #main(String[])} as entry point. * * <p> * To run jobs in parallel: * * <pre> * GuaguaMapReduceClient client = new GuaguaMapReduceClient(); * client.addJob(args); * client.addJob(args); * client.run(); * </pre> * * <p> * WARNING: In one GuaguaMapReduceClient instance, {@link #addJob(String[])} to make sure job names are no duplicated. * * <p> * If one job is failed, it will be re-submitted again and try, if failed times over two, no re-try. */ public class GuaguaMapReduceClient { static { // pick up new conf XML file and populate it with stuff exported from client Configuration.addDefaultResource(GuaguaConstants.GUAGUA_SITE_FILE); } private static final Logger LOG = LoggerFactory.getLogger(GuaguaMapReduceClient.class); private static final String INIT_JOB_ID_PREFIX = "Guagua-MapReduce-"; private static String embededZooKeeperServer = null; /** * Make guagua run jobs in parallel. */ private JobControl jc; private int jobIndex = 0; private Map<String, Integer> jobIndexMap = new HashMap<String, Integer>(); private Map<Integer, Integer> jobRunningTimes = new HashMap<Integer, Integer>(); private Map<Integer, String[]> jobIndexParams = new HashMap<Integer, String[]>(); private Set<String> failedCheckingJobs = new HashSet<String>(); /** * Default constructor. Construct default JobControl instance. */ public GuaguaMapReduceClient() { this.jc = new JobControl(INIT_JOB_ID_PREFIX); } /** * Add new job to JobControl instance. */ public synchronized void addJob(String[] args) throws IOException { Job job = createJob(args); this.jc.addJob(new ControlledJob(job, null)); if (this.jobIndexMap.containsKey(job.getJobName())) { throw new IllegalStateException( "Job name should be unique. please check name with: " + job.getJobName()); } this.jobIndexMap.put(job.getJobName(), this.jobIndex); this.jobIndexParams.put(this.jobIndex, args); this.jobRunningTimes.put(this.jobIndex, 1); this.jobIndex += 1; } /** * Run all jobs added to JobControl. */ public void run() throws IOException { // Initially, all jobs are in wait state. List<ControlledJob> jobsWithoutIds = this.jc.getWaitingJobList(); int totalNeededMRJobs = jobsWithoutIds.size(); LOG.info("{} map-reduce job(s) waiting for submission.", jobsWithoutIds.size()); Thread jcThread = new Thread(this.jc, "Guagua-MapReduce-JobControl"); jcThread.start(); JobClient jobClient = new JobClient(new JobConf(new Configuration())); double lastProg = -1; Set<String> sucessfulJobs = new HashSet<String>(); while (!this.jc.allFinished()) { try { jcThread.join(1000); } catch (InterruptedException ignore) { Thread.currentThread().interrupt(); } List<ControlledJob> jobsAssignedIdInThisRun = new ArrayList<ControlledJob>(totalNeededMRJobs); for (ControlledJob job : jobsWithoutIds) { if (job.getJob().getJobID() != null) { jobsAssignedIdInThisRun.add(job); LOG.info("Job {} is started.", job.getJob().getJobID().toString()); } else { // This job is not assigned an id yet. } } jobsWithoutIds.removeAll(jobsAssignedIdInThisRun); List<ControlledJob> successfulJobs = jc.getSuccessfulJobList(); for (ControlledJob controlledJob : successfulJobs) { String jobId = controlledJob.getJob().getJobID().toString(); if (!sucessfulJobs.contains(jobId)) { LOG.info("Job {} is successful.", jobId); sucessfulJobs.add(jobId); } } List<ControlledJob> failedJobs = jc.getFailedJobList(); for (ControlledJob controlledJob : failedJobs) { String failedJobId = controlledJob.getJob().getJobID().toString(); if (!this.failedCheckingJobs.contains(failedJobId)) { this.failedCheckingJobs.add(failedJobId); String jobName = controlledJob.getJob().getJobName(); Integer jobIndex = this.jobIndexMap.get(jobName); Integer runTimes = this.jobRunningTimes.get(jobIndex); if (runTimes <= 1) { LOG.warn("Job {} is failed, will be submitted again.", jobName); Job newJob = createJob(this.jobIndexParams.get(jobIndex)); this.jc.addJob(new ControlledJob(newJob, null)); this.jobRunningTimes.put(jobIndex, runTimes + 1); this.jobIndexMap.put(newJob.getJobName(), jobIndex); jobsWithoutIds = this.jc.getWaitingJobList(); } else { LOG.warn("Job {} is failed twice, will not be submitted again.", jobName); } } } double prog = calculateProgress(jc, jobClient) / totalNeededMRJobs; notifyProgress(prog, lastProg); lastProg = prog; try { Thread.sleep(2 * 1000); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } } List<ControlledJob> successfulJobs = jc.getSuccessfulJobList(); LOG.info("Sucessful jobs:"); for (ControlledJob controlledJob : successfulJobs) { LOG.info("Job: {} ", controlledJob); } if (totalNeededMRJobs == successfulJobs.size()) { LOG.info("Guagua jobs: 100% complete"); // add failed jobs to debug since all jobs are finished. List<ControlledJob> failedJobs = jc.getFailedJobList(); if (failedJobs != null && failedJobs.size() > 0) { LOG.debug("Failed jobs:"); for (ControlledJob controlledJob : failedJobs) { LOG.debug("Job: {} ", controlledJob); } } } else { List<ControlledJob> failedJobs = jc.getFailedJobList(); if (failedJobs != null && failedJobs.size() > 0) { LOG.info("Failed jobs:"); for (ControlledJob controlledJob : failedJobs) { LOG.warn("Job: {} ", controlledJob); } } } this.jc.stop(); } /** * Log the progress and notify listeners if there is sufficient progress * * @param prog * current progress * @param lastProg * progress last time */ private void notifyProgress(double prog, double lastProg) { if (prog >= (lastProg + 0.01)) { int perCom = (int) (prog * 100); if (perCom != 100) { LOG.info("Guagua jobs: {}% complete", perCom); } } } /** * Compute the progress of the current job submitted through the JobControl object jc to the JobClient jobClient * * @param jc * The JobControl object that has been submitted * @param jobClient * The JobClient to which it has been submitted * @return The progress as a percentage in double format * @throws IOException * In case any IOException connecting to JobTracker. */ protected double calculateProgress(JobControl jc, JobClient jobClient) throws IOException { double prog = 0.0; prog += jc.getSuccessfulJobList().size(); List<ControlledJob> runnJobs = jc.getRunningJobList(); for (ControlledJob cjob : runnJobs) { prog += progressOfRunningJob(cjob, jobClient); } return prog; } /** * Returns the progress of a Job j which is part of a submitted JobControl object. The progress is for this Job. So * it has to be scaled down by the number of jobs that are present in the JobControl. * * @param cjob * - The Job for which progress is required * @param jobClient * - the JobClient to which it has been submitted * @return Returns the percentage progress of this Job * @throws IOException * In case any IOException connecting to JobTracker. */ protected double progressOfRunningJob(ControlledJob cjob, JobClient jobClient) throws IOException { @SuppressWarnings("deprecation") RunningJob rj = jobClient.getJob(cjob.getJob().getJobID().toString()); if (rj == null && cjob.getJobState() == ControlledJob.State.SUCCESS) return 1; else if (rj == null) return 0; else { return rj.mapProgress(); } } public static void addInputPath(Configuration conf, Path path) throws IOException { path = path.getFileSystem(conf).makeQualified(path); String dirStr = StringUtils.escapeString(path.toString()); String dirs = conf.get(GuaguaMapReduceConstants.MAPRED_INPUT_DIR); conf.set(GuaguaMapReduceConstants.MAPRED_INPUT_DIR, dirs == null ? dirStr : dirs + "," + dirStr); } /** * Create Hadoop job according to arguments from main. */ @SuppressWarnings("deprecation") public synchronized Job createJob(String[] args) throws IOException { Configuration conf = new Configuration(); // set it here to make it can be over-written. Set task timeout to a long period 20 minutes. conf.setInt(GuaguaMapReduceConstants.MAPRED_TASK_TIMEOUT, conf.getInt(GuaguaMapReduceConstants.MAPRED_TASK_TIMEOUT, 1800000)); conf.setInt(GuaguaMapReduceConstants.MAPREDUCE_TASK_TIMEOUT, conf.getInt(GuaguaMapReduceConstants.MAPREDUCE_TASK_TIMEOUT, 1800000)); GuaguaOptionsParser parser = new GuaguaOptionsParser(conf, args); // process a bug on hdp 2.2.4 String hdpVersion = HDPUtils.getHdpVersionForHDP224(); if (hdpVersion != null && hdpVersion.length() != 0) { conf.set("hdp.version", hdpVersion); HDPUtils.addFileToClassPath(HDPUtils.findContainingFile("hdfs-site.xml"), conf); HDPUtils.addFileToClassPath(HDPUtils.findContainingFile("core-site.xml"), conf); HDPUtils.addFileToClassPath(HDPUtils.findContainingFile("mapred-site.xml"), conf); HDPUtils.addFileToClassPath(HDPUtils.findContainingFile("yarn-site.xml"), conf); } CommandLine cmdLine = parser.getCommandLine(); checkInputSetting(conf, cmdLine); checkZkServerSetting(conf, cmdLine); checkWorkerClassSetting(conf, cmdLine); checkMasterClassSetting(conf, cmdLine); checkIterationCountSetting(conf, cmdLine); checkResultClassSetting(conf, cmdLine); String name = checkMapReduceNameSetting(cmdLine); @SuppressWarnings("rawtypes") Class<? extends InputFormat> inputFormatClass = checkInputFormatSetting(cmdLine); // set map reduce parameters for specified master-workers architecture // speculative execution should be disabled conf.setBoolean(GuaguaMapReduceConstants.MAPRED_MAP_TASKS_SPECULATIVE_EXECUTION, false); conf.setBoolean(GuaguaMapReduceConstants.MAPRED_REDUCE_TASKS_SPECULATIVE_EXECUTION, false); // set mapreduce.job.max.split.locations to 100 to suppress warnings int maxSplits = conf.getInt(GuaguaMapReduceConstants.MAPREDUCE_JOB_MAX_SPLIT_LOCATIONS, 100); if (maxSplits < 100) { maxSplits = 100; } conf.setInt(GuaguaMapReduceConstants.MAPREDUCE_JOB_MAX_SPLIT_LOCATIONS, maxSplits); // Set cache to 0. conf.setInt(GuaguaMapReduceConstants.IO_SORT_MB, 0); // Most users won't hit this hopefully and can set it higher if desired conf.setInt(GuaguaMapReduceConstants.MAPREDUCE_JOB_COUNTERS_LIMIT, conf.getInt(GuaguaMapReduceConstants.MAPREDUCE_JOB_COUNTERS_LIMIT, 512)); conf.setInt(GuaguaMapReduceConstants.MAPRED_JOB_REDUCE_MEMORY_MB, 0); // append concurrent gc to avoid long gc stop-the-world String childJavaOpts = conf.get(GuaguaMapReduceConstants.MAPRED_CHILD_JAVA_OPTS, ""); if (childJavaOpts == null || childJavaOpts.length() == 0) { conf.set(GuaguaMapReduceConstants.MAPRED_CHILD_JAVA_OPTS, GuaguaMapReduceConstants.MAPRED_DEFAULT_CHILD_JAVA_OPTS); } else { String newChildJavaOpts = GuaguaMapReduceConstants.MAPRED_DEFAULT_CHILD_JAVA_OPTS + " " + childJavaOpts; conf.set(GuaguaMapReduceConstants.MAPRED_CHILD_JAVA_OPTS, newChildJavaOpts.trim()); } Job job = new Job(conf, name); job.setJarByClass(GuaguaMapReduceClient.class); job.setMapperClass(GuaguaMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Text.class); job.setInputFormatClass(inputFormatClass); job.setOutputFormatClass(GuaguaOutputFormat.class); job.setNumReduceTasks(0); return job; } @SuppressWarnings({ "unchecked", "rawtypes" }) private static Class<? extends InputFormat> checkInputFormatSetting(CommandLine cmdLine) { Class<? extends InputFormat> inputFormatClass = GuaguaInputFormat.class; if (cmdLine.hasOption("-inputformat")) { String inputFormatClassName = cmdLine.getOptionValue("inputformat"); try { inputFormatClass = (Class<? extends InputFormat>) Class.forName(inputFormatClassName.trim()); } catch (ClassNotFoundException e) { printUsage(); throw new IllegalArgumentException(String.format( "The inputformat class %s set by '-inputformat' can not be found in class path.", inputFormatClassName.trim()), e); } catch (ClassCastException e) { printUsage(); throw new IllegalArgumentException( "Mapreduce input format class set by 'inputformat' should extend " + "'org.apache.hadoop.mapreduce.InputFormat' class."); } } return inputFormatClass; } private static String checkMapReduceNameSetting(CommandLine cmdLine) { String name = "Guagua Master-Workers Job"; if (cmdLine.hasOption("-n")) { name = cmdLine.getOptionValue("n"); } return name; } private static void checkResultClassSetting(Configuration conf, CommandLine cmdLine) { Class<?> masterResultClass; if (!cmdLine.hasOption("-mr")) { printUsage(); throw new IllegalArgumentException("Master result class name should be provided by '-mr' parameter."); } else { String resultClassName = cmdLine.getOptionValue("mr").trim(); try { masterResultClass = Class.forName(resultClassName); } catch (ClassNotFoundException e) { throw new IllegalArgumentException(String.format( "Master result class %s set by '-mr' can not be found in class path.", resultClassName), e); } if (Writable.class.isAssignableFrom(masterResultClass)) { conf.set(GuaguaConstants.GUAGUA_MASTER_IO_SERIALIZER, GuaguaWritableSerializer.class.getName()); conf.set(GuaguaConstants.GUAGUA_MASTER_RESULT_CLASS, resultClassName); } else if (Bytable.class.isAssignableFrom(masterResultClass)) { conf.set(GuaguaConstants.GUAGUA_MASTER_RESULT_CLASS, resultClassName); if (!ReflectionUtils.hasEmptyParameterConstructor(masterResultClass)) { throw new IllegalArgumentException( "Master result class should have default constuctor without any parameters."); } } else { printUsage(); throw new IllegalArgumentException( "Master result class name provided by '-mr' parameter should implement " + "'com.paypal.guagua.io.Bytable' or 'org.apache.hadoop.io.Writable'."); } } Class<?> workerResultClass; if (!cmdLine.hasOption("-wr")) { printUsage(); throw new IllegalArgumentException("Worker result class name should be provided by '-wr' parameter."); } else { String resultClassName = cmdLine.getOptionValue("wr").trim(); try { workerResultClass = Class.forName(resultClassName); } catch (ClassNotFoundException e) { printUsage(); throw new IllegalArgumentException(String.format( "Worker result class %s set by '-wr' can not be found in class path.", resultClassName), e); } if (Writable.class.isAssignableFrom(workerResultClass)) { conf.set(GuaguaConstants.GUAGUA_WORKER_IO_SERIALIZER, GuaguaWritableSerializer.class.getName()); conf.set(GuaguaConstants.GUAGUA_WORKER_RESULT_CLASS, resultClassName); } else if (Bytable.class.isAssignableFrom(workerResultClass)) { conf.set(GuaguaConstants.GUAGUA_WORKER_RESULT_CLASS, resultClassName); if (!ReflectionUtils.hasEmptyParameterConstructor(workerResultClass)) { throw new IllegalArgumentException( "Worker result class should have default constuctor without any parameters."); } } else { printUsage(); throw new IllegalArgumentException( "Worker result class name provided by '-wr' parameter should implement " + "'com.paypal.guagua.io.Bytable' or 'org.apache.hadoop.io.Writable'."); } } if (HaltBytable.class.isAssignableFrom(masterResultClass) && !HaltBytable.class.isAssignableFrom(workerResultClass) || HaltBytable.class.isAssignableFrom(workerResultClass) && !HaltBytable.class.isAssignableFrom(masterResultClass)) { printUsage(); throw new IllegalArgumentException( "Worker and master result classes should both implementent HaltBytable."); } } private static void checkIterationCountSetting(Configuration conf, CommandLine cmdLine) { if (!cmdLine.hasOption("-c")) { System.err.println("WARN: Total iteration number is not set, default 50 will be used."); System.err.println( "WARN: Total iteration number can be provided by '-c' parameter with non-empty value."); conf.setInt(GuaguaConstants.GUAGUA_ITERATION_COUNT, GuaguaConstants.GUAGUA_DEFAULT_ITERATION_COUNT); } else { int totalIteration = 0; try { totalIteration = Integer.parseInt(cmdLine.getOptionValue("c").trim()); } catch (NumberFormatException e) { printUsage(); throw new IllegalArgumentException("Total iteration number set by '-c' should be a valid number."); } conf.setInt(GuaguaConstants.GUAGUA_ITERATION_COUNT, totalIteration); } } private static void checkMasterClassSetting(Configuration conf, CommandLine cmdLine) { if (!cmdLine.hasOption("-m")) { printUsage(); throw new IllegalArgumentException("Master class name should be provided by '-m' parameter."); } String masterClassOptionValue = cmdLine.getOptionValue("m"); if (masterClassOptionValue == null || masterClassOptionValue.length() == 0) { printUsage(); throw new IllegalArgumentException( "Master class name should be provided by '-m' parameter with non-empty value."); } Class<?> masterClass; try { masterClass = Class.forName(masterClassOptionValue.trim()); } catch (ClassNotFoundException e) { printUsage(); throw new IllegalArgumentException( String.format("The master class %s set by '-m' can not be found in class path.", masterClassOptionValue.trim()), e); } if (!MasterComputable.class.isAssignableFrom(masterClass)) { printUsage(); throw new IllegalArgumentException( "Master class name provided by '-m' should implement 'com.paypal.guagua.master.MasterComputable' interface."); } if (!ReflectionUtils.hasEmptyParameterConstructor(masterClass)) { throw new IllegalArgumentException( "Master class should have default constuctor without any parameters."); } conf.set(GuaguaConstants.MASTER_COMPUTABLE_CLASS, masterClassOptionValue.trim()); } private static void checkWorkerClassSetting(Configuration conf, CommandLine cmdLine) { if (!cmdLine.hasOption("-w")) { printUsage(); throw new IllegalArgumentException("Worker class name should be provided by '-w' parameter."); } String workerClassOptionValue = cmdLine.getOptionValue("w"); if (workerClassOptionValue == null || workerClassOptionValue.length() == 0) { printUsage(); throw new IllegalArgumentException( "Worker class name should be provided by '-w' parameter with non-empty value."); } Class<?> workerClass; try { workerClass = Class.forName(workerClassOptionValue.trim()); } catch (ClassNotFoundException e) { printUsage(); throw new IllegalArgumentException( String.format("The worker class %s set by '-w' can not be found in class path.", workerClassOptionValue.trim()), e); } if (!WorkerComputable.class.isAssignableFrom(workerClass)) { printUsage(); throw new IllegalArgumentException( "Worker class name provided by '-w' should implement 'com.paypal.guagua.worker.WorkerComputable' interface."); } if (!ReflectionUtils.hasEmptyParameterConstructor(workerClass)) { throw new IllegalArgumentException( "Worker class should have default constuctor without any parameters."); } conf.set(GuaguaConstants.WORKER_COMPUTABLE_CLASS, workerClassOptionValue.trim()); } private static void printUsage() { GuaguaOptionsParser.printGenericCommandUsage(System.out); System.out.println("For detailed invalid parameter, please check:"); } private static void checkZkServerSetting(Configuration conf, CommandLine cmdLine) { if (!cmdLine.hasOption("-z")) { System.err.println("WARN: ZooKeeper server is not set, embeded ZooKeeper server will be started."); System.err.println( "WARN: For big data guagua application with fail-over zookeeper servers, independent ZooKeeper instances are recommended."); System.err.println("WARN: Zookeeper servers can be provided by '-z' parameter with non-empty value."); // change default embedded zookeeper server to master zonde boolean isZkInClient = conf.getBoolean(GuaguaConstants.GUAGUA_ZK_EMBEDBED_IS_IN_CLIENT, false); if (isZkInClient) { synchronized (GuaguaMapReduceClient.class) { if (embededZooKeeperServer == null) { // 1. start embed zookeeper server in one thread. int embedZkClientPort = 0; try { embedZkClientPort = ZooKeeperUtils.startEmbedZooKeeper(); } catch (IOException e) { throw new RuntimeException(e); } // 2. check if it is started. ZooKeeperUtils.checkIfEmbedZooKeeperStarted(embedZkClientPort); try { embededZooKeeperServer = InetAddress.getLocalHost().getHostName() + ":" + embedZkClientPort; } catch (UnknownHostException e) { throw new RuntimeException(e); } } } // 3. set local embed zookeeper server address conf.set(GuaguaConstants.GUAGUA_ZK_SERVERS, embededZooKeeperServer); } else { conf.set(GuaguaConstants.GUAGUA_MASTER_SYSTEM_INTERCEPTERS, conf.get( GuaguaConstants.GUAGUA_MASTER_SYSTEM_INTERCEPTERS, "ml.shifu.guagua.master.MasterTimer,ml.shifu.guagua.master.MemoryStatsMasterInterceptor,ml.shifu.guagua.hadoop.ZooKeeperMasterInterceptor,ml.shifu.guagua.master.NettyMasterCoordinator ")); conf.set(GuaguaConstants.GUAGUA_WORKER_SYSTEM_INTERCEPTERS, conf.get( GuaguaConstants.GUAGUA_WORKER_SYSTEM_INTERCEPTERS, "ml.shifu.guagua.worker.WorkerTimer,ml.shifu.guagua.worker.MemoryStatsWorkerInterceptor,ml.shifu.guagua.hadoop.ZooKeeperWorkerInterceptor,ml.shifu.guagua.worker.NettyWorkerCoordinator")); System.err.println("WARN: Zookeeper server will be started in master node of cluster"); } return; } else { String zkServers = cmdLine.getOptionValue("z"); if (zkServers == null || zkServers.length() == 0) { throw new IllegalArgumentException( "Zookeeper servers should be provided by '-z' parameter with non-empty value."); } if (ZooKeeperUtils.checkServers(zkServers)) { conf.set(GuaguaConstants.GUAGUA_ZK_SERVERS, zkServers.trim()); } else { throw new RuntimeException("Your specifed zookeeper instance is not alive, please check."); } } } private static void checkInputSetting(Configuration conf, CommandLine cmdLine) throws IOException { if (!cmdLine.hasOption("-i")) { printUsage(); throw new IllegalArgumentException("Input should be provided by '-i' parameter."); } addInputPath(conf, new Path(cmdLine.getOptionValue("i").trim())); } public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException { if (args.length == 0 || (args.length == 1 && (args[0].equals("h") || args[0].equals("-h") || args[0].equals("-help") || args[0].equals("help")))) { GuaguaOptionsParser.printGenericCommandUsage(System.out); System.exit(0); } GuaguaMapReduceClient client = new GuaguaMapReduceClient(); Job job = client.createJob(args); job.waitForCompletion(true); } }