List of usage examples for org.apache.hadoop.mapreduce Job isComplete
public boolean isComplete() throws IOException
From source file:hr.fer.tel.rovkp.homework02.task03.Program.java
public static void main(String[] args) throws Exception { if (args.length != 2) { System.err.println("Usage: <jar> <input path> <output path>"); return;/*from w w w . java 2 s.c o m*/ } Job firstJob = Job.getInstance(); firstJob.setJarByClass(Program.class); firstJob.setJobName("Locations"); FileInputFormat.addInputPath(firstJob, new Path(args[0])); FileOutputFormat.setOutputPath(firstJob, new Path(INTERMEDIATE_PATH)); firstJob.setMapperClass(LocationsMapper.class); firstJob.setPartitionerClass(LocationsPartitioner.class); firstJob.setReducerClass(LocationsReducer.class); firstJob.setNumReduceTasks(6); firstJob.setOutputKeyClass(IntWritable.class); firstJob.setOutputValueClass(Text.class); MultipleOutputs.addNamedOutput(firstJob, "bins", TextOutputFormat.class, NullWritable.class, Text.class); int code = firstJob.waitForCompletion(true) ? 0 : 1; System.out.println("First job return code: " + code); if (code == 0) { Job job1 = run(INTERMEDIATE_PATH + "center1", args[1] + "/1"); Job job2 = run(INTERMEDIATE_PATH + "not_center1", args[1] + "/2"); Job job3 = run(INTERMEDIATE_PATH + "center2", args[1] + "/3"); Job job4 = run(INTERMEDIATE_PATH + "not_center2", args[1] + "/4"); Job job5 = run(INTERMEDIATE_PATH + "center4", args[1] + "/5"); Job job6 = run(INTERMEDIATE_PATH + "not_center4", args[1] + "/6"); while (!(job1.isComplete() && job2.isComplete() && job3.isComplete() && job4.isComplete() && job5.isComplete() && job6.isComplete())) { Thread.sleep(2000); } } FileSystem.get(firstJob.getConfiguration()).delete(new Path(INTERMEDIATE_PATH), true); }
From source file:io.dataapps.chlorine.hadoop.AbstractPipeline.java
License:Apache License
private synchronized boolean waitForCompletion(Job j, boolean verbose) throws IOException, InterruptedException, ClassNotFoundException { j.submit();//from w w w .ja va2 s .com while (!j.isComplete()) { try { wait(5000); } catch (InterruptedException ie) { } } return j.isSuccessful(); }
From source file:io.hops.erasure_coding.MapReduceBlockRepairManager.java
License:Apache License
@Override public List<Report> computeReports() { List<Report> reports = new ArrayList<Report>(); for (Map.Entry<String, Job> entry : currentRepairs.entrySet()) { String fileName = entry.getKey(); Job job = entry.getValue(); try {//from w w w. j a va2 s . co m if (job.isComplete() && job.isSuccessful()) { LOG.info("REPAIR COMPLETE"); reports.add(new Report(fileName, Report.Status.FINISHED)); cleanup(job); } else if (job.isComplete() && !job.isSuccessful()) { LOG.info("REPAIR FAILED"); reports.add(new Report(fileName, Report.Status.FAILED)); cleanup(job); } /* TODO FIX timeout else if (System.currentTimeMillis() - job.getStartTime() > getMaxFixTimeForFile()) { LOG.info("Timeout: " + (System.currentTimeMillis() - job.getStartTime()) + " " + job.getStartTime()); job.killJob(); reports.add(new Report(fileName, Report.Status.CANCELED)); cleanup(job); }*/ else { LOG.info("REPAIR RUNNING"); reports.add(new Report(fileName, Report.Status.ACTIVE)); } } catch (Exception e) { LOG.info("Exception during completeness check", e); try { job.killJob(); } catch (Exception e1) { } reports.add(new Report(fileName, Report.Status.FAILED)); cleanup(job); } } for (Report report : reports) { Report.Status status = report.getStatus(); if (status == Report.Status.FINISHED || status == Report.Status.FAILED || status == Report.Status.CANCELED) { currentRepairs.remove(report.getFilePath()); } } return reports; }
From source file:mvm.rya.accumulo.mr.fileinput.BulkNtripsInputTool.java
License:Apache License
@Override public int run(final String[] args) throws Exception { final Configuration conf = getConf(); try {/* w w w. ja v a2 s . c o m*/ //conf zk = conf.get(MRUtils.AC_ZK_PROP, zk); ttl = conf.get(MRUtils.AC_TTL_PROP, ttl); instance = conf.get(MRUtils.AC_INSTANCE_PROP, instance); userName = conf.get(MRUtils.AC_USERNAME_PROP, userName); pwd = conf.get(MRUtils.AC_PWD_PROP, pwd); workDirBase = conf.get(WORKDIR_PROP, workDirBase); format = conf.get(MRUtils.FORMAT_PROP, format); conf.set(MRUtils.FORMAT_PROP, format); final String inputDir = args[0]; ZooKeeperInstance zooKeeperInstance = new ZooKeeperInstance(instance, zk); Connector connector = zooKeeperInstance.getConnector(userName, new PasswordToken(pwd)); TableOperations tableOperations = connector.tableOperations(); if (conf.get(AccumuloRdfConfiguration.CONF_ADDITIONAL_INDEXERS) != null) { throw new IllegalArgumentException("Cannot use Bulk N Trips tool with Additional Indexers"); } String tablePrefix = conf.get(MRUtils.TABLE_PREFIX_PROPERTY, null); if (tablePrefix != null) RdfCloudTripleStoreConstants.prefixTables(tablePrefix); String[] tables = { tablePrefix + RdfCloudTripleStoreConstants.TBL_OSP_SUFFIX, tablePrefix + RdfCloudTripleStoreConstants.TBL_SPO_SUFFIX, tablePrefix + RdfCloudTripleStoreConstants.TBL_PO_SUFFIX }; Collection<Job> jobs = new ArrayList<Job>(); for (final String tableName : tables) { PrintStream out = null; try { String workDir = workDirBase + "/" + tableName; System.out.println("Loading data into table[" + tableName + "]"); Job job = new Job(new Configuration(conf), "Bulk Ingest load data to Generic RDF Table[" + tableName + "]"); job.setJarByClass(this.getClass()); //setting long job Configuration jobConf = job.getConfiguration(); jobConf.setBoolean("mapred.map.tasks.speculative.execution", false); jobConf.setBoolean("mapred.reduce.tasks.speculative.execution", false); jobConf.set("io.sort.mb", jobConf.get("io.sort.mb", "256")); jobConf.setBoolean("mapred.compress.map.output", true); // jobConf.set("mapred.map.output.compression.codec", "org.apache.hadoop.io.compress.GzipCodec"); //TODO: I would like LZO compression job.setInputFormatClass(TextInputFormat.class); job.setMapperClass(ParseNtripsMapper.class); job.setMapOutputKeyClass(Key.class); job.setMapOutputValueClass(Value.class); job.setCombinerClass(OutStmtMutationsReducer.class); job.setReducerClass(OutStmtMutationsReducer.class); job.setOutputFormatClass(AccumuloFileOutputFormat.class); // AccumuloFileOutputFormat.setZooKeeperInstance(jobConf, instance, zk); jobConf.set(ParseNtripsMapper.TABLE_PROPERTY, tableName); TextInputFormat.setInputPaths(job, new Path(inputDir)); FileSystem fs = FileSystem.get(conf); Path workPath = new Path(workDir); if (fs.exists(workPath)) fs.delete(workPath, true); //make failures dir Path failures = new Path(workDir, "failures"); fs.delete(failures, true); fs.mkdirs(new Path(workDir, "failures")); AccumuloFileOutputFormat.setOutputPath(job, new Path(workDir + "/files")); out = new PrintStream(new BufferedOutputStream(fs.create(new Path(workDir + "/splits.txt")))); if (!tableOperations.exists(tableName)) tableOperations.create(tableName); Collection<Text> splits = tableOperations.getSplits(tableName, Integer.MAX_VALUE); for (Text split : splits) out.println(new String(Base64.encodeBase64(TextUtil.getBytes(split)))); job.setNumReduceTasks(splits.size() + 1); out.close(); job.setPartitionerClass(KeyRangePartitioner.class); RangePartitioner.setSplitFile(job, workDir + "/splits.txt"); jobConf.set(WORKDIR_PROP, workDir); job.submit(); jobs.add(job); } catch (Exception re) { throw new RuntimeException(re); } finally { if (out != null) out.close(); } } for (Job job : jobs) { while (!job.isComplete()) { Thread.sleep(1000); } } for (String tableName : tables) { String workDir = workDirBase + "/" + tableName; String filesDir = workDir + "/files"; String failuresDir = workDir + "/failures"; FileSystem fs = FileSystem.get(conf); //make sure that the "accumulo" user can read/write/execute into these directories this path fs.setPermission(new Path(filesDir), new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)); fs.setPermission(new Path(failuresDir), new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)); tableOperations.importDirectory(tableName, filesDir, failuresDir, false); } } catch (Exception e) { throw new RuntimeException(e); } return 0; }
From source file:org.apache.blur.mapreduce.lib.BlurOutputFormatTest.java
License:Apache License
public void testBlurOutputFormatCleanupDuringJobKillTest() throws IOException, InterruptedException, ClassNotFoundException { Path input = getInDir();/* www .j a v a 2 s . com*/ Path output = getOutDir(); _fileSystem.delete(input, true); _fileSystem.delete(output, true); // 1500 * 50 = 75,000 writeRecordsFile(new Path(input, "part1"), 1, 50, 1, 1500, "cf1"); // 100 * 5000 = 500,000 writeRecordsFile(new Path(input, "part2"), 1, 5000, 2000, 100, "cf1"); Job job = Job.getInstance(_conf, "blur index"); job.setJarByClass(BlurOutputFormatTest.class); job.setMapperClass(CsvBlurMapper.class); job.setInputFormatClass(TextInputFormat.class); FileInputFormat.addInputPath(job, input); CsvBlurMapper.addColumns(job, "cf1", "col"); Path tablePath = new Path(new Path(_root, "table"), "test"); TableDescriptor tableDescriptor = new TableDescriptor(); tableDescriptor.setShardCount(2); tableDescriptor.setTableUri(tablePath.toString()); tableDescriptor.setName("test"); createShardDirectories(getOutDir(), 2); BlurOutputFormat.setupJob(job, tableDescriptor); BlurOutputFormat.setOutputPath(job, output); BlurOutputFormat.setIndexLocally(job, false); job.submit(); boolean killCalled = false; while (!job.isComplete()) { Thread.sleep(1000); System.out.printf("Killed [" + killCalled + "] Map [%f] Reduce [%f]%n", job.mapProgress() * 100, job.reduceProgress() * 100); if (job.reduceProgress() > 0.7 && !killCalled) { job.killJob(); killCalled = true; } } assertFalse(job.isSuccessful()); for (int i = 0; i < tableDescriptor.getShardCount(); i++) { Path path = new Path(output, ShardUtil.getShardName(i)); FileSystem fileSystem = path.getFileSystem(job.getConfiguration()); FileStatus[] listStatus = fileSystem.listStatus(path); assertEquals(toString(listStatus), 0, listStatus.length); } }
From source file:org.apache.crunch.impl.mr.exec.MRExecutorIT.java
License:Apache License
/** * Tests that the pipeline should be stopped immediately when one of the jobs * get failed. The rest of running jobs should be killed. *//*from w ww . ja v a 2 s. c o m*/ @Test public void testStopPipelineImmediatelyOnJobFailure() throws Exception { String inPath = tmpDir.copyResourceFileName("shakes.txt"); MRPipeline pipeline = new MRPipeline(MRExecutorIT.class); // Issue two jobs that sleep forever. PCollection<String> in = pipeline.read(From.textFile(inPath)); for (int i = 0; i < 2; i++) { in.count().values().parallelDo(new SleepForeverFn(), longs()) .write(To.textFile(tmpDir.getPath("out_" + i))); } MRPipelineExecution exec = pipeline.runAsync(); // Wait until both of the two jobs are submitted. List<MRJob> jobs = exec.getJobs(); assertEquals(2, jobs.size()); StopWatch watch = new StopWatch(); watch.start(); int numOfJobsSubmitted = 0; while (numOfJobsSubmitted < 2 && watch.getTime() < 10000) { numOfJobsSubmitted = 0; for (MRJob job : jobs) { if (job.getJobState() == MRJob.State.RUNNING) { numOfJobsSubmitted++; } } Thread.sleep(100); } assertEquals(2, numOfJobsSubmitted); // Kill one of them. Job job0 = jobs.get(0).getJob(); job0.killJob(); // Expect the pipeline exits and the other job is killed. StopWatch watch2 = new StopWatch(); watch2.start(); Job job1 = jobs.get(1).getJob(); while (!job1.isComplete() && watch2.getTime() < 10000) { Thread.sleep(100); } assertTrue(job1.isComplete()); assertEquals(PipelineExecution.Status.FAILED, exec.getStatus()); }
From source file:org.apache.giraph.utils.CounterUtils.java
License:Apache License
/** * Wait for a counter to appear in a group and then return the name of that * counter. If job finishes before counter appears, return null. * * @param job Job// ww w .j a v a 2 s .c o m * @param group Name of the counter group * @return Name of the counter inside of the group, or null if job finishes * before counter appears */ public static String waitAndGetCounterNameFromGroup(Job job, String group) { try { while (job.getCounters().getGroup(group).size() == 0) { if (job.isComplete()) { return null; } Thread.sleep(SLEEP_MSECS); } return job.getCounters().getGroup(group).iterator().next().getName(); } catch (IOException | InterruptedException e) { throw new IllegalStateException("waitAndGetCounterNameFromGroup: Exception occurred", e); } }
From source file:org.apache.gora.examples.mapreduce.QueryCounter.java
License:Apache License
/** * Returns the number of results to the Query *///from w ww. jav a 2s . com public long countQuery(DataStore<K, T> dataStore, Query<K, T> query) throws Exception { Job job = createJob(dataStore, query); job.waitForCompletion(true); assert (job.isComplete() == true); return job.getCounters().findCounter(COUNTER_GROUP, ROWS).getValue(); }
From source file:org.apache.gora.examples.mapreduce.QueryCounter.java
License:Apache License
/** * Returns the number of results to the Query obtained by the * {@link #getQuery(DataStore)} method.// w ww . j a v a2s .c o m */ public long countQuery(DataStore<K, T> dataStore) throws Exception { Query<K, T> query = getQuery(dataStore); Job job = createJob(dataStore, query); job.waitForCompletion(true); assert (job.isComplete() == true); return job.getCounters().findCounter(COUNTER_GROUP, ROWS).getValue(); }
From source file:org.apache.parquet.avro.TestInputOutputFormat.java
License:Apache License
private void waitForJob(Job job) throws Exception { job.submit();/*from www.j av a2 s. c o m*/ while (!job.isComplete()) { LOG.debug("waiting for job " + job.getJobName()); sleep(100); } LOG.info("status for job " + job.getJobName() + ": " + (job.isSuccessful() ? "SUCCESS" : "FAILURE")); if (!job.isSuccessful()) { throw new RuntimeException("job failed " + job.getJobName()); } }