List of usage examples for org.apache.hadoop.mapreduce Job isSuccessful
public boolean isSuccessful() throws IOException
From source file:org.apache.hive.hcatalog.mapreduce.TestHCatNonPartitioned.java
License:Apache License
@Test public void testHCatNonPartitionedTable() throws Exception { Map<String, String> partitionMap = new HashMap<String, String>(); runMRCreate(null, partitionColumns, writeRecords, 10, true); //Test for duplicate publish -- this will either fail on job creation time // and throw an exception, or will fail at runtime, and fail the job. IOException exc = null;//from w w w .ja v a 2 s. co m try { Job j = runMRCreate(null, partitionColumns, writeRecords, 20, true); assertEquals(!isTableImmutable(), j.isSuccessful()); } catch (IOException e) { exc = e; assertTrue(exc instanceof HCatException); assertEquals(ErrorType.ERROR_NON_EMPTY_TABLE, ((HCatException) exc).getErrorType()); } if (!isTableImmutable()) { assertNull(exc); } //Test for publish with invalid partition key name exc = null; partitionMap.clear(); partitionMap.put("px", "p1value2"); try { Job j = runMRCreate(partitionMap, partitionColumns, writeRecords, 20, true); assertFalse(j.isSuccessful()); } catch (IOException e) { exc = e; assertTrue(exc != null); assertTrue(exc instanceof HCatException); assertEquals(ErrorType.ERROR_INVALID_PARTITION_VALUES, ((HCatException) exc).getErrorType()); } //Read should get 10 rows if immutable, 30 if mutable if (isTableImmutable()) { runMRRead(10); } else { runMRRead(30); } hiveReadTest(); }
From source file:org.apache.hive.hcatalog.mapreduce.TestHCatPartitioned.java
License:Apache License
@Test public void testHCatPartitionedTable() throws Exception { Map<String, String> partitionMap = new HashMap<String, String>(); partitionMap.put("part1", "p1value1"); partitionMap.put("part0", "501"); runMRCreate(partitionMap, partitionColumns, writeRecords, 10, true); partitionMap.clear();//w w w . j ava2s . com partitionMap.put("PART1", "p1value2"); partitionMap.put("PART0", "502"); runMRCreate(partitionMap, partitionColumns, writeRecords, 20, true); //Test for duplicate publish -- this will either fail on job creation time // and throw an exception, or will fail at runtime, and fail the job. IOException exc = null; try { Job j = runMRCreate(partitionMap, partitionColumns, writeRecords, 20, true); assertEquals(!isTableImmutable(), j.isSuccessful()); } catch (IOException e) { exc = e; assertTrue(exc instanceof HCatException); assertTrue(ErrorType.ERROR_DUPLICATE_PARTITION.equals(((HCatException) exc).getErrorType())); } if (!isTableImmutable()) { assertNull(exc); } //Test for publish with invalid partition key name exc = null; partitionMap.clear(); partitionMap.put("px1", "p1value2"); partitionMap.put("px0", "502"); try { Job j = runMRCreate(partitionMap, partitionColumns, writeRecords, 20, true); assertFalse(j.isSuccessful()); } catch (IOException e) { exc = e; assertNotNull(exc); assertTrue(exc instanceof HCatException); assertEquals(ErrorType.ERROR_MISSING_PARTITION_KEY, ((HCatException) exc).getErrorType()); } //Test for publish with missing partition key values exc = null; partitionMap.clear(); partitionMap.put("px", "512"); try { runMRCreate(partitionMap, partitionColumns, writeRecords, 20, true); } catch (IOException e) { exc = e; } assertNotNull(exc); assertTrue(exc instanceof HCatException); assertEquals(ErrorType.ERROR_INVALID_PARTITION_VALUES, ((HCatException) exc).getErrorType()); //Test for null partition value map exc = null; try { runMRCreate(null, partitionColumns, writeRecords, 20, false); } catch (IOException e) { exc = e; } assertTrue(exc == null); // assertTrue(exc instanceof HCatException); // assertEquals(ErrorType.ERROR_PUBLISHING_PARTITION, ((HCatException) exc).getErrorType()); // With Dynamic partitioning, this isn't an error that the keyValues specified didn't values //Read should get 10 + 20 rows if immutable, 50 (10+20+20) if mutable if (isTableImmutable()) { runMRRead(30); } else { runMRRead(50); } //Read with partition filter runMRRead(10, "part1 = \"p1value1\""); runMRRead(10, "part0 = \"501\""); if (isTableImmutable()) { runMRRead(20, "part1 = \"p1value2\""); runMRRead(30, "part1 = \"p1value1\" or part1 = \"p1value2\""); runMRRead(20, "part0 = \"502\""); runMRRead(30, "part0 = \"501\" or part0 = \"502\""); } else { runMRRead(40, "part1 = \"p1value2\""); runMRRead(50, "part1 = \"p1value1\" or part1 = \"p1value2\""); runMRRead(40, "part0 = \"502\""); runMRRead(50, "part0 = \"501\" or part0 = \"502\""); } tableSchemaTest(); columnOrderChangeTest(); hiveReadTest(); }
From source file:org.apache.hive.hcatalog.mapreduce.TestSequenceFileReadWrite.java
License:Apache License
@Test public void testSequenceTableWriteReadMR() throws Exception { String createTable = "CREATE TABLE demo_table_2(a0 int, a1 String, a2 String) STORED AS SEQUENCEFILE"; driver.run("drop table demo_table_2"); int retCode1 = driver.run(createTable).getResponseCode(); assertTrue(retCode1 == 0);/*from w w w .ja v a2 s .c o m*/ Configuration conf = new Configuration(); conf.set(HCatConstants.HCAT_KEY_HIVE_CONF, HCatUtil.serialize(hiveConf.getAllProperties())); Job job = new Job(conf, "Write-hcat-seq-table"); job.setJarByClass(TestSequenceFileReadWrite.class); job.setMapperClass(Map.class); job.setOutputKeyClass(NullWritable.class); job.setOutputValueClass(DefaultHCatRecord.class); job.setInputFormatClass(TextInputFormat.class); TextInputFormat.setInputPaths(job, inputFileName); HCatOutputFormat.setOutput(job, OutputJobInfo.create(MetaStoreUtils.DEFAULT_DATABASE_NAME, "demo_table_2", null)); job.setOutputFormatClass(HCatOutputFormat.class); HCatOutputFormat.setSchema(job, getSchema()); job.setNumReduceTasks(0); assertTrue(job.waitForCompletion(true)); if (!HCatUtil.isHadoop23()) { new FileOutputCommitterContainer(job, null).commitJob(job); } assertTrue(job.isSuccessful()); server.setBatchOn(); server.registerQuery("C = load 'default.demo_table_2' using org.apache.hive.hcatalog.pig.HCatLoader();"); server.executeBatch(); Iterator<Tuple> XIter = server.openIterator("C"); int numTuplesRead = 0; while (XIter.hasNext()) { Tuple t = XIter.next(); assertEquals(3, t.size()); assertEquals(t.get(0).toString(), "" + numTuplesRead); assertEquals(t.get(1).toString(), "a" + numTuplesRead); assertEquals(t.get(2).toString(), "b" + numTuplesRead); numTuplesRead++; } assertEquals(input.length, numTuplesRead); }
From source file:org.apache.hive.hcatalog.mapreduce.TestSequenceFileReadWrite.java
License:Apache License
@Test public void testTextTableWriteReadMR() throws Exception { String createTable = "CREATE TABLE demo_table_3(a0 int, a1 String, a2 String) STORED AS TEXTFILE"; driver.run("drop table demo_table_3"); int retCode1 = driver.run(createTable).getResponseCode(); assertTrue(retCode1 == 0);//from w ww.j a va2s .c o m Configuration conf = new Configuration(); conf.set(HCatConstants.HCAT_KEY_HIVE_CONF, HCatUtil.serialize(hiveConf.getAllProperties())); Job job = new Job(conf, "Write-hcat-text-table"); job.setJarByClass(TestSequenceFileReadWrite.class); job.setMapperClass(Map.class); job.setOutputKeyClass(NullWritable.class); job.setOutputValueClass(DefaultHCatRecord.class); job.setInputFormatClass(TextInputFormat.class); job.setNumReduceTasks(0); TextInputFormat.setInputPaths(job, inputFileName); HCatOutputFormat.setOutput(job, OutputJobInfo.create(MetaStoreUtils.DEFAULT_DATABASE_NAME, "demo_table_3", null)); job.setOutputFormatClass(HCatOutputFormat.class); HCatOutputFormat.setSchema(job, getSchema()); assertTrue(job.waitForCompletion(true)); if (!HCatUtil.isHadoop23()) { new FileOutputCommitterContainer(job, null).commitJob(job); } assertTrue(job.isSuccessful()); server.setBatchOn(); server.registerQuery("D = load 'default.demo_table_3' using org.apache.hive.hcatalog.pig.HCatLoader();"); server.executeBatch(); Iterator<Tuple> XIter = server.openIterator("D"); int numTuplesRead = 0; while (XIter.hasNext()) { Tuple t = XIter.next(); assertEquals(3, t.size()); assertEquals(t.get(0).toString(), "" + numTuplesRead); assertEquals(t.get(1).toString(), "a" + numTuplesRead); assertEquals(t.get(2).toString(), "b" + numTuplesRead); numTuplesRead++; } assertEquals(input.length, numTuplesRead); }
From source file:org.apache.kylin.common.util.HbaseImporter.java
License:Apache License
private static boolean runImport(String[] args, Configuration configuration) throws IOException, InterruptedException, ClassNotFoundException { // need to make a copy of the configuration because to make sure different temp dirs are used. GenericOptionsParser opts = new GenericOptionsParser(new Configuration(configuration), args); Configuration newConf = opts.getConfiguration(); args = opts.getRemainingArgs();/* w w w. jav a 2 s. c o m*/ Job job = Import.createSubmittableJob(newConf, args); job.waitForCompletion(false); return job.isSuccessful(); }
From source file:org.apache.kylin.engine.mr.common.AbstractHadoopJob.java
License:Apache License
protected int waitForCompletion(Job job) throws IOException, InterruptedException, ClassNotFoundException { int retVal = 0; long start = System.nanoTime(); if (isAsync) { job.submit();//from w ww . j ava 2 s . co m } else { job.waitForCompletion(true); retVal = job.isSuccessful() ? 0 : 1; logger.debug("Job '" + job.getJobName() + "' finished " + (job.isSuccessful() ? "successfully in " : "with failures. Time taken ") + formatTime((System.nanoTime() - start) / 1000000L)); } return retVal; }
From source file:org.apache.mahout.math.hadoop.stochasticsvd.ABtDenseOutJob.java
License:Apache License
public static void run(Configuration conf, Path[] inputAPaths, Path inputBtGlob, Path xiPath, Path sqPath, Path sbPath, Path outputPath, int aBlockRows, int minSplitSize, int k, int p, int outerProdBlockHeight, int numReduceTasks, boolean broadcastBInput) throws ClassNotFoundException, InterruptedException, IOException { JobConf oldApiJob = new JobConf(conf); Job job = new Job(oldApiJob); job.setJobName("ABt-job"); job.setJarByClass(ABtDenseOutJob.class); job.setInputFormatClass(SequenceFileInputFormat.class); FileInputFormat.setInputPaths(job, inputAPaths); if (minSplitSize > 0) { FileInputFormat.setMinInputSplitSize(job, minSplitSize); }//from ww w .jav a2s. c o m FileOutputFormat.setOutputPath(job, outputPath); SequenceFileOutputFormat.setOutputCompressionType(job, CompressionType.BLOCK); job.setMapOutputKeyClass(SplitPartitionedWritable.class); job.setMapOutputValueClass(DenseBlockWritable.class); job.setOutputKeyClass(SplitPartitionedWritable.class); job.setOutputValueClass(VectorWritable.class); job.setMapperClass(ABtMapper.class); job.setReducerClass(QRReducer.class); job.getConfiguration().setInt(QJob.PROP_AROWBLOCK_SIZE, aBlockRows); job.getConfiguration().setInt(BtJob.PROP_OUTER_PROD_BLOCK_HEIGHT, outerProdBlockHeight); job.getConfiguration().setInt(QRFirstStep.PROP_K, k); job.getConfiguration().setInt(QRFirstStep.PROP_P, p); job.getConfiguration().set(PROP_BT_PATH, inputBtGlob.toString()); /* * PCA-related options, MAHOUT-817 */ if (xiPath != null) { job.getConfiguration().set(PROP_XI_PATH, xiPath.toString()); job.getConfiguration().set(PROP_SB_PATH, sbPath.toString()); job.getConfiguration().set(PROP_SQ_PATH, sqPath.toString()); } job.setNumReduceTasks(numReduceTasks); // broadcast Bt files if required. if (broadcastBInput) { job.getConfiguration().set(PROP_BT_BROADCAST, "y"); FileSystem fs = FileSystem.get(inputBtGlob.toUri(), conf); FileStatus[] fstats = fs.globStatus(inputBtGlob); if (fstats != null) { for (FileStatus fstat : fstats) { /* * new api is not enabled yet in our dependencies at this time, still * using deprecated one */ DistributedCache.addCacheFile(fstat.getPath().toUri(), job.getConfiguration()); } } } job.submit(); job.waitForCompletion(false); if (!job.isSuccessful()) { throw new IOException("ABt job unsuccessful."); } }
From source file:org.apache.mahout.math.hadoop.stochasticsvd.ABtJob.java
License:Apache License
public static void run(Configuration conf, Path[] inputAPaths, Path inputBtGlob, Path outputPath, int aBlockRows, int minSplitSize, int k, int p, int outerProdBlockHeight, int numReduceTasks, boolean broadcastBInput) throws ClassNotFoundException, InterruptedException, IOException { JobConf oldApiJob = new JobConf(conf); // MultipleOutputs // .addNamedOutput(oldApiJob, // QJob.OUTPUT_QHAT, // org.apache.hadoop.mapred.SequenceFileOutputFormat.class, // SplitPartitionedWritable.class, // DenseBlockWritable.class); ////from www. j a v a 2 s .c om // MultipleOutputs // .addNamedOutput(oldApiJob, // QJob.OUTPUT_RHAT, // org.apache.hadoop.mapred.SequenceFileOutputFormat.class, // SplitPartitionedWritable.class, // VectorWritable.class); Job job = new Job(oldApiJob); job.setJobName("ABt-job"); job.setJarByClass(ABtJob.class); job.setInputFormatClass(SequenceFileInputFormat.class); FileInputFormat.setInputPaths(job, inputAPaths); if (minSplitSize > 0) { FileInputFormat.setMinInputSplitSize(job, minSplitSize); } FileOutputFormat.setOutputPath(job, outputPath); SequenceFileOutputFormat.setOutputCompressionType(job, CompressionType.BLOCK); job.setMapOutputKeyClass(SplitPartitionedWritable.class); job.setMapOutputValueClass(SparseRowBlockWritable.class); job.setOutputKeyClass(SplitPartitionedWritable.class); job.setOutputValueClass(VectorWritable.class); job.setMapperClass(ABtMapper.class); job.setCombinerClass(BtJob.OuterProductCombiner.class); job.setReducerClass(QRReducer.class); job.getConfiguration().setInt(QJob.PROP_AROWBLOCK_SIZE, aBlockRows); job.getConfiguration().setInt(BtJob.PROP_OUTER_PROD_BLOCK_HEIGHT, outerProdBlockHeight); job.getConfiguration().setInt(QRFirstStep.PROP_K, k); job.getConfiguration().setInt(QRFirstStep.PROP_P, p); job.getConfiguration().set(PROP_BT_PATH, inputBtGlob.toString()); // number of reduce tasks doesn't matter. we don't actually // send anything to reducers. job.setNumReduceTasks(numReduceTasks); // broadcast Bt files if required. if (broadcastBInput) { job.getConfiguration().set(PROP_BT_BROADCAST, "y"); FileSystem fs = FileSystem.get(inputBtGlob.toUri(), conf); FileStatus[] fstats = fs.globStatus(inputBtGlob); if (fstats != null) { for (FileStatus fstat : fstats) { /* * new api is not enabled yet in our dependencies at this time, still * using deprecated one */ DistributedCache.addCacheFile(fstat.getPath().toUri(), conf); } } } job.submit(); job.waitForCompletion(false); if (!job.isSuccessful()) { throw new IOException("ABt job unsuccessful."); } }
From source file:org.apache.mahout.math.hadoop.stochasticsvd.BBtJob.java
License:Apache License
public static void run(Configuration conf, Path btPath, Path outputPath, int numReduceTasks) throws IOException, ClassNotFoundException, InterruptedException { Job job = new Job(conf); job.setJobName("BBt-job"); job.setJarByClass(BBtJob.class); // input//from w ww . ja v a 2 s . c om job.setInputFormatClass(SequenceFileInputFormat.class); FileInputFormat.setInputPaths(job, btPath); // map job.setMapOutputKeyClass(IntWritable.class); job.setMapOutputValueClass(VectorWritable.class); job.setMapperClass(BBtMapper.class); job.setReducerClass(BBtReducer.class); // combiner and reducer job.setOutputKeyClass(IntWritable.class); job.setOutputValueClass(VectorWritable.class); // output job.setOutputFormatClass(SequenceFileOutputFormat.class); FileOutputFormat.setOutputPath(job, outputPath); SequenceFileOutputFormat.setOutputCompressionType(job, CompressionType.BLOCK); FileOutputFormat.setOutputCompressorClass(job, DefaultCodec.class); job.getConfiguration().set("mapreduce.output.basename", OUTPUT_BBT); // run job.submit(); job.waitForCompletion(false); if (!job.isSuccessful()) { throw new IOException("BBt job failed."); } }
From source file:org.apache.mahout.math.hadoop.stochasticsvd.BtJob.java
License:Apache License
public static void run(Configuration conf, Path[] inputPathA, Path inputPathQJob, Path xiPath, Path outputPath, int minSplitSize, int k, int p, int btBlockHeight, int numReduceTasks, boolean broadcast, Class<? extends Writable> labelClass, boolean outputBBtProducts) throws ClassNotFoundException, InterruptedException, IOException { JobConf oldApiJob = new JobConf(conf); MultipleOutputs.addNamedOutput(oldApiJob, OUTPUT_Q, org.apache.hadoop.mapred.SequenceFileOutputFormat.class, labelClass, VectorWritable.class); if (outputBBtProducts) { MultipleOutputs.addNamedOutput(oldApiJob, OUTPUT_BBT, org.apache.hadoop.mapred.SequenceFileOutputFormat.class, IntWritable.class, VectorWritable.class); /*/*from w w w.j a v a 2 s .c o m*/ * MAHOUT-1067: if we are asked to output BBT products then named vector * names should be propagated to Q too so that UJob could pick them up * from there. */ oldApiJob.setBoolean(PROP_NV, true); } if (xiPath != null) { // compute pca -related stuff as well MultipleOutputs.addNamedOutput(oldApiJob, OUTPUT_SQ, org.apache.hadoop.mapred.SequenceFileOutputFormat.class, IntWritable.class, VectorWritable.class); MultipleOutputs.addNamedOutput(oldApiJob, OUTPUT_SB, org.apache.hadoop.mapred.SequenceFileOutputFormat.class, IntWritable.class, VectorWritable.class); } /* * HACK: we use old api multiple outputs since they are not available in the * new api of either 0.20.2 or 0.20.203 but wrap it into a new api job so we * can use new api interfaces. */ Job job = new Job(oldApiJob); job.setJobName("Bt-job"); job.setJarByClass(BtJob.class); job.setInputFormatClass(SequenceFileInputFormat.class); job.setOutputFormatClass(SequenceFileOutputFormat.class); FileInputFormat.setInputPaths(job, inputPathA); if (minSplitSize > 0) { FileInputFormat.setMinInputSplitSize(job, minSplitSize); } FileOutputFormat.setOutputPath(job, outputPath); // WARN: tight hadoop integration here: job.getConfiguration().set("mapreduce.output.basename", OUTPUT_BT); FileOutputFormat.setOutputCompressorClass(job, DefaultCodec.class); SequenceFileOutputFormat.setOutputCompressionType(job, CompressionType.BLOCK); job.setMapOutputKeyClass(LongWritable.class); job.setMapOutputValueClass(SparseRowBlockWritable.class); job.setOutputKeyClass(IntWritable.class); job.setOutputValueClass(VectorWritable.class); job.setMapperClass(BtMapper.class); job.setCombinerClass(OuterProductCombiner.class); job.setReducerClass(OuterProductReducer.class); job.getConfiguration().setInt(QJob.PROP_K, k); job.getConfiguration().setInt(QJob.PROP_P, p); job.getConfiguration().set(PROP_QJOB_PATH, inputPathQJob.toString()); job.getConfiguration().setBoolean(PROP_OUPTUT_BBT_PRODUCTS, outputBBtProducts); job.getConfiguration().setInt(PROP_OUTER_PROD_BLOCK_HEIGHT, btBlockHeight); job.setNumReduceTasks(numReduceTasks); /* * PCA-related options, MAHOUT-817 */ if (xiPath != null) { job.getConfiguration().set(PROP_XI_PATH, xiPath.toString()); } /* * we can broadhast Rhat files since all of them are reuqired by each job, * but not Q files which correspond to splits of A (so each split of A will * require only particular Q file, each time different one). */ if (broadcast) { job.getConfiguration().set(PROP_RHAT_BROADCAST, "y"); FileSystem fs = FileSystem.get(inputPathQJob.toUri(), conf); FileStatus[] fstats = fs.globStatus(new Path(inputPathQJob, QJob.OUTPUT_RHAT + "-*")); if (fstats != null) { for (FileStatus fstat : fstats) { /* * new api is not enabled yet in our dependencies at this time, still * using deprecated one */ DistributedCache.addCacheFile(fstat.getPath().toUri(), job.getConfiguration()); } } } job.submit(); job.waitForCompletion(false); if (!job.isSuccessful()) { throw new IOException("Bt job unsuccessful."); } }