List of usage examples for org.apache.hadoop.conf Configuration setClass
public void setClass(String name, Class<?> theClass, Class<?> xface)
name property to the name of a theClass implementing the given interface xface. From source file:com.thinkbiganalytics.kylo.hadoop.FileSystemUtilTest.java
License:Apache License
/** * Verify detecting if an HDFS file exists. *//*from w w w .j a va 2s .com*/ @Test public void fileExistsHdfs() throws IOException { final Configuration conf = new Configuration(false); conf.setClass("fs.hdfs.impl", MockFileSystem.class, FileSystem.class); final File file = File.createTempFile("kylo-", ".tmp"); try { Assert.assertTrue("Expected HDFS file to exist", FileSystemUtil.fileExists(URI.create("hdfs:" + file.getAbsolutePath()), conf)); Assert.assertTrue("Expected HDFS file to exist", FileSystemUtil.fileExists(URI.create("hadoop:hdfs:" + file.getAbsolutePath()), conf)); } finally { Files.delete(file.toPath()); } }
From source file:com.thinkbiganalytics.kylo.hadoop.FileSystemUtilTest.java
License:Apache License
/** * Verify response for unsupported file systems. *//*from ww w . j a v a2 s . co m*/ @Test(expected = IOException.class) public void fileExistsOther() throws IOException { final Configuration conf = new Configuration(false); conf.setClass("fs.mock.impl", MockFileSystem.class, FileSystem.class); FileSystemUtil.fileExists(URI.create("mock:/tmp/" + UUID.randomUUID().toString()), conf); }
From source file:com.thinkbiganalytics.kylo.util.HadoopClassLoaderTest.java
License:Apache License
@Before public void setUp() { final Configuration conf = new Configuration(false); conf.setClass("fs.mock.impl", MockFileSystem.class, FileSystem.class); Handler.register();//from w ww . j ava2s . com Handler.setConfiguration(conf); }
From source file:com.topsoft.botspider.util.MapAvroReadTool.java
License:Apache License
@SuppressWarnings({ "rawtypes", "unchecked" })
@Override//from w w w . java 2 s .co m
public int run(InputStream stdin, PrintStream out, PrintStream err, List<String> args) throws Exception {
if (args.size() < 1) {
err.println("Usage: input_file [-key key] [-count]");
return 1;
}
String key = null;
boolean bcount = false;
for (int i = 0; i < args.size(); i++) {
if ("-key".equals(args.get(i))) {
key = args.get(i + 1);
i++;
} else if ("-count".equals(args.get(i))) {
bcount = true;
}
}
if (key == null)
bcount = true;
Configuration conf = new Configuration();
conf.setClass(MapAvroFile.Reader.DATUM_READER_CLASS, GenericDatumReader.class, DatumReader.class);
FileSystem fs = FileSystem.get(conf);
Path input = new Path(args.get(0));
Path datafile = new Path(input, MapAvroFile.DATA_FILE_NAME);
Path indexfile = new Path(input, MapAvroFile.INDEX_FILE_NAME);
MapAvroFile.Reader[] readers = null;
if (fs.exists(datafile) && fs.exists(indexfile)) {
// readers = new
readers = new MapAvroFile.Reader[] { new MapAvroFile.Reader(fs, input.toString(), conf) };
} else
readers = AvroMapOutputFormat.getReaders(new Path(args.get(0)), conf);
try {
if (key != null) {
Object obj = AvroMapOutputFormat.getEntry(readers, key);
out.println(AvroUtils.toAvroString(obj));
}
if (bcount) {
int total = 0;
for (MapAvroFile.Reader reader : readers) {
total += reader.size();
}
out.println("Total record : " + total);
}
} finally {
for (int i = 0; i < readers.length; i++) {
try {
readers[i].close();
} catch (Exception e) {
}
}
}
return 0;
}
From source file:com.yahoo.glimmer.indexing.generator.TripleIndexGenerator.java
License:Open Source License
public int run(String[] args) throws Exception { SimpleJSAP jsap = new SimpleJSAP(TripleIndexGenerator.class.getName(), "Generates a keyword index from RDF data.", new Parameter[] { new Switch(NO_CONTEXTS_ARG, 'C', "withoutContexts", "Don't process the contexts for each tuple."), new FlaggedOption(METHOD_ARG, JSAP.STRING_PARSER, "horizontal", JSAP.REQUIRED, 'm', METHOD_ARG, "horizontal or vertical."), new FlaggedOption(PREDICATES_ARG, JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.NOT_REQUIRED, 'p', PREDICATES_ARG, "Subset of the properties to be indexed."), new FlaggedOption(RESOURCE_PREFIX_ARG, JSAP.STRING_PARSER, "@", JSAP.NOT_REQUIRED, 'r', RESOURCE_PREFIX_ARG, "Prefix to add to object resource hash values when indexing. Stops queries for numbers matching resource hash values. Default is '@'"), new UnflaggedOption("input", JSAP.STRING_PARSER, JSAP.REQUIRED, "HDFS location for the input data."), new UnflaggedOption(NUMBER_OF_DOCS_ARG, JSAP.LONG_PARSER, JSAP.REQUIRED, "Number of documents to index"), new UnflaggedOption("output", JSAP.STRING_PARSER, JSAP.REQUIRED, "HDFS location for the output."), new UnflaggedOption(RESOURCES_HASH_ARG, JSAP.STRING_PARSER, JSAP.REQUIRED, "HDFS location of the resources hash file."), }); JSAPResult jsapResult = jsap.parse(args); // check whether the command line was valid, and if it wasn't, // display usage information and exit. if (!jsapResult.success()) { System.err.println();// w ww . j a v a2s.c o m System.err.println("Usage: java " + TripleIndexGenerator.class.getName()); System.err.println(" " + jsap.getUsage()); System.err.println(); System.exit(1); } Job job = Job.getInstance(getConf()); job.setJarByClass(TripleIndexGenerator.class); job.setJobName("TripleIndexGenerator" + System.currentTimeMillis()); FileInputFormat.setInputPaths(job, new Path(jsapResult.getString("input"))); job.setInputFormatClass(TextInputFormat.class); job.setMapperClass(DocumentMapper.class); job.setMapOutputKeyClass(TermKey.class); job.setMapOutputValueClass(TermValue.class); job.setPartitionerClass(TermKey.FirstPartitioner.class); job.setGroupingComparatorClass(TermKey.FirstGroupingComparator.class); job.setReducerClass(TermReduce.class); job.setOutputKeyClass(IntWritable.class); job.setOutputValueClass(IndexRecordWriterValue.class); job.setOutputFormatClass(IndexRecordWriter.OutputFormat.class); FileOutputFormat.setOutputPath(job, new Path(jsapResult.getString("output"))); Configuration conf = job.getConfiguration(); conf.setClass("mapred.output.key.comparator.class", TermKey.Comparator.class, WritableComparator.class); conf.set("mapreduce.user.classpath.first", "true"); long numDocs = jsapResult.getLong(NUMBER_OF_DOCS_ARG); conf.setLong(NUMBER_OF_DOCUMENTS, numDocs); // Set this in a attempt to get around the 2GB of ram task limit on our cluster. // Setting this in the hope of fixing Direct buffer memory errors conf.setInt(INDEX_WRITER_CACHE_SIZE, 1024 * 1024); conf.set(OUTPUT_DIR, jsapResult.getString("output")); boolean withContexts = !jsapResult.getBoolean(NO_CONTEXTS_ARG, false); if (jsapResult.getString(METHOD_ARG).equalsIgnoreCase(METHOD_ARG_VALUE_HORIZONTAL)) { HorizontalDocumentFactory.setupConf(conf, withContexts, jsapResult.getString(RESOURCES_HASH_ARG), jsapResult.getString(RESOURCE_PREFIX_ARG)); } else if (jsapResult.getString(METHOD_ARG).equalsIgnoreCase(METHOD_ARG_VALUE_VERTICAL)) { if (!jsapResult.contains(PREDICATES_ARG)) { throw new IllegalArgumentException("When '" + METHOD_ARG + "' is '" + METHOD_ARG_VALUE_VERTICAL + "' you have to give a predicates file too."); } VerticalDocumentFactory.setupConf(conf, withContexts, jsapResult.getString(RESOURCES_HASH_ARG), jsapResult.getString(RESOURCE_PREFIX_ARG), jsapResult.getString(PREDICATES_ARG)); } else { throw new IllegalArgumentException(METHOD_ARG + " should be '" + METHOD_ARG_VALUE_HORIZONTAL + "' or '" + METHOD_ARG_VALUE_VERTICAL + "'"); } conf.setInt("mapreduce.input.linerecordreader.line.maxlength", 1024 * 1024); boolean success = job.waitForCompletion(true); return success ? 0 : 1; }
From source file:com.zjy.mongo.util.MapredMongoConfigUtil.java
License:Apache License
public static void setMapper(final Configuration conf, final Class<? extends Mapper> val) { conf.setClass(MongoConfigUtil.JOB_MAPPER, val, Mapper.class); }
From source file:com.zjy.mongo.util.MapredMongoConfigUtil.java
License:Apache License
public static void setCombiner(final Configuration conf, final Class<? extends Reducer> val) { conf.setClass(MongoConfigUtil.JOB_COMBINER, val, Reducer.class); }
From source file:com.zjy.mongo.util.MapredMongoConfigUtil.java
License:Apache License
public static void setReducer(final Configuration conf, final Class<? extends Reducer> val) { conf.setClass(MongoConfigUtil.JOB_REDUCER, val, Reducer.class); }
From source file:com.zjy.mongo.util.MapredMongoConfigUtil.java
License:Apache License
public static void setPartitioner(final Configuration conf, final Class<? extends Partitioner> val) { conf.setClass(MongoConfigUtil.JOB_PARTITIONER, val, Partitioner.class); }
From source file:com.zjy.mongo.util.MapredMongoConfigUtil.java
License:Apache License
public static void setOutputFormat(final Configuration conf, final Class<? extends OutputFormat> val) { conf.setClass(MongoConfigUtil.JOB_OUTPUT_FORMAT, val, OutputFormat.class); }