List of usage examples for org.apache.hadoop.conf Configuration setFloat
public void setFloat(String name, float value)
name property to a float. From source file:hrider.hbase.Connection.java
License:Apache License
/** * Saves a table locally to an HFile./* w w w.j a va 2s .c o m*/ * * @param tableName The name of the table. * @param path The path tot he file. * @throws IOException Error accessing hbase. */ public void saveTable(String tableName, String path) throws IOException { FileSystem fs = FileSystem.getLocal(this.getConfiguration()); HTable table = this.factory.get(tableName); Configuration cacheConfig = new Configuration(this.getConfiguration()); cacheConfig.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f); StoreFile.Writer writer = new StoreFile.WriterBuilder(this.getConfiguration(), new CacheConfig(cacheConfig), fs, HColumnDescriptor.DEFAULT_BLOCKSIZE).withFilePath(new Path(path)).build(); ResultScanner scanner = null; try { Scan scan = new Scan(); scan.setCaching(GlobalConfig.instance().getBatchSizeForRead()); scanner = table.getScanner(scan); boolean isValid; do { Result result = scanner.next(); isValid = result != null; if (isValid) { for (KeyValue keyValue : result.list()) { writer.append(keyValue); } for (HbaseActionListener listener : this.listeners) { listener.saveOperation(tableName, path, result); } } } while (isValid); } finally { if (scanner != null) { scanner.close(); } writer.close(); } }
From source file:io.bfscan.clueweb12.LMRetrieval.java
License:Apache License
/** * Runs this tool./*from ww w . ja va 2 s . c om*/ */ @SuppressWarnings({ "static-access" }) public int run(String[] args) throws Exception { Options options = new Options(); options.addOption(OptionBuilder.withArgName("path").hasArg() .withDescription("input path (pfor format expected, add * to retrieve files)") .create(DOCVECTOR_OPTION)); options.addOption( OptionBuilder.withArgName("path").hasArg().withDescription("output path").create(OUTPUT_OPTION)); options.addOption( OptionBuilder.withArgName("path").hasArg().withDescription("dictionary").create(DICTIONARY_OPTION)); options.addOption( OptionBuilder.withArgName("path").hasArg().withDescription("queries").create(QUERIES_OPTION)); options.addOption( OptionBuilder.withArgName("float").hasArg().withDescription("smoothing").create(SMOOTHING)); options.addOption(OptionBuilder.withArgName("int").hasArg().withDescription("topk").create(TOPK)); options.addOption(OptionBuilder.withArgName("string " + AnalyzerFactory.getOptions()).hasArg() .withDescription("preprocessing").create(PREPROCESSING)); CommandLine cmdline; CommandLineParser parser = new GnuParser(); try { cmdline = parser.parse(options, args); } catch (ParseException exp) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp(this.getClass().getName(), options); ToolRunner.printGenericCommandUsage(System.out); System.err.println("Error parsing command line: " + exp.getMessage()); return -1; } if (!cmdline.hasOption(DOCVECTOR_OPTION) || !cmdline.hasOption(OUTPUT_OPTION) || !cmdline.hasOption(DICTIONARY_OPTION) || !cmdline.hasOption(QUERIES_OPTION) || !cmdline.hasOption(SMOOTHING) || !cmdline.hasOption(TOPK) || !cmdline.hasOption(PREPROCESSING)) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp(this.getClass().getName(), options); ToolRunner.printGenericCommandUsage(System.out); return -1; } String docvector = cmdline.getOptionValue(DOCVECTOR_OPTION); String output = cmdline.getOptionValue(OUTPUT_OPTION); String dictionary = cmdline.getOptionValue(DICTIONARY_OPTION); String queries = cmdline.getOptionValue(QUERIES_OPTION); String smoothing = cmdline.getOptionValue(SMOOTHING); String topk = cmdline.getOptionValue(TOPK); String preprocessing = cmdline.getOptionValue(PREPROCESSING); LOG.info("Tool name: " + LMRetrieval.class.getSimpleName()); LOG.info(" - docvector: " + docvector); LOG.info(" - output: " + output); LOG.info(" - dictionary: " + dictionary); LOG.info(" - queries: " + queries); LOG.info(" - smoothing: " + smoothing); LOG.info(" - topk: " + topk); LOG.info(" - preprocessing: " + preprocessing); Configuration conf = getConf(); conf.set(DICTIONARY_OPTION, dictionary); conf.set(QUERIES_OPTION, queries); conf.setFloat(SMOOTHING, Float.parseFloat(smoothing)); conf.setInt(TOPK, Integer.parseInt(topk)); conf.set(PREPROCESSING, preprocessing); conf.set("mapreduce.map.memory.mb", "10048"); conf.set("mapreduce.map.java.opts", "-Xmx10048m"); conf.set("mapreduce.reduce.memory.mb", "10048"); conf.set("mapreduce.reduce.java.opts", "-Xmx10048m"); conf.set("mapred.task.timeout", "6000000"); // default is 600000 FileSystem fs = FileSystem.get(conf); if (fs.exists(new Path(output))) { fs.delete(new Path(output), true); } Job job = new Job(conf, LMRetrieval.class.getSimpleName() + ":" + docvector); job.setJarByClass(LMRetrieval.class); FileInputFormat.setInputPaths(job, docvector); FileOutputFormat.setOutputPath(job, new Path(output)); job.setInputFormatClass(SequenceFileInputFormat.class); job.setMapOutputKeyClass(PairOfIntString.class); job.setMapOutputValueClass(FloatWritable.class); job.setOutputKeyClass(NullWritable.class); job.setOutputValueClass(Text.class); job.setMapperClass(MyMapper.class); job.setPartitionerClass(MyPartitioner.class); job.setReducerClass(MyReducer.class); long startTime = System.currentTimeMillis(); job.waitForCompletion(true); LOG.info("Job Finished in " + (System.currentTimeMillis() - startTime) / 1000.0 + " seconds"); return 0; }
From source file:ivory.app.BuildIndex.java
License:Apache License
@SuppressWarnings({ "static-access" })
@Override// ww w . j a v a2s.c o m
public int run(String[] args) throws Exception {
Options options = new Options();
options.addOption(new Option(POSITIONAL_INDEX_IP, "build positional index (IP algorithm)"));
options.addOption(new Option(POSITIONAL_INDEX_LP, "build positional index (LP algorithm)"));
options.addOption(new Option(NONPOSITIONAL_INDEX_IP, "build nonpositional index (IP algorithm)"));
options.addOption(OptionBuilder.withArgName("path").hasArg().withDescription("(required) index path")
.create(INDEX_PATH));
options.addOption(OptionBuilder.withArgName("num").hasArg()
.withDescription("(optional) number of index partitions: 64 default").create(INDEX_PARTITIONS));
CommandLine cmdline;
CommandLineParser parser = new GnuParser();
try {
cmdline = parser.parse(options, args);
} catch (ParseException exp) {
System.err.println("Error parsing command line: " + exp.getMessage());
return -1;
}
if (!cmdline.hasOption(INDEX_PATH)) {
HelpFormatter formatter = new HelpFormatter();
formatter.setWidth(120);
formatter.printHelp(this.getClass().getName(), options);
ToolRunner.printGenericCommandUsage(System.out);
return -1;
}
String indexPath = cmdline.getOptionValue(INDEX_PATH);
int indexPartitions = cmdline.hasOption(INDEX_PARTITIONS)
? Integer.parseInt(cmdline.getOptionValue(INDEX_PARTITIONS))
: 64;
Configuration conf = getConf();
LOG.info("Tool name: " + this.getClass().getSimpleName());
LOG.info(String.format(" -%s %s", INDEX_PATH, indexPath));
LOG.info(String.format(" -%s %d", INDEX_PARTITIONS, indexPartitions));
if (cmdline.hasOption(POSITIONAL_INDEX_IP)) {
LOG.info(String.format(" -%s", POSITIONAL_INDEX_IP));
conf.set(Constants.IndexPath, indexPath);
conf.setInt(Constants.NumReduceTasks, indexPartitions);
conf.set(Constants.PostingsListsType,
ivory.core.data.index.PostingsListDocSortedPositional.class.getCanonicalName());
new BuildIPInvertedIndexDocSorted(conf).run();
new BuildIntPostingsForwardIndex(conf).run();
} else if (cmdline.hasOption(POSITIONAL_INDEX_LP)) {
LOG.info(String.format(" -%s", POSITIONAL_INDEX_LP));
conf.set(Constants.IndexPath, indexPath);
conf.setInt(Constants.NumReduceTasks, indexPartitions);
conf.set(Constants.PostingsListsType,
ivory.core.data.index.PostingsListDocSortedPositional.class.getCanonicalName());
conf.setFloat("Ivory.IndexingMapMemoryThreshold", 0.9f);
conf.setFloat("Ivory.IndexingReduceMemoryThreshold", 0.9f);
conf.setInt("Ivory.MaxHeap", 2048);
conf.setInt("Ivory.MaxNDocsBeforeFlush", 50000);
new BuildLPInvertedIndexDocSorted(conf).run();
new BuildIntPostingsForwardIndex(conf).run();
} else if (cmdline.hasOption(NONPOSITIONAL_INDEX_IP)) {
LOG.info(String.format(" -%s", NONPOSITIONAL_INDEX_IP));
conf.set(Constants.IndexPath, indexPath);
conf.setInt(Constants.NumReduceTasks, indexPartitions);
conf.set(Constants.PostingsListsType,
ivory.core.data.index.PostingsListDocSortedNonPositional.class.getCanonicalName());
new BuildIPInvertedIndexDocSorted(conf).run();
new BuildIntPostingsForwardIndex(conf).run();
} else {
LOG.info(String.format("Nothing to do. Specify one of the following: %s, %s, %s", POSITIONAL_INDEX_IP,
POSITIONAL_INDEX_LP, NONPOSITIONAL_INDEX_IP));
}
return 0;
}
From source file:jobs.MatrixBlockAdd.java
License:Apache License
public int run(String[] args) throws Exception { Configuration conf = getConf(); conf.setFloat("ALPHA", Float.parseFloat(args[3])); conf.setFloat("BETA", Float.parseFloat(args[4])); conf.setInt("mapred.reduce.tasks", Integer.parseInt(args[5])); if (args.length >= 7) conf.setInt("SR", Integer.parseInt(args[6])); if (args.length >= 8) conf.setInt("SC", Integer.parseInt(args[7])); conf.set("LEFTNAME", args[0]); conf.set("RESNAME", args[2]); //heap space - again - should be passed with the -D option conf.set("mapred.map.child.java.opts", "-Xmx3G"); conf.set("mapred.reduce.child.java.opts", "-Xmx3G"); //job// w ww.j a va 2s .c om Job job1 = new Job(conf, "MatrixBlockAdd"); job1.setJarByClass(MatrixBlockAdd.class); // No Map FileInputFormat.addInputPath(job1, new Path(args[0])); FileInputFormat.addInputPath(job1, new Path(args[1])); job1.setInputFormatClass(SequenceFileInputFormat.class); job1.setMapperClass(NoNameMapper.class); //Reduce job1.setReducerClass(MatrixBlockAddReducer.class); job1.setOutputKeyClass(Text.class); job1.setOutputValueClass(MatrixBlock.class); FileOutputFormat.setOutputPath(job1, new Path(args[2])); job1.setOutputFormatClass(SequenceFileOutputFormat.class); //job1.setOutputFormatClass(TextOutputFormat.class); return job1.waitForCompletion(false) ? 0 : 1; }
From source file:jobs.MatrixBlockMult.java
License:Apache License
public int run(String[] args) throws Exception { Configuration conf = getConf(); conf.setFloat("SCALAR", Float.parseFloat(args[3])); conf.setBoolean("LTRANS", Boolean.parseBoolean(args[4])); conf.setBoolean("RTRANS", Boolean.parseBoolean(args[5])); conf.setInt("NRL", Integer.parseInt(args[6])); conf.setInt("NCL", Integer.parseInt(args[7])); conf.setInt("NRR", Integer.parseInt(args[8])); conf.setInt("NCR", Integer.parseInt(args[9])); //set # of reducers conf.setInt("mapred.reduce.tasks", Integer.parseInt(args[10])); //Get optional blocksize parameters if (args.length >= 12) conf.setInt("SRL", Integer.parseInt(args[11])); if (args.length >= 13) conf.setInt("SCL", Integer.parseInt(args[12])); if (args.length >= 14) conf.setInt("SRR", Integer.parseInt(args[13])); if (args.length >= 15) conf.setInt("SCR", Integer.parseInt(args[14])); conf.set("LEFTNAME", args[0]); conf.set("RIGHTNAME", args[1]); conf.set("RESNAME", args[2]); //heap space - should be entered with the -D format and not dealt with by the program. conf.set("mapred.map.child.java.opts", "-Xmx3G"); conf.set("mapred.reduce.child.java.opts", "-Xmx3G"); //job//from ww w .jav a2s.c o m Job job1 = new Job(conf, "MatrixBlockMult"); job1.setJarByClass(MatrixBlockMult.class); // Map FileInputFormat.addInputPath(job1, new Path(args[0])); FileInputFormat.addInputPath(job1, new Path(args[1])); job1.setInputFormatClass(SequenceFileInputFormat.class); job1.setMapperClass(BlockMultiplicationGroupingMapper.class); job1.setMapOutputKeyClass(Text.class); job1.setMapOutputValueClass(MatrixBlock.class); //Reduce job1.setReducerClass(MatrixBlockMultReducer.class); job1.setOutputKeyClass(Text.class); job1.setOutputValueClass(MatrixBlock.class); FileOutputFormat.setOutputPath(job1, new Path(args[2])); job1.setOutputFormatClass(SequenceFileOutputFormat.class); //job1.setOutputFormatClass(TextOutputFormat.class); return job1.waitForCompletion(false) ? 0 : 1; }
From source file:jobs.MatrixBlockTraceMult.java
License:Apache License
public int run(String[] args) throws Exception { Configuration conf = getConf(); conf.setFloat("SCALAR", Float.parseFloat(args[3])); conf.setBoolean("LTRANS", Boolean.parseBoolean(args[4])); conf.setBoolean("RTRANS", Boolean.parseBoolean(args[5])); //set # of reducers conf.setInt("mapred.reduce.tasks", Integer.parseInt(args[6])); //Get optional blocksize parameters if (args.length >= 8) conf.setInt("SRL", Integer.parseInt(args[7])); if (args.length >= 9) conf.setInt("SCL", Integer.parseInt(args[8])); if (args.length >= 10) conf.setInt("SRR", Integer.parseInt(args[9])); if (args.length >= 11) conf.setInt("SCR", Integer.parseInt(args[10])); conf.set("LEFTNAME", args[0]); conf.set("RIGHTNAME", args[1]); //heap space - should be entered with the -D format and not dealt with by the program. conf.set("mapred.map.child.java.opts", "-Xmx3G"); conf.set("mapred.reduce.child.java.opts", "-Xmx3G"); //job/*from w w w. j a va 2 s . c om*/ Job job1 = new Job(conf, "MatrixBlockTraceMult"); job1.setJarByClass(MatrixBlockMult.class); // Map FileInputFormat.addInputPath(job1, new Path(args[0])); FileInputFormat.addInputPath(job1, new Path(args[1])); job1.setInputFormatClass(SequenceFileInputFormat.class); job1.setMapperClass(SquareBlockTraceMultiplicationGroupingMapper.class); job1.setMapOutputKeyClass(Text.class); job1.setMapOutputValueClass(MatrixBlock.class); //Reduce job1.setReducerClass(SquareMatrixBlockTraceMultReducer.class); job1.setOutputKeyClass(NullWritable.class); job1.setOutputValueClass(DoubleWritable.class); FileOutputFormat.setOutputPath(job1, new Path(args[2])); job1.setOutputFormatClass(TextOutputFormat.class); return job1.waitForCompletion(false) ? 0 : 1; }
From source file:ml.shifu.shifu.core.processor.VarSelectModelProcessor.java
License:Apache License
private void prepareSEJobConf(SourceType source, Configuration conf) throws IOException { // add jars to hadoop mapper and reducer new GenericOptionsParser(conf, new String[] { "-libjars", addRuntimeJars() }); conf.setBoolean(GuaguaMapReduceConstants.MAPRED_MAP_TASKS_SPECULATIVE_EXECUTION, true); conf.setBoolean(GuaguaMapReduceConstants.MAPRED_REDUCE_TASKS_SPECULATIVE_EXECUTION, true); conf.set(Constants.SHIFU_MODEL_CONFIG, ShifuFileUtils.getFileSystemBySourceType(source) .makeQualified(new Path(super.getPathFinder().getModelConfigPath(source))).toString()); conf.set(Constants.SHIFU_COLUMN_CONFIG, ShifuFileUtils.getFileSystemBySourceType(source) .makeQualified(new Path(super.getPathFinder().getColumnConfigPath(source))).toString()); conf.set(NNConstants.MAPRED_JOB_QUEUE_NAME, Environment.getProperty(Environment.HADOOP_JOB_QUEUE, "default")); conf.set(Constants.SHIFU_MODELSET_SOURCE_TYPE, source.toString()); // set mapreduce.job.max.split.locations to 30 to suppress warnings conf.setInt(GuaguaMapReduceConstants.MAPREDUCE_JOB_MAX_SPLIT_LOCATIONS, 30); // Tmp set to false because of some cluster by default use gzip while CombineInputFormat will split gzip file (a // bug)// w w w .jav a 2 s . co m conf.setBoolean(CombineInputFormat.SHIFU_VS_SPLIT_COMBINABLE, false); conf.set("mapred.reduce.slowstart.completed.maps", Environment.getProperty("mapred.reduce.slowstart.completed.maps", "0.9")); Float wrapperRatio = this.modelConfig.getVarSelect().getWrapperRatio(); if (wrapperRatio == null) { log.warn("wrapperRatio in var select is not set. Using default value 0.05."); wrapperRatio = 0.05f; } if (wrapperRatio.compareTo(Float.valueOf(1.0f)) >= 0) { throw new IllegalArgumentException("WrapperRatio should be in (0, 1)."); } conf.setFloat(Constants.SHIFU_VARSELECT_WRAPPER_RATIO, wrapperRatio); String hdpVersion = HDPUtils.getHdpVersionForHDP224(); if (StringUtils.isNotBlank(hdpVersion)) { // for hdp 2.2.4, hdp.version should be set and configuration files should be add to container class path conf.set("hdp.version", hdpVersion); HDPUtils.addFileToClassPath(HDPUtils.findContainingFile("hdfs-site.xml"), conf); HDPUtils.addFileToClassPath(HDPUtils.findContainingFile("core-site.xml"), conf); HDPUtils.addFileToClassPath(HDPUtils.findContainingFile("mapred-site.xml"), conf); HDPUtils.addFileToClassPath(HDPUtils.findContainingFile("yarn-site.xml"), conf); } }
From source file:org.apache.apex.engine.YarnAppLauncherImpl.java
License:Apache License
private void setConfiguration(Configuration conf, String property, Object value) { if (value instanceof Integer) { conf.setInt(property, (Integer) value); } else if (value instanceof Boolean) { conf.setBoolean(property, (Boolean) value); } else if (value instanceof Long) { conf.setLong(property, (Long) value); } else if (value instanceof Float) { conf.setFloat(property, (Float) value); } else if (value instanceof Double) { conf.setDouble(property, (Double) value); } else {//from w w w.j a v a2s.c om conf.set(property, value.toString()); } }
From source file:org.apache.giraph.conf.FloatConfOption.java
License:Apache License
/** * Set value if it's not already present * * @param conf Configuration//from w w w .j a v a 2s . com * @param value to set */ public void setIfUnset(Configuration conf, float value) { if (!contains(conf)) { conf.setFloat(getKey(), value); } }
From source file:org.apache.giraph.hive.jython.HiveJythonUtils.java
License:Apache License
/** * Set arbitrary option of unknown type in Configuration * * @param conf Configuration/* ww w .java 2 s . c o m*/ * @param key String key * @param value Object to set */ private static void setOption(Configuration conf, String key, Object value) { if (value instanceof Boolean) { conf.getBoolean(key, (Boolean) value); } else if (value instanceof Byte || value instanceof Short || value instanceof Integer) { conf.setInt(key, ((Number) value).intValue()); } else if (value instanceof Long) { conf.setLong(key, (Long) value); } else if (value instanceof Float || value instanceof Double) { conf.setFloat(key, ((Number) value).floatValue()); } else if (value instanceof String) { conf.set(key, value.toString()); } else if (value instanceof Class) { conf.set(key, ((Class) value).getName()); } else { throw new IllegalArgumentException("Don't know how to handle option key: " + key + ", value: " + value + ", value type: " + value.getClass()); } }