List of usage examples for org.apache.hadoop.conf Configuration getStrings
public String[] getStrings(String name)
name property as an array of Strings. From source file:org.imageterrier.indexers.hadoop.HadoopIndexer.java
License:Mozilla Public License
private static HadoopIndexerOptions getOptions(Configuration conf) throws IOException { final String[] args = conf.getStrings(INDEXER_ARGS_STRING); final HadoopIndexerOptions options = new HadoopIndexerOptions(); final CmdLineParser parser = new CmdLineParser(options); try {// w w w . j ava 2 s .c om parser.parseArgument(args); } catch (final CmdLineException e) { throw new IOException(e); } return options; }
From source file:org.jc.mrsqoophelper.mapper.Mapper.java
@Override protected void setup(Context context) throws IOException, InterruptedException { super.setup(context); //To change body of generated methods, choose Tools | Templates. Configuration jobConf = context.getConfiguration(); this.fieldConditionOutputTriple = jobConf.getStrings(SqoopHelperMain.FIELD_COND_OUTPUT_TRIPLET); this.tripletDelimiter = jobConf.get(SqoopHelperMain.TRIPLET_ELEMENTS_DELIMITER, ","); this.fqcnRecord = jobConf.getStrings(SqoopHelperMain.FQCN_RECORD_CLASS)[0]; this.avsc = new Schema.Parser().parse(jobConf.get(SqoopHelperMain.AVRO_SCHEMA_AS_JSON)); boolean classIsAvailable = true; if (!Utils.classExists(this.fqcnRecord)) { try {/* w w w . j a v a2 s . c om*/ Utils.ClassBuilder(jobConf.get(SqoopHelperMain.AVRO_SCHEMA_AS_JSON), jobConf.get(SqoopHelperMain.PACKAGE_NAME), jobConf.get(SqoopHelperMain.CLASS_ABSOLUTE_PATH), jobConf.get(SqoopHelperMain.SRC_ABSOLUTE_PATH)); } catch (Exception e) { Logger.getLogger(Mapper.class.getName()).log(Level.SEVERE, null, e); classIsAvailable = false; } } if (classIsAvailable) { try { this.clazz = Class.forName(this.fqcnRecord); Constructor<BaseRecord> ctr = this.clazz.getDeclaredConstructor(Schema.class, String.class); this.instance = ctr.newInstance(avsc, jobConf.get(SqoopHelperMain.AVRO_SCHEMA_AS_JSON)); } catch (ClassNotFoundException ex) { Logger.getLogger(Mapper.class.getName()).log(Level.SEVERE, null, ex); } catch (InstantiationException ex) { Logger.getLogger(Mapper.class.getName()).log(Level.SEVERE, null, ex); } catch (NoSuchMethodException ex) { Logger.getLogger(Mapper.class.getName()).log(Level.SEVERE, null, ex); } catch (IllegalAccessException ex) { Logger.getLogger(Mapper.class.getName()).log(Level.SEVERE, null, ex); } catch (InvocationTargetException ex) { Logger.getLogger(Mapper.class.getName()).log(Level.SEVERE, null, ex); } } }
From source file:org.jc.mrsqoophelper.reducer.Reducer.java
@Override protected void setup(Context context) throws IOException, InterruptedException { Configuration jobConf = context.getConfiguration(); this.fqcnRecordClassName = jobConf.getStrings(SqoopHelperMain.FQCN_RECORD_CLASS)[0]; this.avroSchemaAsJson = jobConf.get(SqoopHelperMain.AVRO_SCHEMA_AS_JSON); if (!Utils.classExists(this.fqcnRecordClassName)) { try {/*from w ww. ja v a 2 s .c o m*/ Utils.ClassBuilder(jobConf.get(this.avroSchemaAsJson), jobConf.get(SqoopHelperMain.PACKAGE_NAME), jobConf.get(SqoopHelperMain.CLASS_ABSOLUTE_PATH), jobConf.get(SqoopHelperMain.SRC_ABSOLUTE_PATH)); } catch (Exception e) { Logger.getLogger(Mapper.class.getName()).log(Level.SEVERE, null, e); } } this.schema = new Schema.Parser().parse(this.avroSchemaAsJson); this.writer = new GenericDatumWriter<>(this.schema); this.writer.setSchema(this.schema); this.dataFileWriter = new DataFileWriter<>(writer); }
From source file:org.schedoscope.export.ftp.outputformat.FtpUploadOutputFormat.java
License:Apache License
@Override public RecordWriter<K, V> getRecordWriter(TaskAttemptContext context) throws IOException { Configuration conf = context.getConfiguration(); boolean isCompressed = getCompressOutput(context); CompressionCodec codec = null;/*from w w w. ja v a 2 s. co m*/ if (isCompressed) { Class<? extends CompressionCodec> codecClass = getOutputCompressorClass(context, GzipCodec.class); // only support gzip and bzip2 compression if (codecClass.equals(BZip2Codec.class) || codecClass.equals(GzipCodec.class)) { codec = ReflectionUtils.newInstance(codecClass, conf); extension = codec.getDefaultExtension(); } else { LOG.warn("neither gzip nor bzip2 compression codec found - disabling compression"); isCompressed = false; extension = ""; } } char delimiter = conf.get(FTP_EXPORT_CVS_DELIMITER, "\t").charAt(0); String[] header = conf.getStrings(FTP_EXPORT_HEADER_COLUMNS); Path file = getDefaultWorkFile(context, extension); FileSystem fs = file.getFileSystem(conf); FSDataOutputStream fileOut = fs.create(file, false); RecordWriter<K, V> writer; if (conf.get(FTP_EXPORT_FILE_TYPE).equals(FileOutputType.csv.toString())) { if (!isCompressed) { writer = new CSVRecordWriter<K, V>(fileOut, header, delimiter); } else { writer = new CSVRecordWriter<K, V>(new DataOutputStream(codec.createOutputStream(fileOut)), header, delimiter); } } else if (conf.get(FTP_EXPORT_FILE_TYPE).equals(FileOutputType.json.toString())) { if (!isCompressed) { writer = new JsonRecordWriter<K, V>(fileOut); } else { writer = new JsonRecordWriter<K, V>(new DataOutputStream(codec.createOutputStream(fileOut))); } } else { throw new IllegalArgumentException("unknown file output type"); } return writer; }
From source file:pl.edu.icm.coansys.richimporttsv.jobs.mapreduce.RichImportTsv.java
License:Apache License
public static void main(Configuration conf, String[] args) throws Exception { if (conf == null) { conf = HBaseConfiguration.create(); }/*w ww. ja v a 2s. co m*/ String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); if (otherArgs.length < 2) { usage("Wrong number of arguments: " + otherArgs.length); System.exit(-1); } // Make sure columns are specified String columns[] = conf.getStrings(COLUMNS_CONF_KEY); if (columns == null) { usage("No columns specified. Please specify with -D" + COLUMNS_CONF_KEY + "=..."); System.exit(-1); } // Make sure they specify exactly one column as the row key int rowkeysFound = 0; for (String col : columns) { if (col.equals(ROWKEY_COLUMN_SPEC)) { rowkeysFound++; } } if (rowkeysFound != 1) { usage("Must specify exactly one column as " + ROWKEY_COLUMN_SPEC); System.exit(-1); } // Make sure one or more columns are specified if (columns.length < 2) { usage("One or more columns in addition to the row key are required"); System.exit(-1); } Job job = createSubmittableJob(conf, otherArgs); System.exit(job.waitForCompletion(true) ? 0 : 1); }
From source file:uk.ac.cam.eng.extraction.hadoop.features.lexical.TTableServer.java
License:Apache License
private void setup(Configuration conf, String direction, boolean source2Target) throws IOException, InterruptedException { int serverPort; if (source2Target) { serverPort = Integer.parseInt(conf.get(TTABLE_S2T_SERVER_PORT)); } else {/* www . j ava 2 s. c o m*/ serverPort = Integer.parseInt(conf.get(TTABLE_T2S_SERVER_PORT)); } minLexProb = Double.parseDouble(conf.get("min_lex_prob")); serverSocket = new ServerSocket(serverPort); String lexTemplate = conf.get(LEX_TABLE_TEMPLATE); String allString = lexTemplate.replace(GENRE, "ALL").replace(DIRECTION, direction); System.out.println("Loading " + allString); String[] provenances = conf.getStrings(ProvenanceCountMap.PROV); ExecutorService loaderThreadPool = Executors.newFixedThreadPool(4); model.put((byte) 0, new HashMap<Integer, Map<Integer, Double>>()); loaderThreadPool.execute(new LoadTask(allString, (byte) 0)); for (int i = 0; i < provenances.length; ++i) { String provString = lexTemplate.replace(GENRE, provenances[i]).replace(DIRECTION, direction); System.out.println("Loading " + provString); byte prov = (byte) (i + 1); model.put(prov, new HashMap<Integer, Map<Integer, Double>>()); loaderThreadPool.execute(new LoadTask(provString, prov)); } loaderThreadPool.shutdown(); loaderThreadPool.awaitTermination(3, TimeUnit.HOURS); System.gc(); }
From source file:uk.ac.cam.eng.rule.retrieval.RuleFilter.java
License:Apache License
public RuleFilter(Configuration conf) throws FileNotFoundException, IOException { int alls2t = conf.getInt("source2target_probability-mapreduce", 0); s2tIndices.put("", alls2t); t2sIndices.put("", conf.getInt("target2source_probability-mapreduce", 0)); comparators.put("", new RuleCountComparator(IntWritableCache.createIntWritable(alls2t + 1))); for (String provenance : conf.getStrings(ProvenanceCountMap.PROV)) { int s2tIndex = conf.getInt("provenance_source2target_probability-" + provenance + "-mapreduce", 0); s2tIndices.put(provenance, s2tIndex); int t2sIndex = conf.getInt("provenance_target2source_probability-" + provenance + "-mapreduce", 0); t2sIndices.put(provenance, t2sIndex); comparators.put(provenance, new RuleCountComparator(IntWritableCache.createIntWritable(s2tIndex + 1))); }//from w w w .ja v a2 s . c om String filterConfig = conf.get("filter_config"); if (filterConfig == null) { System.err.println("Missing property 'filter_config' in the config"); System.exit(1); } loadConfig(filterConfig); System.out.println(sourcePatternConstraints); }
From source file:uk.ac.gla.terrier.probos.controller.KittenUtils2.java
License:Open Source License
public static boolean detectUseCp(Configuration pConf, final String path) { boolean cp = false; final String[] USE_RCP_DIRS = pConf.getStrings(PConfiguration.KEY_RCP_USE); if (USE_RCP_DIRS != null) { if (LOG.isDebugEnabled()) { LOG.debug("Checking is " + path + " matches in " + Arrays.deepToString(USE_RCP_DIRS)); }// www . j a va2 s. c om for (String prefix : USE_RCP_DIRS) { if (path.startsWith(prefix)) { cp = true; LOG.debug(path + " matches in " + prefix); break; } } } else { LOG.warn(PConfiguration.KEY_RCP_USE + " was null, it should normally be at least empty."); } return cp; }