Example usage for org.apache.hadoop.conf Configuration getStrings

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration getStrings.

Prototype

public String[] getStrings(String name) 

Source Link

Document

Get the comma delimited values of the name property as an array of Strings.

Usage

From source file:FormatStorage1.IHead.java

License:Open Source License

public void fromJobConf(Configuration conf) {
    this.magic = conf.getInt(ConstVar.HD_magic, ConstVar.NewFormatMagic);
    this.var = (byte) conf.getInt(ConstVar.HD_var, 0);
    this.ver = (byte) conf.getInt(ConstVar.HD_ver, 0);
    this.lineindex = (byte) conf.getInt(ConstVar.HD_lineindex, 1);
    this.primaryIndex = (short) conf.getInt(ConstVar.HD_primaryIndex, -1);
    this.compress = (byte) conf.getInt(ConstVar.HD_compress, 0);
    this.compressStyle = (byte) conf.getInt(ConstVar.HD_compressStyle, 0);
    this.encode = (byte) conf.getInt(ConstVar.HD_encode, 0);
    this.encodeStyle = (byte) conf.getInt(ConstVar.HD_encodeStyle, 0);

    this.fieldMap = new IFieldMap();
    String[] fieldStrings = conf.getStrings(ConstVar.HD_fieldMap);
    if (fieldStrings != null)
        for (int i = 0; i < fieldStrings.length; i++) {
            String[] def = fieldStrings[i].split(ConstVar.RecordSplit);
            byte type = Byte.valueOf(def[0]);
            int index = Integer.valueOf(def[2]);
            fieldMap.addFieldType(new IRecord.IFType(type, index));
        }/*from   w w  w.  j a va  2 s . c  o m*/

    this.udi = new IUserDefinedHeadInfo();
    String[] udistrs = conf.getStrings(ConstVar.HD_udi);
    if (udistrs != null)
        for (int i = 0; i < udistrs.length; i++) {
            String[] def = udistrs[i].split(ConstVar.RecordSplit);
            udi.addInfo(Integer.valueOf(def[0]), def[1]);
        }
}

---

From source file:full_MapReduce.AttributeInfoMapper.java

License:Open Source License

public void map(TextArrayWritable key, IntWritable value, Context context)
        throws IOException, InterruptedException {
    Configuration conf = context.getConfiguration();

    Writable[] key_values = key.get();/*  w  w w.j  a  v a  2 s  .c om*/
    Text[] attributes_and_class = new Text[key_values.length];
    for (Integer i = 0; i < key_values.length; ++i) {
        String[] value_needed = conf.getStrings(i.toString());

        if (value_needed == null) {
            attributes_and_class[i] = (Text) key_values[i];
        } else if (!value_needed[0].equals(((Text) key_values[i]).toString())) {
            return;
        }
    }

    Text classification = attributes_and_class[attributes_and_class.length - 1];

    for (Integer i = 0; i < attributes_and_class.length - 1; ++i) {
        if (attributes_and_class[i] != null) {
            context.write(new Text(i.toString()),
                    new AttributeCounterWritable(attributes_and_class[i], classification, value));
        }
    }

}

---

From source file:IndexService.IColumnRecordReader.java

License:Open Source License

IColumnRecordReader(Configuration conf, IColumnInputSplit split) throws IOException {

    this.conf = conf;

    String[] indexfilemap = conf.getStrings(ConstVar.HD_index_filemap);
    String splitfilename = split.getPath().toString();
    for (int i = 0; i < indexfilemap.length; i++) {
        if (splitfilename.endsWith(indexfilemap[i])) {
            this.fileindex = i;
            break;
        }/*  ww  w. jav a 2  s .c  om*/
    }

    indexkeys = new ArrayList<Integer>();
    String idss = conf.get("index.ids");

    String[] strs = idss.split(",");
    for (int i = 0; i < strs.length; i++) {
        indexkeys.add(Integer.parseInt(strs[i]));
    }

    icdf = new IColumnDataFile(conf);

    icdf.open(split.getPath().toString());

    if (split.wholefileASasplit) {
        this.beginLine = 0;
        this.endLine = icdf.recnum() - 1;
    } else {
        beginLine = split.beginline;
        endLine = split.beginline + split.recnum - 1;
    }

    currentline = beginLine;
    icdf.seek(currentline);
    record = icdf.getIRecordObj();

}

---

From source file:IndexService.IFormatRecordReader.java

License:Open Source License

public IFormatRecordReader(Configuration conf, IFormatInputSplit split) throws IOException {

    this.conf = conf;

    String[] indexfilemap = conf.getStrings(ConstVar.HD_index_filemap);
    String splitfilename = split.getPath().toString();
    for (int i = 0; i < indexfilemap.length; i++) {
        if (splitfilename.endsWith(indexfilemap[i])) {
            this.fileindex = i;
            break;
        }/*w ww  .  j  a  v  a2  s  .  c o  m*/
    }

    indexkeys = new ArrayList<Integer>();
    String idss = conf.get("index.ids");

    String[] strs = idss.split(",");
    for (int i = 0; i < strs.length; i++) {
        indexkeys.add(Integer.parseInt(strs[i]));
    }

    ifdf = new IFormatDataFile(conf);
    ifdf.open(split.getPath().toString());

    if (split.wholefileASasplit) {
        this.beginLine = 0;
        this.endLine = ifdf.segIndex().recnum() - 1;
    } else {
        beginLine = split.beginline;
        endLine = split.beginline + split.recnum - 1;
    }

    currentline = beginLine;
    ifdf.seek(currentline);
    record = ifdf.getIRecordObj();
}

---

From source file:IndexService.IndexMergeIFormatRecordReader.java

License:Open Source License

public IndexMergeIFormatRecordReader(Configuration conf, IndexMergeIFormatSplit split) throws IOException {
    this.conf = conf;

    this.filesmap = new HashMap<String, Integer>();
    String[] strs = conf.getStrings(ConstVar.HD_index_filemap);

    for (int i = 0; i < strs.length; i++) {
        filesmap.put(strs[i], i);//from ww  w  . j a  v a 2 s  .  c o  m
    }

    ifdf = new IFormatDataFile(conf);
    ifdf.open(split.getPath().toString());
    this.splitinfos = ifdf.fileInfo().head().getUdi().infos();

    if (split.wholefileASasplit) {
        this.recnum = ifdf.segIndex().recnum();
    } else {
        this.recnum = split.recnum;
    }

    ifdf.seek(split.beginline);
    record = ifdf.getIRecordObj();

}

---

From source file:io.apigee.lembos.mapreduce.LembosMapReduceEnvironment.java

License:Apache License

/**
 * Retrieves an initialized MapReduce environment.
 *
 * @param conf the Hadoop configuration//from w  ww. j  a v a2 s.c o  m
 *
 * @return the initialized MapReduce environment
 *
 * @throws ExecutionException if something goes wrong executing the Node.js module
 * @throws IOException if something goes wrong setting up the environment
 * @throws InterruptedException if the execution of the Node.js module gets interrupted
 * @throws NodeException if there is an issue with the Node.js module
 */
public static LembosMapReduceEnvironment fromConf(final Configuration conf)
        throws ExecutionException, InterruptedException, IOException, NodeException {
    File modulePath = new File(LembosConstants.MR_DISTRIBUTED_CACHE_SYMLINK);

    if (modulePath.exists()) {
        // Locate module in DistributedCache
        final File[] children = modulePath.listFiles();

        if (children == null) {
            throw new RuntimeException("Unable to find Node.js module in DistributedCache");
        }

        for (final File child : children) {
            final String moduleName = conf.get(LembosConstants.MR_MODULE_NAME);
            final String[] possibleNames = new String[] { moduleName, moduleName + ".js", moduleName + ".json",
                    moduleName + ".node" };

            if (Arrays.asList(possibleNames).contains(child.getName())) {
                modulePath = child;
                break;
            }
        }
    } else if (!modulePath.exists() && conf.get(LembosConstants.MR_MODULE_PATH) != null) {
        // Default to module path for testing and for LocalJobRunner (DistributedCache does not work in local mode)
        modulePath = new File(conf.get(LembosConstants.MR_MODULE_PATH));
    }

    final LembosMapReduceEnvironment mrEnv = new LembosMapReduceEnvironment(
            conf.get(LembosConstants.MR_MODULE_NAME), modulePath,
            conf.getStrings(LembosConstants.MR_MODULE_ARGS));

    mrEnv.initialize();

    return mrEnv;
}

---

From source file:mvm.rya.accumulo.pig.IndexWritingTool.java

License:Apache License

@Override
public int run(final String[] args) throws Exception {
    Preconditions.checkArgument(args.length == 7, "java " + IndexWritingTool.class.getCanonicalName()
            + " hdfsSaveLocation sparqlFile cbinstance cbzk cbuser cbpassword rdfTablePrefix.");

    final String inputDir = args[0];
    final String sparqlFile = args[1];
    final String instStr = args[2];
    final String zooStr = args[3];
    final String userStr = args[4];
    final String passStr = args[5];
    final String tablePrefix = args[6];

    String sparql = FileUtils.readFileToString(new File(sparqlFile));

    Job job = new Job(getConf(), "Write HDFS Index to Accumulo");
    job.setJarByClass(this.getClass());

    Configuration jobConf = job.getConfiguration();
    jobConf.setBoolean("mapred.map.tasks.speculative.execution", false);
    setVarOrders(sparql, jobConf);//from  w  w  w .  ja  v a2 s .  c o  m

    TextInputFormat.setInputPaths(job, inputDir);
    job.setInputFormatClass(TextInputFormat.class);

    job.setMapperClass(MyMapper.class);
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(Mutation.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Mutation.class);

    job.setNumReduceTasks(0);

    String tableName;
    if (zooStr.equals("mock")) {
        tableName = tablePrefix;
    } else {
        tableName = tablePrefix + "INDEX_" + UUID.randomUUID().toString().replace("-", "").toUpperCase();
    }
    setAccumuloOutput(instStr, zooStr, userStr, passStr, job, tableName);

    jobConf.set(sparql_key, sparql);

    int complete = job.waitForCompletion(true) ? 0 : -1;

    if (complete == 0) {

        String[] varOrders = jobConf.getStrings("varOrders");
        String orders = Joiner.on("\u0000").join(varOrders);
        Instance inst;

        if (zooStr.equals("mock")) {
            inst = new MockInstance(instStr);
        } else {
            inst = new ZooKeeperInstance(instStr, zooStr);
        }

        Connector conn = inst.getConnector(userStr, passStr.getBytes());
        BatchWriter bw = conn.createBatchWriter(tableName, 10, 5000, 1);

        Counters counters = job.getCounters();
        Counter c1 = counters.findCounter(cardCounter, cardCounter);

        Mutation m = new Mutation("~SPARQL");
        Value v = new Value(sparql.getBytes());
        m.put(new Text("" + c1.getValue()), new Text(orders), v);
        bw.addMutation(m);

        bw.close();

        return complete;
    } else {
        return complete;
    }

}

---

From source file:org.apache.accumulo.examples.wikisearch.ingest.WikipediaConfiguration.java

License:Apache License

/**
 * Helper method to get properties from Hadoop configuration
 *
 * @throws IllegalArgumentException/*from  ww w . j av a  2s. c om*/
 *           if property is not defined, null, or empty. Or if resultClass is not handled.
 * @return value of property
 */
@SuppressWarnings("unchecked")
public static <T> T isNull(Configuration conf, String propertyName, Class<T> resultClass) {
    String p = conf.get(propertyName);
    if (StringUtils.isEmpty(p)) {
        throw new IllegalArgumentException(propertyName + " must be specified");
    }

    if (resultClass.equals(String.class)) {
        return (T) p;
    } else if (resultClass.equals(String[].class)) {
        return (T) conf.getStrings(propertyName);
    } else if (resultClass.equals(Boolean.class)) {
        return (T) Boolean.valueOf(p);
    } else if (resultClass.equals(Long.class)) {
        return (T) Long.valueOf(p);
    } else if (resultClass.equals(Integer.class)) {
        return (T) Integer.valueOf(p);
    } else if (resultClass.equals(Float.class)) {
        return (T) Float.valueOf(p);
    } else if (resultClass.equals(Double.class)) {
        return (T) Double.valueOf(p);
    } else {
        throw new IllegalArgumentException(resultClass.getSimpleName() + " is unhandled.");
    }

}

---

From source file:org.apache.blur.mapreduce.lib.CsvBlurMapper.java

License:Apache License

public static Map<String, List<String>> getFamilyAndColumnNameMap(Configuration configuration) {
    Map<String, List<String>> columnNameMap = new HashMap<String, List<String>>();
    for (String family : getFamilyNames(configuration)) {
        String[] columnsNames = configuration.getStrings(BLUR_CSV_FAMILY_COLUMN_PREFIX + family);
        columnNameMap.put(family, Arrays.asList(columnsNames));
    }//w  w  w.j  a  v a 2 s  .co  m
    return columnNameMap;
}

---

From source file:org.apache.gora.mapreduce.GoraMapReduceUtils.java

License:Apache License

/**
 * Add our own serializer (obtained via the {@link PersistentSerialization} 
 * wrapper) to any other <code>io.serializations</code> which may be specified 
 * within existing Hadoop configuration.
 * //from   w ww .  j  ava  2 s  . c  o m
 * @param conf the Hadoop configuration object
 * @param reuseObjects boolean parameter to reuse objects
 */
public static void setIOSerializations(Configuration conf, boolean reuseObjects) {
    String serializationClass = PersistentSerialization.class.getCanonicalName();
    String[] serializations = StringUtils.joinStringArrays(conf.getStrings("io.serializations"),
            "org.apache.hadoop.io.serializer.WritableSerialization",
            StringSerialization.class.getCanonicalName(), serializationClass);
    conf.setStrings("io.serializations", serializations);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• »