Example usage for org.apache.hadoop.conf Configuration setStrings

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration setStrings.

Prototype

public void setStrings(String name, String... values) 

Source Link

Document

Set the array of string values for the name property as as comma delimited values.

Usage

From source file:andromache.config.CassandraConfigHelper.java

License:Apache License

public static void setInputColumnFamilies(Configuration conf, String inputKeyspace,
        List<String> inputColumnFamilies) {
    String cf = "";
    if (inputKeyspace == null) {
        throw new UnsupportedOperationException("keyspace may not be null");
    }//from   ww  w.j  ava 2 s  .  co  m
    if (cf == null) {
        throw new UnsupportedOperationException("columnfamily may not be null");
    }

    conf.set(INPUT_KEYSPACE_CONFIG, inputKeyspace);
    conf.setStrings(INPUT_COLUMNFAMILIES_CONFIG,
            inputColumnFamilies.toArray(new String[inputColumnFamilies.size()]));
}

---

From source file:be.uantwerpen.adrem.bigfim.BigFIMDriver.java

License:Apache License

private static void setConfigurationValues(Configuration conf, FIMOptions opt) {
    conf.set(DELIMITER_KEY, opt.delimiter);
    conf.setInt(MIN_SUP_KEY, opt.minSup);
    conf.setInt(NUMBER_OF_MAPPERS_KEY, opt.nrMappers);
    conf.setInt(NUMBER_OF_CHUNKS, opt.nrMappers);
    conf.setInt(PREFIX_LENGTH_KEY, opt.prefixLength);
    conf.setStrings(OUTPUT_DIR_KEY, opt.outputDir);
}

---

From source file:be.uantwerpen.adrem.bigfim.ComputeTidListReducerTest.java

License:Apache License

private Configuration createConfiguration() throws Exception {
    Configuration conf = new Configuration();
    conf.setInt(MIN_SUP_KEY, 1);//w ww  .j a v  a2s.  c om
    conf.setInt(NUMBER_OF_MAPPERS_KEY, 2);
    conf.setInt(SUBDB_SIZE, 10);
    conf.setStrings(OUTPUT_DIR_KEY, "file:///out");
    return conf;
}

---

From source file:be.uantwerpen.adrem.disteclat.DistEclatDriver.java

License:Apache License

/**
 * Passes all configuration flags to the Hadoop Configuration framework
 * //from   w w  w. j ava2  s  .  c om
 * @param conf
 *          the Hadoop configuration
 * @param config
 *          the configuration that has user-defined flags
 */
private static void setConfigurationValues(Configuration conf, FIMOptions opt) {
    conf.set(DELIMITER_KEY, opt.delimiter);
    conf.setInt(MIN_SUP_KEY, opt.minSup);
    conf.setInt(NUMBER_OF_MAPPERS_KEY, opt.nrMappers);
    conf.setInt(NUMBER_OF_CHUNKS, opt.nrMappers);
    conf.setInt(PREFIX_LENGTH_KEY, opt.prefixLength);
    conf.setStrings(OUTPUT_DIR_KEY, opt.outputDir);
}

---

From source file:ca.uwaterloo.cs.bigdata2017w.assignment4.BuildPersonalizedPageRankRecords.java

License:Apache License

/**
 * Runs this tool.//from   w  w  w . ja v  a 2  s.  c om
 */
@SuppressWarnings({ "static-access" })
public int run(String[] args) throws Exception {
    Options options = new Options();

    options.addOption(OptionBuilder.withArgName("path").hasArg().withDescription("input path").create(INPUT));
    options.addOption(OptionBuilder.withArgName("path").hasArg().withDescription("output path").create(OUTPUT));
    options.addOption(
            OptionBuilder.withArgName("num").hasArg().withDescription("number of nodes").create(NUM_NODES));
    options.addOption(
            OptionBuilder.withArgName("sources").hasArg().withDescription("source nodes").create(SOURCES));

    CommandLine cmdline;
    CommandLineParser parser = new GnuParser();

    try {
        cmdline = parser.parse(options, args);
    } catch (ParseException exp) {
        System.err.println("Error parsing command line: " + exp.getMessage());
        return -1;
    }

    if (!cmdline.hasOption(INPUT) || !cmdline.hasOption(OUTPUT) || !cmdline.hasOption(NUM_NODES)) {
        System.out.println("args: " + Arrays.toString(args));
        HelpFormatter formatter = new HelpFormatter();
        formatter.setWidth(120);
        formatter.printHelp(this.getClass().getName(), options);
        ToolRunner.printGenericCommandUsage(System.out);
        return -1;
    }

    String inputPath = cmdline.getOptionValue(INPUT);
    String outputPath = cmdline.getOptionValue(OUTPUT);
    int n = Integer.parseInt(cmdline.getOptionValue(NUM_NODES));
    String sourcesString = cmdline.getOptionValue(SOURCES);
    String[] sources = sourcesString.split(",");
    for (int i = 0; i < sources.length; i++) {
        sources[i] = sources[i].trim();
    }

    LOG.info("Tool name: " + BuildPersonalizedPageRankRecords.class.getSimpleName());
    LOG.info(" - inputDir: " + inputPath);
    LOG.info(" - outputDir: " + outputPath);
    LOG.info(" - numNodes: " + n);
    LOG.info(" - use sources: " + sourcesString);

    Configuration conf = getConf();
    conf.setInt(NODE_CNT_FIELD, n);
    conf.setInt("mapred.min.split.size", 1024 * 1024 * 1024);
    conf.setStrings(SOURCES, sources);

    Job job = Job.getInstance(conf);
    job.setJobName(BuildPersonalizedPageRankRecords.class.getSimpleName() + ":" + inputPath);
    job.setJarByClass(BuildPersonalizedPageRankRecords.class);

    job.setNumReduceTasks(0);

    FileInputFormat.addInputPath(job, new Path(inputPath));
    FileOutputFormat.setOutputPath(job, new Path(outputPath));

    job.setInputFormatClass(TextInputFormat.class);
    job.setOutputFormatClass(SequenceFileOutputFormat.class);

    job.setMapOutputKeyClass(IntWritable.class);
    job.setMapOutputValueClass(PageRankNode.class);

    job.setOutputKeyClass(IntWritable.class);
    job.setOutputValueClass(PageRankNode.class);

    job.setMapperClass(MyMapper.class);

    // Delete the output directory if it exists already.
    FileSystem.get(conf).delete(new Path(outputPath), true);

    job.waitForCompletion(true);

    return 0;
}

---

From source file:cascading.scheme.DeprecatedAvroScheme.java

License:Apache License

private void addAvroSerializations(Configuration conf) {
    Collection<String> serializations = conf.getStringCollection("io.serializations");
    if (!serializations.contains(AvroSerialization.class.getName())) {
        serializations.add(AvroSerialization.class.getName());
        serializations.add(AvroSpecificRecordSerialization.class.getName());
    }//from   w  w  w  .  j  ava2 s  .c  o m

    conf.setStrings("io.serializations", serializations.toArray(new String[serializations.size()]));
}

---

From source file:cascading.tap.hadoop.DistCacheTap.java

License:Open Source License

@Override
protected void addLocalCacheFiles(Configuration conf, URI uri) {
    String key = CASCADING_LOCAL_RESOURCES + Tap.id(this);
    Collection<String> resources = conf.getStringCollection(key);

    if (resources == null)
        resources = new ArrayList<>();

    resources.add(uri.toString());/*from   ww w  . j  a v a  2s .co  m*/

    conf.setStrings(key, resources.toArray(new String[resources.size()]));
}

---

From source file:co.cask.cdap.data2.transaction.snapshot.SnapshotCodecCompatibilityTest.java

License:Apache License

@Test
public void testV1CodecV2Compat() throws Exception {

    long now = System.currentTimeMillis();

    // NOTE: set visibilityUpperBound to 0 as this is expected default for decoding older version that doesn't store it
    TreeMap<Long, TransactionManager.InProgressTx> inProgress = Maps
            .newTreeMap(ImmutableSortedMap.of(16L, new TransactionManager.InProgressTx(0L, now + 1000), 17L,
                    new TransactionManager.InProgressTx(0L, now + 1000)));

    TransactionSnapshot snapshot = new TransactionSnapshot(now, 15, 18, Lists.newArrayList(5L, 7L), inProgress,
            ImmutableMap.<Long, Set<ChangeId>>of(17L,
                    Sets.newHashSet(new ChangeId(Bytes.toBytes("ch1")), new ChangeId(Bytes.toBytes("ch2")))),
            ImmutableMap.<Long, Set<ChangeId>>of(16L,
                    Sets.newHashSet(new ChangeId(Bytes.toBytes("ch2")), new ChangeId(Bytes.toBytes("ch3")))));

    Configuration configV1 = HBaseConfiguration.create();
    configV1.setStrings(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, SnapshotCodecV1.class.getName());

    SnapshotCodecProvider codecV1 = new SnapshotCodecProvider(configV1);

    // encoding with codec of v1
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    try {//from  ww  w . ja v a 2s.  c  om
        codecV1.encode(out, snapshot);
    } finally {
        out.close();
    }

    // decoding
    Configuration configV1V2 = HBaseConfiguration.create();
    configV1V2.setStrings(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, SnapshotCodecV1.class.getName(),
            SnapshotCodecV2.class.getName());
    SnapshotCodecProvider codecV1V2 = new SnapshotCodecProvider(configV1V2);
    TransactionSnapshot decoded = codecV1V2.decode(new ByteArrayInputStream(out.toByteArray()));

    assertEquals(snapshot, decoded);
}

---

From source file:com.asakusafw.runtime.stage.output.StageOutputDriver.java

License:Apache License

private static void addOutput(Job job, String name, Class<?> formatClass, Class<?> keyClass,
        Class<?> valueClass) {
    assert job != null;
    assert name != null;
    assert formatClass != null;
    assert keyClass != null;
    assert valueClass != null;
    if (isValidName(name) == false) {
        throw new IllegalArgumentException(MessageFormat.format("Output name \"{0}\" is not valid", name));
    }//from w w w  . ja  va  2  s .  c  o m
    Configuration conf = job.getConfiguration();
    Set<String> names = new TreeSet<>(conf.getStringCollection(K_NAMES));
    if (names.contains(name)) {
        throw new IllegalArgumentException(
                MessageFormat.format("Output name \"{0}\" is already declared", name));
    }
    names.add(name);
    conf.setStrings(K_NAMES, names.toArray(new String[names.size()]));
    conf.setClass(getPropertyName(K_FORMAT_PREFIX, name), formatClass, OutputFormat.class);
    conf.setClass(getPropertyName(K_KEY_PREFIX, name), keyClass, Object.class);
    conf.setClass(getPropertyName(K_VALUE_PREFIX, name), valueClass, Object.class);
}

---

From source file:com.asakusafw.runtime.stage.resource.StageResourceDriver.java

License:Apache License

/**
 * Adds a resource path into the target job object.
 * @param job the target job/*from w w  w .ja va 2s. co  m*/
 * @param resourcePath the resource path expression (this must be accessible from task execution nodes)
 * @param resourceName the resource name
 * @throws IOException if failed to detect resources on the path
 * @throws IllegalArgumentException if some parameters are {@code null}
 */
public static void add(Job job, String resourcePath, String resourceName) throws IOException {
    if (job == null) {
        throw new IllegalArgumentException("job must not be null"); //$NON-NLS-1$
    }
    if (resourcePath == null) {
        throw new IllegalArgumentException("resourcePath must not be null"); //$NON-NLS-1$
    }
    if (resourceName == null) {
        throw new IllegalArgumentException("resourceName must not be null"); //$NON-NLS-1$
    }
    Configuration conf = job.getConfiguration();
    List<FileStatus> list = TemporaryStorage.listStatus(conf, new Path(resourcePath));
    if (list.isEmpty()) {
        throw new IOException(MessageFormat.format("Resource not found: {0}", resourcePath));
    }
    List<String> localNames = restoreStrings(conf, getLocalCacheNameKey(resourceName));
    List<String> remotePaths = restoreStrings(conf, getRemotePathKey(resourceName));
    long size = conf.getLong(KEY_SIZE, 0L);
    int index = localNames.size();
    for (FileStatus status : list) {
        String name = String.format("%s-%04d", resourceName, index++); //$NON-NLS-1$
        StringBuilder buf = new StringBuilder();
        buf.append(status.getPath().toString());
        buf.append('#');
        buf.append(name);
        String cachePath = buf.toString();

        remotePaths.add(status.getPath().toString());
        localNames.add(name);
        try {
            URI uri = new URI(cachePath);
            DistributedCache.addCacheFile(uri, conf);
        } catch (URISyntaxException e) {
            throw new IllegalStateException(e);
        }
        size += status.getLen();
    }
    conf.setStrings(getLocalCacheNameKey(resourceName), localNames.toArray(new String[localNames.size()]));
    conf.setStrings(getRemotePathKey(resourceName), remotePaths.toArray(new String[remotePaths.size()]));
    conf.setLong(KEY_SIZE, size);
    if (JobCompatibility.isLocalMode(job)) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("symlinks for distributed cache will not be created in standalone mode"); //$NON-NLS-1$
        }
    } else {
        DistributedCache.createSymlink(conf);
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• »