Example usage for org.apache.hadoop.conf Configuration getStringCollection

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration getStringCollection.

Prototype

public Collection<String> getStringCollection(String name)

Source Link

Document

Get the comma delimited values of the name property as a collection of Strings.

Usage

From source file:org.elasticsearch.hadoop.mr.MultiOutputFormat.java

License:Apache License

public static void addOutputFormat(Configuration cfg, Class<? extends OutputFormat>... formats) {
    Collection<String> of = cfg.getStringCollection(CFG_FIELD);
    for (Class<? extends OutputFormat> format : formats) {
        of.add(format.getName());/*  w ww . j a v  a  2s .c om*/
    }
    cfg.setStrings(CFG_FIELD, StringUtils.join(of, ","));
}

From source file:org.kududb.mapreduce.KuduTableMapReduceUtil.java

License:Apache License

/**
 * Add the jars containing the given classes to the job's configuration
 * such that JobClient will ship them to the cluster and add them to
 * the DistributedCache./*from  w  w w .j  a v a  2s .  co m*/
 */
public static void addDependencyJars(Configuration conf, Class<?>... classes) throws IOException {

    FileSystem localFs = FileSystem.getLocal(conf);
    Set<String> jars = new HashSet<String>();
    // Add jars that are already in the tmpjars variable
    jars.addAll(conf.getStringCollection("tmpjars"));

    // add jars as we find them to a map of contents jar name so that we can avoid
    // creating new jars for classes that have already been packaged.
    Map<String, String> packagedClasses = new HashMap<String, String>();

    // Add jars containing the specified classes
    for (Class<?> clazz : classes) {
        if (clazz == null)
            continue;

        Path path = findOrCreateJar(clazz, localFs, packagedClasses);
        if (path == null) {
            LOG.warn("Could not find jar for class " + clazz + " in order to ship it to the cluster.");
            continue;
        }
        if (!localFs.exists(path)) {
            LOG.warn("Could not validate jar file " + path + " for class " + clazz);
            continue;
        }
        jars.add(path.toString());
    }
    if (jars.isEmpty())
        return;

    conf.set("tmpjars", StringUtils.arrayToString(jars.toArray(new String[jars.size()])));
}

From source file:org.mrgeo.utils.DependencyLoader.java

License:Apache License

@SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", justification = "File used for addinj to HDFS classpath")
public static Set<String> copyDependencies(final Set<String> localDependencies, Configuration conf)
        throws IOException {
    if (conf == null) {
        conf = HadoopUtils.createConfiguration();
    }/* w ww  . j  a  va2 s .  co m*/

    FileSystem fs = HadoopFileUtils.getFileSystem(conf);
    Path hdfsBase = new Path(MrGeoProperties.getInstance()
            .getProperty(MrGeoConstants.MRGEO_HDFS_DISTRIBUTED_CACHE, "/mrgeo/jars"));

    Set<String> deps = new HashSet<>();

    // prime the set with any dependencies already added
    deps.addAll(conf.getStringCollection("mapreduce.job.classpath.files"));
    //    deps.addAll(conf.getStringCollection(MRJobConfig.CLASSPATH_FILES));

    // copy the dependencies to hdfs, if needed
    for (String local : localDependencies) {
        File file = new File(local);

        addFileToClasspath(conf, deps, fs, hdfsBase, file);
    }

    Set<String> qualified = new HashSet<>();

    // fully qualify the dependency
    for (String dep : deps) {
        qualified.add(fs.makeQualified(new Path(dep)).toString());
    }

    return qualified;

}

From source file:org.springframework.data.hadoop.serialization.AbstractSequenceFileFormat.java

License:Apache License

/**
 * Adds the {@link Serialization} scheme to the configuration, so {@link SerializationFactory} instances are aware
 * of it./*from  w  ww .j  a  v a2  s.  c o m*/
 * 
 * @param serializationClass The Serialization classes to register to underlying configuration.
 */
@SuppressWarnings("rawtypes")
protected void registerSeqFileSerialization(Class<? extends Serialization>... serializationClasses) {

    Configuration conf = getConfiguration();

    Collection<String> serializations = conf.getStringCollection(HADOOP_IO_SERIALIZATIONS);

    for (Class<?> serializationClass : serializationClasses) {

        if (!serializations.contains(serializationClass.getName())) {

            serializations.add(serializationClass.getName());
        }
    }

    conf.setStrings(HADOOP_IO_SERIALIZATIONS, serializations.toArray(new String[serializations.size()]));
}