Example usage for org.apache.hadoop.conf Configuration getClass

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration getClass.

Prototype

public Class<?> getClass(String name, Class<?> defaultValue)

Source Link

Document

Get the value of the name property as a Class.

Usage

From source file:org.mrgeo.format.FilteredFeatureInputFormat.java

License:Apache License

public static Class<? extends InputFormat<LongWritable, Geometry>> getInputFormat(Configuration conf) {
    if (conf.get(FilteredFeatureInputFormat.class.getName() + "input.format") == null) {
        return AutoFeatureInputFormat.class;
    }// w  w  w.j a  v  a 2  s.  c  om
    return (Class<? extends InputFormat<LongWritable, Geometry>>) conf
            .getClass(FilteredFeatureInputFormat.class.getName() + "input.format", null);
}

From source file:org.mrgeo.hadoop.multipleoutputs.MultipleOutputFormat.java

License:Apache License

public FileOutputFormat<K, V> getRootOutputFormat(final TaskAttemptContext job) {
    if (_innerFormat == null) {
        final Configuration conf = job.getConfiguration();
        final Class<?> c = conf.getClass(ROOT_OUTPUT_FORMAT, FileOutputFormat.class);
        try {/*from   w  ww . j  av  a 2 s. c om*/
            _innerFormat = (FileOutputFormat<K, V>) c.newInstance();
        } catch (final InstantiationException e) {
            throw new RuntimeException(e);
        } catch (final IllegalAccessException e) {
            throw new RuntimeException(e);
        }
    }

    return _innerFormat;
}

From source file:org.smartfrog.services.hadoop.operations.utils.DfsUtils.java

License:Open Source License

/**
 * This loads but does not initialise a filesystem.
 *
 * @param conf configuration//from w  w w .ja  va  2 s  .  c o  m
 * @param uri URI of the filesystem
 * @return an instance of that filesystem
 * @throws IOException if there is no filesystem of that type
 */
public static FileSystem loadFS(final Configuration conf, final URI uri) throws IOException {
    String scheme = uri.getScheme();
    String authority = uri.getAuthority();

    if (scheme == null) { // no scheme: use default FS
        return FileSystem.get(conf);
    }

    if (authority == null) { // no authority
        URI defaultUri = FileSystem.getDefaultUri(conf);
        if (scheme.equals(defaultUri.getScheme()) // if scheme matches default
                && defaultUri.getAuthority() != null) { // & default has authority
            return loadFS(conf, defaultUri); // return default
        }
    }

    String filesystemProp = "fs." + uri.getScheme() + ".impl";
    String implclass = conf.get(filesystemProp);
    Class<?> clazz = conf.getClass(filesystemProp, null);
    FileSystem.LOG.debug("Creating filesystem for " + uri + " implementation is implclass");
    if (clazz == null) {
        throw new IOException(
                "No FileSystem for scheme: " + uri.getScheme() + " and configuration option " + filesystemProp);
    }
    try {
        FileSystem fs = (FileSystem) ReflectionUtils.newInstance(clazz, conf);
        return fs;
    } catch (RuntimeException e) {
        throw new IOException(
                "Failed to create an instance of " + implclass + " to process " + uri.getScheme() + " : " + e,
                e);
    }
}

From source file:org.warcbase.mapreduce.lib.Chain.java

License:Apache License

/**
 * Add mapper(the first mapper) that reads input from the input
 * context and writes to queue//w  w w. j  av  a 2s.c om
 */
@SuppressWarnings("unchecked")
void addMapper(TaskInputOutputContext inputContext, ChainBlockingQueue<KeyValuePair<?, ?>> output, int index)
        throws IOException, InterruptedException {
    Configuration conf = getConf(index);
    Class<?> keyOutClass = conf.getClass(MAPPER_OUTPUT_KEY_CLASS, Object.class);
    Class<?> valueOutClass = conf.getClass(MAPPER_OUTPUT_VALUE_CLASS, Object.class);

    RecordReader rr = new ChainRecordReader(inputContext);
    RecordWriter rw = new ChainRecordWriter(keyOutClass, valueOutClass, output, conf);
    Mapper.Context mapperContext = createMapContext(rr, rw, (MapContext) inputContext, getConf(index));
    MapRunner runner = new MapRunner(mappers.get(index), mapperContext, rr, rw);
    threads.add(runner);
}

From source file:org.warcbase.mapreduce.lib.Chain.java

License:Apache License

/**
 * Add mapper(the last mapper) that reads input from
 * queue and writes output to the output context
 *//*  w  ww. j a  v  a  2s. co m*/
@SuppressWarnings("unchecked")
void addMapper(ChainBlockingQueue<KeyValuePair<?, ?>> input, TaskInputOutputContext outputContext, int index)
        throws IOException, InterruptedException {
    Configuration conf = getConf(index);
    Class<?> keyClass = conf.getClass(MAPPER_INPUT_KEY_CLASS, Object.class);
    Class<?> valueClass = conf.getClass(MAPPER_INPUT_VALUE_CLASS, Object.class);
    RecordReader rr = new ChainRecordReader(keyClass, valueClass, input, conf);
    RecordWriter rw = new ChainRecordWriter(outputContext);
    MapRunner runner = new MapRunner(mappers.get(index),
            createMapContext(rr, rw, outputContext, getConf(index)), rr, rw);
    threads.add(runner);
}

From source file:org.warcbase.mapreduce.lib.Chain.java

License:Apache License

/**
 * Add mapper that reads and writes from/to the queue
 *//*from w  ww .  java 2  s .  c  om*/
@SuppressWarnings("unchecked")
void addMapper(ChainBlockingQueue<KeyValuePair<?, ?>> input, ChainBlockingQueue<KeyValuePair<?, ?>> output,
        TaskInputOutputContext context, int index) throws IOException, InterruptedException {
    Configuration conf = getConf(index);
    Class<?> keyClass = conf.getClass(MAPPER_INPUT_KEY_CLASS, Object.class);
    Class<?> valueClass = conf.getClass(MAPPER_INPUT_VALUE_CLASS, Object.class);
    Class<?> keyOutClass = conf.getClass(MAPPER_OUTPUT_KEY_CLASS, Object.class);
    Class<?> valueOutClass = conf.getClass(MAPPER_OUTPUT_VALUE_CLASS, Object.class);
    RecordReader rr = new ChainRecordReader(keyClass, valueClass, input, conf);
    RecordWriter rw = new ChainRecordWriter(keyOutClass, valueOutClass, output, conf);
    MapRunner runner = new MapRunner(mappers.get(index), createMapContext(rr, rw, context, getConf(index)), rr,
            rw);
    threads.add(runner);
}

From source file:org.warcbase.mapreduce.lib.Chain.java

License:Apache License

protected static void validateKeyValueTypes(Configuration jobConf, Class<?> inputKeyClass,
        Class<?> inputValueClass, Class<?> outputKeyClass, Class<?> outputValueClass, int index,
        String prefix) {/*w  ww. j ava2  s.c  om*/
    if (index > 0) {
        // check the that the new Mapper in the chain key and value input classes
        // match those of the previous Mapper output.
        Configuration previousMapperConf = getChainElementConf(jobConf,
                prefix + CHAIN_MAPPER_CONFIG + (index - 1));
        if (!inputKeyClass.isAssignableFrom(previousMapperConf.getClass(MAPPER_OUTPUT_KEY_CLASS, null))) {
            throw new IllegalArgumentException("The specified Mapper input key class does"
                    + " not match the previous Mapper's output key class.");
        }
        if (!inputValueClass.isAssignableFrom(previousMapperConf.getClass(MAPPER_OUTPUT_VALUE_CLASS, null))) {
            throw new IllegalArgumentException("The specified Mapper input value class"
                    + " does not match the previous Mapper's output value class.");
        }
    }
}