Example usage for org.apache.hadoop.conf Configuration getClass

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration getClass.

Prototype

public <U> Class<? extends U> getClass(String name, Class<? extends U> defaultValue, Class<U> xface)

Source Link

Document

Get the value of the name property as a Class implementing the interface specified by xface.

Usage

From source file:com.thinkbiganalytics.kylo.catalog.aws.S3FileSystemProvider.java

License:Apache License

@Nonnull
@Override//from w w w .  jav  a 2  s .  c  o m
public List<DataSetFile> listFiles(@Nonnull final Path path, @Nonnull final Configuration conf) {
    // Determine the credentials
    final AmazonS3 s3;
    final URI uri = path.toUri();

    if ("s3".equalsIgnoreCase(uri.getScheme()) || "s3bfs".equalsIgnoreCase(uri.getScheme())
            || "s3n".equalsIgnoreCase(uri.getScheme())) {
        s3 = createS3Client(uri, conf);
    } else if ("s3a".equalsIgnoreCase(uri.getScheme())) {
        final Class<? extends S3ClientFactory> s3ClientFactoryClass = conf.getClass(
                Constants.S3_CLIENT_FACTORY_IMPL, Constants.DEFAULT_S3_CLIENT_FACTORY_IMPL,
                S3ClientFactory.class);
        try {
            s3 = ReflectionUtils.newInstance(s3ClientFactoryClass, conf).createS3Client(uri);
        } catch (final IOException e) {
            throw new IllegalArgumentException("Unable to create S3 client: " + e, e);
        }
    } else {
        log.debug("Scheme {} not supported for S3 path: {}", uri.getScheme(), path);
        throw new CatalogException("catalog.fs.s3.invalidScheme", uri.getScheme());
    }

    // Fetch the list of buckets
    try {
        return s3.listBuckets().stream().map(bucket -> {
            final DataSetFile file = new DataSetFile();
            file.setName(bucket.getName());
            file.setDirectory(true);
            file.setModificationTime(bucket.getCreationDate().getTime());
            file.setPath(uri.getScheme() + "://" + bucket.getName() + "/");
            return file;
        }).collect(Collectors.toList());
    } finally {
        s3.shutdown();
    }
}

From source file:com.zjy.mongo.util.MapredMongoConfigUtil.java

License:Apache License

public static Class<? extends Mapper> getMapper(final Configuration conf) {
    /** TODO - Support multiple inputs via getClasses ? **/
    return conf.getClass(MongoConfigUtil.JOB_MAPPER, null, Mapper.class);
}

From source file:com.zjy.mongo.util.MapredMongoConfigUtil.java

License:Apache License

public static Class<? extends Reducer> getCombiner(final Configuration conf) {
    return conf.getClass(MongoConfigUtil.JOB_COMBINER, null, Reducer.class);
}

From source file:com.zjy.mongo.util.MapredMongoConfigUtil.java

License:Apache License

public static Class<? extends Reducer> getReducer(final Configuration conf) {
    /** TODO - Support multiple outputs via getClasses ? **/
    return conf.getClass(MongoConfigUtil.JOB_REDUCER, null, Reducer.class);
}

From source file:com.zjy.mongo.util.MapredMongoConfigUtil.java

License:Apache License

public static Class<? extends Partitioner> getPartitioner(final Configuration conf) {
    return conf.getClass(MongoConfigUtil.JOB_PARTITIONER, null, Partitioner.class);
}

From source file:com.zjy.mongo.util.MapredMongoConfigUtil.java

License:Apache License

public static Class<? extends OutputFormat> getOutputFormat(final Configuration conf) {
    return conf.getClass(MongoConfigUtil.JOB_OUTPUT_FORMAT, null, OutputFormat.class);
}

From source file:com.zjy.mongo.util.MapredMongoConfigUtil.java

License:Apache License

public static Class<? extends InputFormat> getInputFormat(final Configuration conf) {
    return conf.getClass(MongoConfigUtil.JOB_INPUT_FORMAT, null, InputFormat.class);
}

From source file:edu.uci.ics.pregelix.api.util.BspUtils.java

License:Apache License

/**
 * Get the user's subclassed {@link VertexInputFormat}.
 * //from w w w . jav a2s .  co m
 * @param conf
 *            Configuration to check
 * @return User's vertex input format class
 */
@SuppressWarnings({ "rawtypes", "unchecked" })
public static <I extends WritableComparable, V extends Writable, E extends Writable, M extends Writable> Class<? extends VertexInputFormat<I, V, E, M>> getVertexInputFormatClass(
        Configuration conf) {
    return (Class<? extends VertexInputFormat<I, V, E, M>>) conf.getClass(PregelixJob.VERTEX_INPUT_FORMAT_CLASS,
            null, VertexInputFormat.class);
}

From source file:edu.uci.ics.pregelix.api.util.BspUtils.java

License:Apache License

/**
 * Get the user's subclassed {@link VertexOutputFormat}.
 * /*from ww  w  .ja  v  a  2 s.  c om*/
 * @param conf
 *            Configuration to check
 * @return User's vertex output format class
 */
@SuppressWarnings({ "rawtypes", "unchecked" })
public static <I extends WritableComparable, V extends Writable, E extends Writable> Class<? extends VertexOutputFormat<I, V, E>> getVertexOutputFormatClass(
        Configuration conf) {
    return (Class<? extends VertexOutputFormat<I, V, E>>) conf.getClass(PregelixJob.VERTEX_OUTPUT_FORMAT_CLASS,
            null, VertexOutputFormat.class);
}

From source file:edu.uci.ics.pregelix.api.util.BspUtils.java

License:Apache License

/**
 * Get the user's subclassed {@link MessageCombiner}.
 * /*from   ww  w . j  ava  2  s  . co m*/
 * @param conf
 *            Configuration to check
 * @return User's vertex combiner class
 */
@SuppressWarnings({ "rawtypes", "unchecked" })
public static <I extends WritableComparable, M extends Writable, P extends Writable> Class<? extends MessageCombiner<I, M, P>> getMessageCombinerClass(
        Configuration conf) {
    return (Class<? extends MessageCombiner<I, M, P>>) conf.getClass(PregelixJob.Message_COMBINER_CLASS,
            DefaultMessageCombiner.class, MessageCombiner.class);
}