Example usage for org.apache.hadoop.conf Configuration getClassByName

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration getClassByName.

Prototype

public Class<?> getClassByName(String name) throws ClassNotFoundException

Source Link

Document

Load a class by name.

Usage

From source file:org.commoncrawl.server.CommonCrawlServer.java

License:Open Source License

public static void main(String argv[]) throws Exception {

    try {//from   ww  w.  j a va  2s  .c o m
        Configuration conf = new Configuration();

        conf.addResource("nutch-default.xml");
        conf.addResource("nutch-site.xml");
        conf.addResource("core-site.xml");
        conf.addResource("mapred-site.xml");
        conf.addResource("hdfs-site.xml");
        // conf.addResource("hadoop-site.xml");
        // conf.addResource("commoncrawl-default.xml");
        // conf.addResource("commoncrawl-site.xml");

        /*
         * conf.setClassLoader( new ClassLoader() {
         * 
         * @Override protected Class<?> findClass(String name) throws
         * ClassNotFoundException { if (name.startsWith("org.crawlcommons")) { //
         * this is a hack to deal with the problem of having a bunch of serialized
         * data in hdfs sequence files that referes to // protocol buffer with the
         * old crawler package name of org.crawlcommons instead of the new package
         * name of org.commoncrawl // we replace the crawlcommons string and call
         * back into the class loader to re-resolve the name... name =
         * name.replaceFirst("org.crawlcommons", "org.commoncrawl"); return
         * loadClass(name); }
         * 
         * return super.findClass(name); }
         * 
         * });
         */

        // set this config object as our global config
        CrawlEnvironment.setHadoopConfig(conf);

        _commonConfig = parseCommonConfig(argv);

        if (_commonConfig._className == null) {
            printCommonUsage();
            return;
        }

        if (_commonConfig._hostName != null) {
            conf.set("org.commoncrawl.hostname", _commonConfig._hostName);
        }

        if (_commonConfig._rpcInterface != null) {
            conf.set("org.commoncrawl.rpcInterface", _commonConfig._rpcInterface);
        }

        if (_commonConfig._webInterface != null) {
            conf.set("org.commoncrawl.httpInterface", _commonConfig._webInterface);
        }

        if (_commonConfig._rpcPort != -1) {
            conf.setInt("org.commoncrawl.rpcPort", _commonConfig._rpcPort);
        }
        if (_commonConfig._webPort != -1) {
            conf.setInt("org.commoncrawl.httpPort", _commonConfig._webPort);
        }
        if (_commonConfig._dataDir != null) {
            conf.set("org.commoncrawl.dataDir", _commonConfig._dataDir);
        }
        if (_commonConfig._dnsThreadPoolSize != -1) {
            conf.setInt("org.commoncrawl.dnsThreadPoolSize", _commonConfig._dnsThreadPoolSize);
        }
        LOG.info("Log File Is:" + System.getProperty("commoncrawl.log.file"));

        LOG.info("Instantiating Class:" + _commonConfig._className);
        Class theClass = conf.getClassByName(_commonConfig._className);

        Object serverInstance = theClass.newInstance();
        CommonCrawlServer server = CommonCrawlServer.class.cast(serverInstance);

        StringUtils.startupShutdownMessage(theClass, argv, LOG);

        if (server.init(argv, conf)) {
            try {

                server.start();
                server.join();

            } catch (IOException e) {
                LOG.error(StringUtils.stringifyException(e));
                throw e;
            } finally {
                server.stopDaemons();
                server.stop();
            }
        }

    } catch (Throwable e) {
        LOG.error(StringUtils.stringifyException(e));
        e.printStackTrace();
        System.exit(-1);
    }
}

From source file:skewtune.mapreduce.lib.input.MapOutputInputStream.java

License:Apache License

/**
 * From org.apache.hadoop.mapred.JobConf
 * /*from  w  w w .j  a  va  2  s . co  m*/
 * Get the {@link CompressionCodec} for compressing the map outputs.
 * 
 * @param defaultValue
 *            the {@link CompressionCodec} to return if not set
 * @return the {@link CompressionCodec} class that should be used to
 *         compress the map outputs.
 * @throws IllegalArgumentException
 *             if the class was specified, but not found
 */
Class<? extends CompressionCodec> getMapOutputCompressorClass(Configuration conf,
        Class<? extends CompressionCodec> defaultValue) {
    Class<? extends CompressionCodec> codecClass = defaultValue;
    String name = conf.get(JobContext.MAP_OUTPUT_COMPRESS_CODEC);
    if (name != null) {
        try {
            codecClass = conf.getClassByName(name).asSubclass(CompressionCodec.class);
        } catch (ClassNotFoundException e) {
            throw new IllegalArgumentException("Compression codec " + name + " was not found.", e);
        }
    }
    return codecClass;
}

From source file:tap.formats.avro.TapAvroSerialization.java

License:Apache License

private static final Class<?> getMapOutClass(Configuration conf) {
    try {/*from  w  w  w .j  a  v  a2  s . c  o  m*/
        return conf.getClassByName(conf.get(Phase.MAP_OUT_CLASS));
    } catch (ClassNotFoundException e) {
        throw new RuntimeException(e);
    }
}