Example usage for org.apache.hadoop.conf Configuration setClass

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration setClass.

Prototype

public void setClass(String name, Class<?> theClass, Class<?> xface) 

Source Link

Document

Set the value of the name property to the name of a theClass implementing the given interface xface.

Usage

From source file:com.cloudera.oryx.computation.common.JobStep.java

License:Open Source License

protected final Target compressedTextOutput(Configuration conf, String outputPathKey) {
    // The way this is used, it doesn't seem like we can just set the object in getConf(). Need
    // to set the copy in the MRPipeline directly?
    conf.setClass(FileOutputFormat.COMPRESS_CODEC, GzipCodec.class, CompressionCodec.class);
    conf.setClass(MRJobConfig.MAP_OUTPUT_COMPRESS_CODEC, SnappyCodec.class, CompressionCodec.class);
    return To.textFile(Namespaces.toPath(outputPathKey));
}

---

From source file:com.cloudera.sqoop.mapreduce.HBaseImportJob.java

License:Apache License

@Override
protected void configureOutputFormat(Job job, String tableName, String tableClassName)
        throws ClassNotFoundException, IOException {

    // Use the DelegatingOutputFormat with the HBasePutProcessor.
    job.setOutputFormatClass(getOutputFormatClass());

    Configuration conf = job.getConfiguration();
    conf.setClass("sqoop.output.delegate.field.map.processor.class", HBasePutProcessor.class,
            FieldMapProcessor.class);

    // Set the HBase parameters (table, column family, row key):
    conf.set(HBasePutProcessor.TABLE_NAME_KEY, options.getHBaseTable());
    conf.set(HBasePutProcessor.COL_FAMILY_KEY, options.getHBaseColFamily());

    // What column of the input becomes the row key?
    String rowKeyCol = options.getHBaseRowKeyColumn();
    if (null == rowKeyCol) {
        // User didn't explicitly set one. If there's a split-by column set,
        // use that.
        rowKeyCol = options.getSplitByCol();
    }//from   w  w w .j av  a2 s  .c om

    if (null == rowKeyCol) {
        // No split-by column is explicitly set.
        // If the table has a primary key, use that.
        ConnManager manager = getContext().getConnManager();
        rowKeyCol = manager.getPrimaryKey(tableName);
    }

    if (null == rowKeyCol) {
        // Give up here if this is still unset.
        throw new IOException("Could not determine the row-key column. "
                + "Use --hbase-row-key to specify the input column that " + "names each row.");
    }

    conf.set(HBasePutProcessor.ROW_KEY_COLUMN_KEY, rowKeyCol);
}

---

From source file:com.cloudera.sqoop.mapreduce.TestImportJob.java

License:Apache License

public void testFailedImportDueToJobFail() throws IOException {
    // Test that if the job returns 'false' it still fails and informs
    // the user./*from   ww  w  .j  ava 2 s.c  o m*/

    // Create a table to attempt to import.
    createTableForColType("VARCHAR(32)", "'meep2'");

    Configuration conf = new Configuration();

    // Use the dependency-injection manager.
    conf.setClass(ConnFactory.FACTORY_CLASS_NAMES_KEY, InjectableManagerFactory.class, ManagerFactory.class);

    String[] argv = getArgv(true, new String[] { "DATA_COL0" }, conf);

    // Use dependency injection to specify a mapper that we know
    // will fail.
    conf.setClass(InjectableConnManager.MAPPER_KEY, NullDereferenceMapper.class, Mapper.class);

    conf.setClass(InjectableConnManager.IMPORT_JOB_KEY, DummyImportJob.class, ImportJobBase.class);

    Sqoop importer = new Sqoop(new ImportTool(), conf);
    try {
        int ret = Sqoop.runSqoop(importer, argv);
        assertTrue("Expected ImportException running this job.", 1 == ret);
    } catch (Exception e) {
        // In debug mode, ImportException is wrapped in RuntimeException.
        LOG.info("Got exceptional return (expected: ok). msg is: " + e);
    }
}

---

From source file:com.datasalt.pangool.tuplemr.mapred.lib.output.PangoolMultipleOutputs.java

License:Apache License

/**
 * Adds a named output for the job. Returns the instance file that has been created.
 *//*from www. j  a va 2  s.com*/
public static String addNamedOutput(Job job, String namedOutput, OutputFormat outputFormat, Class<?> keyClass,
        Class<?> valueClass) throws FileNotFoundException, IOException, URISyntaxException {
    checkNamedOutputName(job, namedOutput, true);
    Configuration conf = job.getConfiguration();
    String uniqueName = UUID.randomUUID().toString() + '.' + "out-format.dat";
    InstancesDistributor.distribute(outputFormat, uniqueName, conf);
    conf.set(MULTIPLE_OUTPUTS, conf.get(MULTIPLE_OUTPUTS, "") + " " + namedOutput);
    conf.set(MO_PREFIX + namedOutput + FORMAT_INSTANCE_FILE, uniqueName);
    conf.setClass(MO_PREFIX + namedOutput + KEY, keyClass, Object.class);
    conf.setClass(MO_PREFIX + namedOutput + VALUE, valueClass, Object.class);
    return uniqueName;
}

---

From source file:com.example.cloud.bigtable.helloworld.BigtableHelper.java

License:Open Source License

/**
 * Connect will establish the connection to Cloud Bigtable.
 **//*from   w  w w .  jav a2 s . co m*/
public static void connect() throws IOException {
    Configuration c = HBaseConfiguration.create();

    c.setClass("hbase.client.connection.impl", com.google.cloud.bigtable.hbase1_1.BigtableConnection.class,
            org.apache.hadoop.hbase.client.Connection.class); // Required for Cloud Bigtable
    c.set("google.bigtable.endpoint.host", "bigtable.googleapis.com");
    c.set("google.bigtable.admin.endpoint.host", "bigtabletableadmin.googleapis.com");

    c.set("google.bigtable.project.id", PROJECT_ID);
    c.set("google.bigtable.cluster.name", CLUSTER_ID);
    c.set("google.bigtable.zone.name", ZONE);

    connection = ConnectionFactory.createConnection(c);
}

---

From source file:com.facebook.hiveio.conf.ClassConfOption.java

License:Apache License

/**
 * Set value for key/*w  w  w .jav a 2 s.  c  om*/
 * @param conf Configuration
 * @param klass Class to set
 */
public void set(Configuration conf, Class<? extends C> klass) {
    conf.setClass(getKey(), klass, interfaceClass);
}

---

From source file:com.facebook.presto.hive.HdfsConfiguration.java

License:Apache License

protected Configuration createConfiguration() {
    Configuration config = new Configuration();

    if (resourcePaths != null) {
        for (String resourcePath : resourcePaths) {
            config.addResource(new Path(resourcePath));
        }/*w ww .j  ava 2 s.c  om*/
    }

    // this is to prevent dfs client from doing reverse DNS lookups to determine whether nodes are rack local
    config.setClass("topology.node.switch.mapping.impl", NoOpDNSToSwitchMapping.class,
            DNSToSwitchMapping.class);

    if (socksProxy != null) {
        config.setClass("hadoop.rpc.socket.factory.class.default", SocksSocketFactory.class,
                SocketFactory.class);
        config.set("hadoop.socks.server", socksProxy.toString());
    }

    if (domainSocketPath != null) {
        config.setStrings("dfs.domain.socket.path", domainSocketPath);
    }

    // only enable short circuit reads if domain socket path is properly configured
    if (!config.get("dfs.domain.socket.path", "").trim().isEmpty()) {
        config.setBooleanIfUnset("dfs.client.read.shortcircuit", true);
    }

    config.setInt("dfs.socket.timeout", Ints.checkedCast(dfsTimeout.toMillis()));
    config.setInt("ipc.ping.interval", Ints.checkedCast(dfsTimeout.toMillis()));
    config.setInt("ipc.client.connect.timeout", Ints.checkedCast(dfsConnectTimeout.toMillis()));
    config.setInt("ipc.client.connect.max.retries", dfsConnectMaxRetries);

    // re-map filesystem schemes to match Amazon Elastic MapReduce
    config.set("fs.s3.impl", PrestoS3FileSystem.class.getName());
    config.set("fs.s3n.impl", PrestoS3FileSystem.class.getName());
    config.set("fs.s3bfs.impl", "org.apache.hadoop.fs.s3.S3FileSystem");

    // set AWS credentials for S3
    for (String scheme : ImmutableList.of("s3", "s3bfs", "s3n")) {
        if (s3AwsAccessKey != null) {
            config.set(format("fs.%s.awsAccessKeyId", scheme), s3AwsAccessKey);
        }
        if (s3AwsSecretKey != null) {
            config.set(format("fs.%s.awsSecretAccessKey", scheme), s3AwsSecretKey);
        }
    }

    // set config for S3
    config.setBoolean(PrestoS3FileSystem.S3_SSL_ENABLED, s3SslEnabled);
    config.setInt(PrestoS3FileSystem.S3_MAX_CLIENT_RETRIES, s3MaxClientRetries);
    config.setInt(PrestoS3FileSystem.S3_MAX_ERROR_RETRIES, s3MaxErrorRetries);
    config.set(PrestoS3FileSystem.S3_CONNECT_TIMEOUT, s3ConnectTimeout.toString());
    config.set(PrestoS3FileSystem.S3_STAGING_DIRECTORY, s3StagingDirectory.toString());

    updateConfiguration(config);

    return config;
}

---

From source file:com.facebook.presto.hive.HdfsConfigurationUpdater.java

License:Apache License

public void updateConfiguration(Configuration config) {
    if (resourcePaths != null) {
        for (String resourcePath : resourcePaths) {
            config.addResource(new Path(resourcePath));
        }//from  ww  w. ja v  a  2 s  .c o m
    }

    // this is to prevent dfs client from doing reverse DNS lookups to determine whether nodes are rack local
    config.setClass("topology.node.switch.mapping.impl", NoOpDNSToSwitchMapping.class,
            DNSToSwitchMapping.class);

    if (socksProxy != null) {
        config.setClass("hadoop.rpc.socket.factory.class.default", SocksSocketFactory.class,
                SocketFactory.class);
        config.set("hadoop.socks.server", socksProxy.toString());
    }

    if (domainSocketPath != null) {
        config.setStrings("dfs.domain.socket.path", domainSocketPath);
    }

    // only enable short circuit reads if domain socket path is properly configured
    if (!config.get("dfs.domain.socket.path", "").trim().isEmpty()) {
        config.setBooleanIfUnset("dfs.client.read.shortcircuit", true);
    }

    config.setInt("dfs.socket.timeout", toIntExact(dfsTimeout.toMillis()));
    config.setInt("ipc.ping.interval", toIntExact(ipcPingInterval.toMillis()));
    config.setInt("ipc.client.connect.timeout", toIntExact(dfsConnectTimeout.toMillis()));
    config.setInt("ipc.client.connect.max.retries", dfsConnectMaxRetries);

    // re-map filesystem schemes to match Amazon Elastic MapReduce
    config.set("fs.s3.impl", PrestoS3FileSystem.class.getName());
    config.set("fs.s3a.impl", PrestoS3FileSystem.class.getName());
    config.set("fs.s3n.impl", PrestoS3FileSystem.class.getName());
    config.set("fs.s3bfs.impl", "org.apache.hadoop.fs.s3.S3FileSystem");

    // set AWS credentials for S3
    if (s3AwsAccessKey != null) {
        config.set(PrestoS3FileSystem.S3_ACCESS_KEY, s3AwsAccessKey);
        config.set("fs.s3bfs.awsAccessKeyId", s3AwsAccessKey);
    }
    if (s3AwsSecretKey != null) {
        config.set(PrestoS3FileSystem.S3_SECRET_KEY, s3AwsSecretKey);
        config.set("fs.s3bfs.awsSecretAccessKey", s3AwsSecretKey);
    }
    if (s3Endpoint != null) {
        config.set(PrestoS3FileSystem.S3_ENDPOINT, s3Endpoint);
        config.set("fs.s3bfs.Endpoint", s3Endpoint);
    }
    if (s3SignerType != null) {
        config.set(PrestoS3FileSystem.S3_SIGNER_TYPE, s3SignerType.getSignerType());
    }

    config.setInt("fs.cache.max-size", fileSystemMaxCacheSize);

    configureCompression(config, compressionCodec);

    // set config for S3
    config.setBoolean(PrestoS3FileSystem.S3_USE_INSTANCE_CREDENTIALS, s3UseInstanceCredentials);
    config.setBoolean(PrestoS3FileSystem.S3_SSL_ENABLED, s3SslEnabled);
    config.setBoolean(PrestoS3FileSystem.S3_SSE_ENABLED, s3SseEnabled);
    if (s3EncryptionMaterialsProvider != null) {
        config.set(PrestoS3FileSystem.S3_ENCRYPTION_MATERIALS_PROVIDER, s3EncryptionMaterialsProvider);
    }
    if (s3KmsKeyId != null) {
        config.set(PrestoS3FileSystem.S3_KMS_KEY_ID, s3KmsKeyId);
    }
    config.setInt(PrestoS3FileSystem.S3_MAX_CLIENT_RETRIES, s3MaxClientRetries);
    config.setInt(PrestoS3FileSystem.S3_MAX_ERROR_RETRIES, s3MaxErrorRetries);
    config.set(PrestoS3FileSystem.S3_MAX_BACKOFF_TIME, s3MaxBackoffTime.toString());
    config.set(PrestoS3FileSystem.S3_MAX_RETRY_TIME, s3MaxRetryTime.toString());
    config.set(PrestoS3FileSystem.S3_CONNECT_TIMEOUT, s3ConnectTimeout.toString());
    config.set(PrestoS3FileSystem.S3_SOCKET_TIMEOUT, s3SocketTimeout.toString());
    config.set(PrestoS3FileSystem.S3_STAGING_DIRECTORY, s3StagingDirectory.toString());
    config.setInt(PrestoS3FileSystem.S3_MAX_CONNECTIONS, s3MaxConnections);
    config.setLong(PrestoS3FileSystem.S3_MULTIPART_MIN_FILE_SIZE, s3MultipartMinFileSize.toBytes());
    config.setLong(PrestoS3FileSystem.S3_MULTIPART_MIN_PART_SIZE, s3MultipartMinPartSize.toBytes());
    config.setBoolean(PrestoS3FileSystem.S3_PIN_CLIENT_TO_CURRENT_REGION, pinS3ClientToCurrentRegion);
    config.set(PrestoS3FileSystem.S3_USER_AGENT_PREFIX, s3UserAgentPrefix);
}

---

From source file:com.google.appengine.demos.mapreduce.TestServlet.java

License:Apache License

@Override
public void doGet(HttpServletRequest req, HttpServletResponse resp) {
    // Generate the configuration programmatically
    Configuration conf = new Configuration(false);
    try {//from   w  w w  .ja  va 2  s .c o  m
        conf.setClass("mapreduce.map.class", TestMapper.class, Mapper.class);
        conf.setClass("mapreduce.inputformat.class", DatastoreInputFormat.class, InputFormat.class);
        conf.set(DatastoreInputFormat.ENTITY_KIND_KEY, "PBFVotes");

        // Render it as an HTML form so that the user can edit it.
        String html = generateHtml(ConfigurationXmlUtil.convertConfigurationToXml(conf));
        PrintWriter pw = new PrintWriter(resp.getOutputStream());
        pw.println(html);
        pw.close();
    } catch (IOException e) {
        e.printStackTrace();
    }
}

---

From source file:com.google.appengine.tools.mapreduce.contrib.MapperTask.java

License:Apache License

public static String createConfiguration(String entityKind, Class<?> mapperClass,
        Map<String, String> additionalParams) {
    Configuration conf = new Configuration(false);
    conf.setClass("mapreduce.map.class", mapperClass, Mapper.class);
    conf.setClass("mapreduce.inputformat.class", DatastoreInputFormat.class, InputFormat.class);
    conf.set(DatastoreInputFormat.ENTITY_KIND_KEY, entityKind);
    for (Entry<String, String> entry : additionalParams.entrySet()) {
        conf.set(entry.getKey(), entry.getValue());
    }//from   w  w w .jav  a 2s  .c o  m

    return ConfigurationXmlUtil.convertConfigurationToXml(conf);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• »