Example usage for org.apache.hadoop.conf Configuration setInt

List of usage examples for org.apache.hadoop.conf Configuration setInt

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.conf.*
  5. Configuration
  6. setInt

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration setInt.

Prototype

public void setInt(String name, int value)

Source Link

Document

Set the value of the name property to an int.

Usage

From source file:com.ery.hadoop.mrddx.file.RCFileOutputFormat.java

License:Apache License

public static void setColumnNumber(Configuration conf, int columnNum) {
    assert columnNum > 0;
    conf.setInt(RCFile.COLUMN_NUMBER_CONF_STR, columnNum);
}

From source file:com.facebook.hive.orc.OrcConf.java

License:Open Source License

public static void setIntVar(Configuration conf, ConfVars var, int val) {
    conf.setInt(var.varname, val);
}

From source file:com.facebook.hiveio.common.HadoopUtils.java

License:Apache License

/**
 * Set number of map task attempts//from   ww  w . j av a 2s.  c o m
 * @param conf Configuration
 * @param numAttempts number of attempts
 */
public static void setMapAttempts(Configuration conf, int numAttempts) {
    conf.setInt("mapred.map.max.attempts", numAttempts);
}

From source file:com.facebook.hiveio.conf.IntConfOption.java

License:Apache License

/**
 * Set value/*from   w  ww.ja v a  2 s .com*/
 * @param conf Configuration
 * @param value to set
 */
public void set(Configuration conf, int value) {
    conf.setInt(getKey(), value);
}

From source file:com.facebook.hiveio.conf.IntConfOption.java

License:Apache License

/**
 * Set value if it's not already present
 * @param conf Configuration/*w  w  w  . ja va2  s  .  c o m*/
 * @param value to set
 */
public void setIfUnset(Configuration conf, int value) {
    if (conf.get(getKey()) == null) {
        conf.setInt(getKey(), value);
    }
}

From source file:com.facebook.presto.hive.HdfsConfiguration.java

License:Apache License

protected Configuration createConfiguration() {
    Configuration config = new Configuration();

    if (resourcePaths != null) {
        for (String resourcePath : resourcePaths) {
            config.addResource(new Path(resourcePath));
        }//from ww  w. j  av  a 2 s . c om
    }

    // this is to prevent dfs client from doing reverse DNS lookups to determine whether nodes are rack local
    config.setClass("topology.node.switch.mapping.impl", NoOpDNSToSwitchMapping.class,
            DNSToSwitchMapping.class);

    if (socksProxy != null) {
        config.setClass("hadoop.rpc.socket.factory.class.default", SocksSocketFactory.class,
                SocketFactory.class);
        config.set("hadoop.socks.server", socksProxy.toString());
    }

    if (domainSocketPath != null) {
        config.setStrings("dfs.domain.socket.path", domainSocketPath);
    }

    // only enable short circuit reads if domain socket path is properly configured
    if (!config.get("dfs.domain.socket.path", "").trim().isEmpty()) {
        config.setBooleanIfUnset("dfs.client.read.shortcircuit", true);
    }

    config.setInt("dfs.socket.timeout", Ints.checkedCast(dfsTimeout.toMillis()));
    config.setInt("ipc.ping.interval", Ints.checkedCast(dfsTimeout.toMillis()));
    config.setInt("ipc.client.connect.timeout", Ints.checkedCast(dfsConnectTimeout.toMillis()));
    config.setInt("ipc.client.connect.max.retries", dfsConnectMaxRetries);

    // re-map filesystem schemes to match Amazon Elastic MapReduce
    config.set("fs.s3.impl", PrestoS3FileSystem.class.getName());
    config.set("fs.s3n.impl", PrestoS3FileSystem.class.getName());
    config.set("fs.s3bfs.impl", "org.apache.hadoop.fs.s3.S3FileSystem");

    // set AWS credentials for S3
    for (String scheme : ImmutableList.of("s3", "s3bfs", "s3n")) {
        if (s3AwsAccessKey != null) {
            config.set(format("fs.%s.awsAccessKeyId", scheme), s3AwsAccessKey);
        }
        if (s3AwsSecretKey != null) {
            config.set(format("fs.%s.awsSecretAccessKey", scheme), s3AwsSecretKey);
        }
    }

    // set config for S3
    config.setBoolean(PrestoS3FileSystem.S3_SSL_ENABLED, s3SslEnabled);
    config.setInt(PrestoS3FileSystem.S3_MAX_CLIENT_RETRIES, s3MaxClientRetries);
    config.setInt(PrestoS3FileSystem.S3_MAX_ERROR_RETRIES, s3MaxErrorRetries);
    config.set(PrestoS3FileSystem.S3_CONNECT_TIMEOUT, s3ConnectTimeout.toString());
    config.set(PrestoS3FileSystem.S3_STAGING_DIRECTORY, s3StagingDirectory.toString());

    updateConfiguration(config);

    return config;
}

From source file:com.facebook.presto.hive.HdfsConfigurationUpdater.java

License:Apache License

public void updateConfiguration(Configuration config) {
    if (resourcePaths != null) {
        for (String resourcePath : resourcePaths) {
            config.addResource(new Path(resourcePath));
        }//from  w  w w.j a v a  2s. c  om
    }

    // this is to prevent dfs client from doing reverse DNS lookups to determine whether nodes are rack local
    config.setClass("topology.node.switch.mapping.impl", NoOpDNSToSwitchMapping.class,
            DNSToSwitchMapping.class);

    if (socksProxy != null) {
        config.setClass("hadoop.rpc.socket.factory.class.default", SocksSocketFactory.class,
                SocketFactory.class);
        config.set("hadoop.socks.server", socksProxy.toString());
    }

    if (domainSocketPath != null) {
        config.setStrings("dfs.domain.socket.path", domainSocketPath);
    }

    // only enable short circuit reads if domain socket path is properly configured
    if (!config.get("dfs.domain.socket.path", "").trim().isEmpty()) {
        config.setBooleanIfUnset("dfs.client.read.shortcircuit", true);
    }

    config.setInt("dfs.socket.timeout", toIntExact(dfsTimeout.toMillis()));
    config.setInt("ipc.ping.interval", toIntExact(ipcPingInterval.toMillis()));
    config.setInt("ipc.client.connect.timeout", toIntExact(dfsConnectTimeout.toMillis()));
    config.setInt("ipc.client.connect.max.retries", dfsConnectMaxRetries);

    // re-map filesystem schemes to match Amazon Elastic MapReduce
    config.set("fs.s3.impl", PrestoS3FileSystem.class.getName());
    config.set("fs.s3a.impl", PrestoS3FileSystem.class.getName());
    config.set("fs.s3n.impl", PrestoS3FileSystem.class.getName());
    config.set("fs.s3bfs.impl", "org.apache.hadoop.fs.s3.S3FileSystem");

    // set AWS credentials for S3
    if (s3AwsAccessKey != null) {
        config.set(PrestoS3FileSystem.S3_ACCESS_KEY, s3AwsAccessKey);
        config.set("fs.s3bfs.awsAccessKeyId", s3AwsAccessKey);
    }
    if (s3AwsSecretKey != null) {
        config.set(PrestoS3FileSystem.S3_SECRET_KEY, s3AwsSecretKey);
        config.set("fs.s3bfs.awsSecretAccessKey", s3AwsSecretKey);
    }
    if (s3Endpoint != null) {
        config.set(PrestoS3FileSystem.S3_ENDPOINT, s3Endpoint);
        config.set("fs.s3bfs.Endpoint", s3Endpoint);
    }
    if (s3SignerType != null) {
        config.set(PrestoS3FileSystem.S3_SIGNER_TYPE, s3SignerType.getSignerType());
    }

    config.setInt("fs.cache.max-size", fileSystemMaxCacheSize);

    configureCompression(config, compressionCodec);

    // set config for S3
    config.setBoolean(PrestoS3FileSystem.S3_USE_INSTANCE_CREDENTIALS, s3UseInstanceCredentials);
    config.setBoolean(PrestoS3FileSystem.S3_SSL_ENABLED, s3SslEnabled);
    config.setBoolean(PrestoS3FileSystem.S3_SSE_ENABLED, s3SseEnabled);
    if (s3EncryptionMaterialsProvider != null) {
        config.set(PrestoS3FileSystem.S3_ENCRYPTION_MATERIALS_PROVIDER, s3EncryptionMaterialsProvider);
    }
    if (s3KmsKeyId != null) {
        config.set(PrestoS3FileSystem.S3_KMS_KEY_ID, s3KmsKeyId);
    }
    config.setInt(PrestoS3FileSystem.S3_MAX_CLIENT_RETRIES, s3MaxClientRetries);
    config.setInt(PrestoS3FileSystem.S3_MAX_ERROR_RETRIES, s3MaxErrorRetries);
    config.set(PrestoS3FileSystem.S3_MAX_BACKOFF_TIME, s3MaxBackoffTime.toString());
    config.set(PrestoS3FileSystem.S3_MAX_RETRY_TIME, s3MaxRetryTime.toString());
    config.set(PrestoS3FileSystem.S3_CONNECT_TIMEOUT, s3ConnectTimeout.toString());
    config.set(PrestoS3FileSystem.S3_SOCKET_TIMEOUT, s3SocketTimeout.toString());
    config.set(PrestoS3FileSystem.S3_STAGING_DIRECTORY, s3StagingDirectory.toString());
    config.setInt(PrestoS3FileSystem.S3_MAX_CONNECTIONS, s3MaxConnections);
    config.setLong(PrestoS3FileSystem.S3_MULTIPART_MIN_FILE_SIZE, s3MultipartMinFileSize.toBytes());
    config.setLong(PrestoS3FileSystem.S3_MULTIPART_MIN_PART_SIZE, s3MultipartMinPartSize.toBytes());
    config.setBoolean(PrestoS3FileSystem.S3_PIN_CLIENT_TO_CURRENT_REGION, pinS3ClientToCurrentRegion);
    config.set(PrestoS3FileSystem.S3_USER_AGENT_PREFIX, s3UserAgentPrefix);
}

From source file:com.facebook.presto.hive.s3.PrestoS3ConfigurationUpdater.java

License:Apache License

@Override
public void updateConfiguration(Configuration config) {
    // re-map filesystem schemes to match Amazon Elastic MapReduce
    config.set("fs.s3.impl", PrestoS3FileSystem.class.getName());
    config.set("fs.s3a.impl", PrestoS3FileSystem.class.getName());
    config.set("fs.s3n.impl", PrestoS3FileSystem.class.getName());

    if (awsAccessKey != null) {
        config.set(PrestoS3FileSystem.S3_ACCESS_KEY, awsAccessKey);
    }/*  w  w w . ja v a  2s  . com*/
    if (awsSecretKey != null) {
        config.set(PrestoS3FileSystem.S3_SECRET_KEY, awsSecretKey);
    }
    if (endpoint != null) {
        config.set(PrestoS3FileSystem.S3_ENDPOINT, endpoint);
    }
    if (signerType != null) {
        config.set(PrestoS3FileSystem.S3_SIGNER_TYPE, signerType.name());
    }
    config.setBoolean(PrestoS3FileSystem.S3_PATH_STYLE_ACCESS, pathStyleAccess);
    config.setBoolean(PrestoS3FileSystem.S3_USE_INSTANCE_CREDENTIALS, useInstanceCredentials);
    config.setBoolean(PrestoS3FileSystem.S3_SSL_ENABLED, sslEnabled);
    config.setBoolean(PrestoS3FileSystem.S3_SSE_ENABLED, sseEnabled);
    config.set(PrestoS3FileSystem.S3_SSE_TYPE, sseType.name());
    if (encryptionMaterialsProvider != null) {
        config.set(PrestoS3FileSystem.S3_ENCRYPTION_MATERIALS_PROVIDER, encryptionMaterialsProvider);
    }
    if (kmsKeyId != null) {
        config.set(PrestoS3FileSystem.S3_KMS_KEY_ID, kmsKeyId);
    }
    if (sseKmsKeyId != null) {
        config.set(PrestoS3FileSystem.S3_SSE_KMS_KEY_ID, sseKmsKeyId);
    }
    config.setInt(PrestoS3FileSystem.S3_MAX_CLIENT_RETRIES, maxClientRetries);
    config.setInt(PrestoS3FileSystem.S3_MAX_ERROR_RETRIES, maxErrorRetries);
    config.set(PrestoS3FileSystem.S3_MAX_BACKOFF_TIME, maxBackoffTime.toString());
    config.set(PrestoS3FileSystem.S3_MAX_RETRY_TIME, maxRetryTime.toString());
    config.set(PrestoS3FileSystem.S3_CONNECT_TIMEOUT, connectTimeout.toString());
    config.set(PrestoS3FileSystem.S3_SOCKET_TIMEOUT, socketTimeout.toString());
    config.set(PrestoS3FileSystem.S3_STAGING_DIRECTORY, stagingDirectory.toString());
    config.setInt(PrestoS3FileSystem.S3_MAX_CONNECTIONS, maxConnections);
    config.setLong(PrestoS3FileSystem.S3_MULTIPART_MIN_FILE_SIZE, multipartMinFileSize.toBytes());
    config.setLong(PrestoS3FileSystem.S3_MULTIPART_MIN_PART_SIZE, multipartMinPartSize.toBytes());
    config.setBoolean(PrestoS3FileSystem.S3_PIN_CLIENT_TO_CURRENT_REGION, pinClientToCurrentRegion);
    config.set(PrestoS3FileSystem.S3_USER_AGENT_PREFIX, userAgentPrefix);
}

From source file:com.facebook.presto.hive.s3.TestPrestoS3FileSystem.java

License:Apache License

@SuppressWarnings({ "ResultOfMethodCallIgnored", "OverlyStrongTypeCast", "ConstantConditions" })
@Test//  w ww .  j  av  a 2  s.c  o m
public void testReadRetryCounters() throws Exception {
    try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) {
        int maxRetries = 2;
        MockAmazonS3 s3 = new MockAmazonS3();
        s3.setGetObjectHttpErrorCode(SC_INTERNAL_SERVER_ERROR);
        Configuration configuration = new Configuration();
        configuration.set(S3_MAX_BACKOFF_TIME, "1ms");
        configuration.set(S3_MAX_RETRY_TIME, "5s");
        configuration.setInt(S3_MAX_CLIENT_RETRIES, maxRetries);
        fs.initialize(new URI("s3n://test-bucket/"), configuration);
        fs.setS3Client(s3);
        try (FSDataInputStream inputStream = fs.open(new Path("s3n://test-bucket/test"))) {
            inputStream.read();
        } catch (Throwable expected) {
            assertInstanceOf(expected, AmazonS3Exception.class);
            assertEquals(((AmazonS3Exception) expected).getStatusCode(), SC_INTERNAL_SERVER_ERROR);
            assertEquals(PrestoS3FileSystem.getFileSystemStats().getReadRetries().getTotalCount(), maxRetries);
            assertEquals(PrestoS3FileSystem.getFileSystemStats().getGetObjectRetries().getTotalCount(),
                    (maxRetries + 1L) * maxRetries);
        }
    }
}

From source file:com.facebook.presto.hive.s3.TestPrestoS3FileSystem.java

License:Apache License

@SuppressWarnings({ "OverlyStrongTypeCast", "ConstantConditions" })
@Test// ww w.  jav  a  2s.  c o  m
public void testGetMetadataRetryCounter() {
    int maxRetries = 2;
    try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) {
        MockAmazonS3 s3 = new MockAmazonS3();
        s3.setGetObjectMetadataHttpCode(SC_INTERNAL_SERVER_ERROR);
        Configuration configuration = new Configuration();
        configuration.set(S3_MAX_BACKOFF_TIME, "1ms");
        configuration.set(S3_MAX_RETRY_TIME, "5s");
        configuration.setInt(S3_MAX_CLIENT_RETRIES, maxRetries);
        fs.initialize(new URI("s3n://test-bucket/"), configuration);
        fs.setS3Client(s3);
        fs.getS3ObjectMetadata(new Path("s3n://test-bucket/test"));
    } catch (Throwable expected) {
        assertInstanceOf(expected, AmazonS3Exception.class);
        assertEquals(((AmazonS3Exception) expected).getStatusCode(), SC_INTERNAL_SERVER_ERROR);
        assertEquals(PrestoS3FileSystem.getFileSystemStats().getGetMetadataRetries().getTotalCount(),
                maxRetries);
    }
}