Example usage for org.apache.hadoop.conf Configuration getLong

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration getLong.

Prototype

public long getLong(String name, long defaultValue) 

Source Link

Document

Get the value of the name property as a long.

Usage

From source file:com.alibaba.wasp.master.GeneralBulkAssigner.java

License:Apache License

@Override
protected long getTimeoutOnRIT() {
    // Guess timeout. Multiply the max number of entityGroups on a server
    // by how long we think one entityGroup takes opening.
    Configuration conf = server.getConfiguration();
    long perEntityGroupOpenTimeGuesstimate = conf.getLong("wasp.bulk.assignment.perentityGroup.open.time",
            1000);/* www.j a v  a2s . co  m*/
    int maxEntityGroupsPerServer = 1;
    for (List<EntityGroupInfo> entityGroupList : bulkPlan.values()) {
        int size = entityGroupList.size();
        if (size > maxEntityGroupsPerServer) {
            maxEntityGroupsPerServer = size;
        }
    }
    long timeout = perEntityGroupOpenTimeGuesstimate * maxEntityGroupsPerServer
            + conf.getLong("wasp.fserver.rpc.startup.waittime", 60000)
            + conf.getLong("wasp.bulk.assignment.perfserver.rpc.waittime", 30000) * bulkPlan.size();
    LOG.debug("Timeout-on-RIT=" + timeout);
    return timeout;
}

---

From source file:com.alibaba.wasp.meta.FMetaStore.java

License:Apache License

public FMetaStore(Configuration conf) throws MetaException {
    this.setConf(conf);
    fMetaServices = new FMetaServicesImplWithoutRetry(conf);
    maxRetries = conf.getInt(FConstants.WASP_FMETA_RETRIES_NUMBER,
            FConstants.DEFAULT_WASP_FMETA_RETRIES_NUMBER);
    pause = conf.getLong(HConstants.HBASE_CLIENT_PAUSE, HConstants.DEFAULT_HBASE_CLIENT_PAUSE);
}

---

From source file:com.aliyun.fs.oss.blk.JetOssFileSystemStore.java

License:Apache License

private ClientConfiguration initializeOSSClientConfig(Configuration conf) {
    ClientConfiguration cc = new ClientConfiguration();
    cc.setConnectionTimeout(//  w w w . j a va 2  s . co m
            conf.getInt("fs.oss.client.connection.timeout", ClientConfiguration.DEFAULT_CONNECTION_TIMEOUT));
    cc.setSocketTimeout(
            conf.getInt("fs.oss.client.socket.timeout", ClientConfiguration.DEFAULT_SOCKET_TIMEOUT));
    cc.setConnectionTTL(
            conf.getLong("fs.oss.client.connection.ttl", ClientConfiguration.DEFAULT_CONNECTION_TTL));
    cc.setMaxConnections(conf.getInt("fs.oss.connection.max", ClientConfiguration.DEFAULT_MAX_CONNECTIONS));

    return cc;
}

---

From source file:com.aliyun.fs.oss.blk.OssFileSystem.java

License:Apache License

private static FileSystemStore createDefaultStore(Configuration conf) {
    FileSystemStore store = new JetOssFileSystemStore();

    RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
            conf.getInt("fs.oss.maxRetries", 4), conf.getLong("fs.oss.sleepTimeSeconds", 10), TimeUnit.SECONDS);
    Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>();
    exceptionToPolicyMap.put(IOException.class, basePolicy);
    exceptionToPolicyMap.put(OssException.class, basePolicy);

    RetryPolicy methodPolicy = RetryPolicies.retryByException(RetryPolicies.TRY_ONCE_THEN_FAIL,
            exceptionToPolicyMap);//from   w  w  w .  ja  v  a  2  s  .c  o m
    Map<String, RetryPolicy> methodNameToPolicyMap = new HashMap<String, RetryPolicy>();
    methodNameToPolicyMap.put("storeBlock", methodPolicy);
    methodNameToPolicyMap.put("retrieveBlock", methodPolicy);

    return (FileSystemStore) RetryProxy.create(FileSystemStore.class, store, methodNameToPolicyMap);
}

---

From source file:com.aliyun.fs.oss.nat.JetOssNativeFileSystemStore.java

License:Apache License

public void initialize(URI uri, Configuration conf) throws Exception {
    if (uri.getHost() == null) {
        throw new IllegalArgumentException("Invalid hostname in URI " + uri);
    }/*w  w  w.  ja  v a2 s . com*/

    String userInfo = uri.getUserInfo();
    if (userInfo != null) {
        throw new IllegalArgumentException("Disallow set ak information in OSS URI.");
    }

    this.conf = conf;
    String host = uri.getHost();
    if (!StringUtils.isEmpty(host) && !host.contains(".")) {
        bucket = host;
    } else if (!StringUtils.isEmpty(host)) {
        bucket = host.substring(0, host.indexOf("."));
        endpoint = host.substring(host.indexOf(".") + 1);
    }

    // try to get accessKeyId, accessJeySecret, securityToken, endpoint from configuration.
    if (accessKeyId == null) {
        accessKeyId = conf.getTrimmed("fs.oss.accessKeyId");
    }
    if (accessKeySecret == null) {
        accessKeySecret = conf.getTrimmed("fs.oss.accessKeySecret");
    }
    if (securityToken == null) {
        securityToken = conf.getTrimmed("fs.oss.securityToken");
    }

    if (endpoint == null) {
        endpoint = conf.getTrimmed("fs.oss.endpoint");
    }

    // try to get accessKeyId, accessJeySecret, securityToken, endpoint from MetaService.
    LOG.debug("Try to get accessKeyId, accessJeySecret, securityToken endpoint from MetaService.");
    if (accessKeyId == null || accessKeySecret == null) {
        accessKeyId = MetaClient.getRoleAccessKeyId();
        accessKeySecret = MetaClient.getRoleAccessKeySecret();
        securityToken = MetaClient.getRoleSecurityToken();

        if (StringUtils.isEmpty(accessKeyId) || StringUtils.isEmpty(accessKeySecret)
                || StringUtils.isEmpty(securityToken)) {
            throw new IllegalArgumentException(
                    "AccessKeyId/AccessKeySecret/SecurityToken is not available, you "
                            + "can set them in configuration.");
        }
    }

    if (endpoint == null) {
        endpoint = EndpointEnum.getEndpoint("oss", MetaClient.getClusterRegionName(),
                MetaClient.getClusterNetworkType());
        if (endpoint == null) {
            throw new IllegalArgumentException(
                    "Can not find any suitable " + "endpoint, you can set it in OSS URI");
        }
    }

    if (securityToken == null) {
        this.ossClient = new OSSClientAgent(endpoint, accessKeyId, accessKeySecret, conf);
    } else {
        this.ossClient = new OSSClientAgent(endpoint, accessKeyId, accessKeySecret, securityToken, conf);
    }
    this.numCopyThreads = conf.getInt("fs.oss.uploadPartCopy.thread.number", 10);
    this.numPutThreads = conf.getInt("fs.oss.uploadPart.thread.number", 5);
    this.maxSplitSize = conf.getInt("fs.oss.multipart.split.max.byte", 5 * 1024 * 1024);
    this.numSplits = conf.getInt("fs.oss.multipart.split.number", 10);
    this.maxSimpleCopySize = conf.getLong("fs.oss.copy.simple.max.byte", 64 * 1024 * 1024L);
    this.maxSimplePutSize = conf.getLong("fs.oss.put.simple.max.byte", 5 * 1024 * 1024);
}

---

From source file:com.aliyun.fs.oss.nat.NativeOssFileSystem.java

License:Apache License

private static NativeFileSystemStore createDefaultStore(Configuration conf) {
    NativeFileSystemStore store = new JetOssNativeFileSystemStore();

    RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
            conf.getInt("fs.oss.maxRetries", 4), conf.getLong("fs.oss.sleepTimeSeconds", 10), TimeUnit.SECONDS);
    Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>();
    // for reflection invoke.
    exceptionToPolicyMap.put(InvocationTargetException.class, basePolicy);
    exceptionToPolicyMap.put(IOException.class, basePolicy);
    exceptionToPolicyMap.put(OssException.class, basePolicy);

    RetryPolicy methodPolicy = RetryPolicies.retryByException(RetryPolicies.TRY_ONCE_THEN_FAIL,
            exceptionToPolicyMap);//from  www  .ja  v a 2  s. c o m
    Map<String, RetryPolicy> methodNameToPolicyMap = new HashMap<String, RetryPolicy>();
    methodNameToPolicyMap.put("storeFile", methodPolicy);
    methodNameToPolicyMap.put("storeFiles", methodPolicy);
    methodNameToPolicyMap.put("storeEmptyFile", methodPolicy);
    methodNameToPolicyMap.put("retrieveMetadata", methodPolicy);
    methodNameToPolicyMap.put("retrieve", methodPolicy);
    methodNameToPolicyMap.put("purge", methodPolicy);
    methodNameToPolicyMap.put("dump", methodPolicy);
    methodNameToPolicyMap.put("doesObjectExist", methodPolicy);
    methodNameToPolicyMap.put("copy", methodPolicy);
    methodNameToPolicyMap.put("list", methodPolicy);
    methodNameToPolicyMap.put("delete", methodPolicy);

    return (NativeFileSystemStore) RetryProxy.create(NativeFileSystemStore.class, store, methodNameToPolicyMap);
}

---

From source file:com.aliyun.fs.oss.utils.OSSClientAgent.java

License:Apache License

@SuppressWarnings("unchecked")
private Object initializeOSSClientConfig(Configuration conf, Class ClientConfigurationClz)
        throws IOException, ServiceException, ClientException {
    try {//from   w  w w . j  av a 2 s .  c  o  m
        Constructor cons = ClientConfigurationClz.getConstructor();
        Object clientConfiguration = cons.newInstance();
        Method method0 = ClientConfigurationClz.getMethod("setConnectionTimeout", Integer.TYPE);
        method0.invoke(clientConfiguration, conf.getInt("fs.oss.client.connection.timeout",
                ClientConfiguration.DEFAULT_CONNECTION_TIMEOUT));
        Method method1 = ClientConfigurationClz.getMethod("setSocketTimeout", Integer.TYPE);
        method1.invoke(clientConfiguration,
                conf.getInt("fs.oss.client.socket.timeout", ClientConfiguration.DEFAULT_SOCKET_TIMEOUT));
        Method method2 = ClientConfigurationClz.getMethod("setConnectionTTL", Long.TYPE);
        method2.invoke(clientConfiguration,
                conf.getLong("fs.oss.client.connection.ttl", ClientConfiguration.DEFAULT_CONNECTION_TTL));
        Method method3 = ClientConfigurationClz.getMethod("setMaxConnections", Integer.TYPE);
        method3.invoke(clientConfiguration,
                conf.getInt("fs.oss.connection.max", ClientConfiguration.DEFAULT_MAX_CONNECTIONS));

        return clientConfiguration;
    } catch (Exception e) {
        handleException(e);
        return null;
    }
}

---

From source file:com.aliyun.odps.volume.VolumeFSInputStream.java

License:Apache License

public VolumeFSInputStream(String path, VolumeFSClient volumeClient, Long fileLength, Configuration conf)
        throws IOException {
    this.path = path;
    this.volumeFSClient = volumeClient;
    this.seekOptimization = conf.getBoolean(VolumeFileSystemConfigKeys.ODPS_VOLUME_SEEK_OPTIMIZATION_ENABLED,
            false);//from   ww  w . jav a2 s . c  o  m
    if (this.seekOptimization) {
        this.blockSize = conf.getLong(VolumeFileSystemConfigKeys.ODPS_VOLUME_BLOCK_SIZE,
                VolumeFSConstants.DEFAULT_VOLUME_BLOCK_SIZE);
    }
    this.fileLength = fileLength;
    this.closed = false;
    this.uuid = UUID.randomUUID().toString();
    buffer_block_dir = new File(conf.get(VolumeFileSystemConfigKeys.ODPS_VOLUME_BLOCK_BUFFER_DIR,
            VolumeFSConstants.DEFAULT_VOLUME_BLOCK_BUFFER_DIR));
    if (!buffer_block_dir.exists() && !buffer_block_dir.mkdirs()) {
        throw new IOException("Cannot create Volume block buffer directory: " + buffer_block_dir);
    }
    if (seekOptimization) {
        executorService = Executors.newFixedThreadPool(1);
    }
}

---

From source file:com.asakusafw.runtime.mapreduce.simple.SimpleJobRunner.java

License:Apache License

private KeyValueSorter.Options getSorterOptions(Configuration configuration) {
    long bufferSize = configuration.getLong(KEY_BUFFER_SIZE, -1);
    if (bufferSize < 0) {
        bufferSize = DEFAULT_BUFFER_SIZE;
    } else {//from   w w w.jav  a  2 s  .co m
        bufferSize = Math.max(MIN_BUFFER_SIZE, Math.min(MAX_BUFFER_SIZE, bufferSize));
    }
    File temporaryDirectory = null;
    String tempdirString = configuration.get(KEY_TEMPORARY_LOCATION);
    if (tempdirString != null) {
        temporaryDirectory = new File(tempdirString);
        if (temporaryDirectory.mkdirs() == false && temporaryDirectory.isDirectory() == false) {
            LOG.warn(MessageFormat.format("failed to prepare shuffle temporary directory: {0}={1}",
                    KEY_TEMPORARY_LOCATION, temporaryDirectory));
        }
    }
    boolean compress = configuration.getBoolean(KEY_COMPRESS_BLOCK, DEFAULT_COMPRESS_BLOCK);
    KeyValueSorter.Options options = new KeyValueSorter.Options().withBufferSize((int) bufferSize)
            .withTemporaryDirectory(temporaryDirectory).withCompressBlock(compress);
    return options;
}

---

From source file:com.asakusafw.runtime.stage.configurator.AutoLocalStageConfigurator.java

License:Apache License

@Override
public void configure(Job job) throws IOException, InterruptedException {
    if (isLocal(job)) {
        return;//w w  w  .j  a v  a  2  s.c  o m
    }
    Configuration conf = job.getConfiguration();
    long limit = conf.getLong(KEY_LIMIT, -1L);
    if (limit < 0L) {
        if (LOG.isDebugEnabled()) {
            LOG.debug(MessageFormat.format("Auto stage localize is disabled: {0}", job.getJobName()));
        }
        return;
    }
    if (isProtected(job)) {
        return;
    }
    long estimated = StageInputDriver.estimateInputSize(job);
    if (LOG.isDebugEnabled()) {
        LOG.debug(MessageFormat.format("Auto stage localize: job={0}, limit={1}, estimated={2}",
                job.getJobName(), limit, estimated));
    }
    if (estimated < 0L || estimated > limit) {
        return;
    }

    LOG.info(MessageFormat.format("The job \"{0}\" will run in local mode", job.getJobName()));

    localize(job);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• »