Example usage for org.apache.hadoop.conf Configuration getTrimmed

List of usage examples for org.apache.hadoop.conf Configuration getTrimmed

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.conf.*
  5. Configuration
  6. getTrimmed

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration getTrimmed.

Prototype

public String getTrimmed(String name)

Source Link

Document

Get the value of the name property as a trimmed String, null if no such property exists.

Usage

From source file:com.aliyun.fs.oss.blk.JetOssFileSystemStore.java

License:Apache License

public void initialize(URI uri, Configuration conf) throws IOException {
    if (uri.getHost() == null) {
        throw new IllegalArgumentException("Invalid hostname in URI " + uri);
    }/*from   w w w.  j a v a2 s.  c o  m*/
    this.conf = conf;
    String userInfo = uri.getUserInfo();
    if (userInfo != null) {
        String[] ossCredentials = userInfo.split(":");
        if (ossCredentials.length >= 2) {
            accessKeyId = ossCredentials[0];
            accessKeySecret = ossCredentials[1];
        }
        if (ossCredentials.length == 3) {
            securityToken = ossCredentials[2];
        }
    }

    String host = uri.getHost();
    if (!StringUtils.isEmpty(host) && !host.contains(".")) {
        bucket = host;
    } else if (!StringUtils.isEmpty(host)) {
        bucket = host.substring(0, host.indexOf("."));
        endpoint = host.substring(host.indexOf(".") + 1);
    }

    if (accessKeyId == null) {
        accessKeyId = conf.getTrimmed("fs.oss.accessKeyId");
    }
    if (accessKeySecret == null) {
        accessKeySecret = conf.getTrimmed("fs.oss.accessKeySecret");
    }
    if (securityToken == null) {
        securityToken = conf.getTrimmed("fs.oss.securityToken");
    }
    if (endpoint == null) {
        endpoint = conf.getTrimmed("fs.oss.endpoint");
    }
    ClientConfiguration cc = initializeOSSClientConfig(conf);
    if (securityToken == null) {
        this.ossClient = new OSSClient(endpoint, accessKeyId, accessKeySecret, cc);
    } else {
        this.ossClient = new OSSClient(endpoint, accessKeyId, accessKeySecret, securityToken, cc);
    }
    this.bufferSize = conf.getInt("io.file.buffer.size", 4096);
}

From source file:com.aliyun.fs.oss.nat.JetOssNativeFileSystemStore.java

License:Apache License

public void initialize(URI uri, Configuration conf) throws Exception {
    if (uri.getHost() == null) {
        throw new IllegalArgumentException("Invalid hostname in URI " + uri);
    }/*  w ww .jav a 2s.com*/

    String userInfo = uri.getUserInfo();
    if (userInfo != null) {
        throw new IllegalArgumentException("Disallow set ak information in OSS URI.");
    }

    this.conf = conf;
    String host = uri.getHost();
    if (!StringUtils.isEmpty(host) && !host.contains(".")) {
        bucket = host;
    } else if (!StringUtils.isEmpty(host)) {
        bucket = host.substring(0, host.indexOf("."));
        endpoint = host.substring(host.indexOf(".") + 1);
    }

    // try to get accessKeyId, accessJeySecret, securityToken, endpoint from configuration.
    if (accessKeyId == null) {
        accessKeyId = conf.getTrimmed("fs.oss.accessKeyId");
    }
    if (accessKeySecret == null) {
        accessKeySecret = conf.getTrimmed("fs.oss.accessKeySecret");
    }
    if (securityToken == null) {
        securityToken = conf.getTrimmed("fs.oss.securityToken");
    }

    if (endpoint == null) {
        endpoint = conf.getTrimmed("fs.oss.endpoint");
    }

    // try to get accessKeyId, accessJeySecret, securityToken, endpoint from MetaService.
    LOG.debug("Try to get accessKeyId, accessJeySecret, securityToken endpoint from MetaService.");
    if (accessKeyId == null || accessKeySecret == null) {
        accessKeyId = MetaClient.getRoleAccessKeyId();
        accessKeySecret = MetaClient.getRoleAccessKeySecret();
        securityToken = MetaClient.getRoleSecurityToken();

        if (StringUtils.isEmpty(accessKeyId) || StringUtils.isEmpty(accessKeySecret)
                || StringUtils.isEmpty(securityToken)) {
            throw new IllegalArgumentException(
                    "AccessKeyId/AccessKeySecret/SecurityToken is not available, you "
                            + "can set them in configuration.");
        }
    }

    if (endpoint == null) {
        endpoint = EndpointEnum.getEndpoint("oss", MetaClient.getClusterRegionName(),
                MetaClient.getClusterNetworkType());
        if (endpoint == null) {
            throw new IllegalArgumentException(
                    "Can not find any suitable " + "endpoint, you can set it in OSS URI");
        }
    }

    if (securityToken == null) {
        this.ossClient = new OSSClientAgent(endpoint, accessKeyId, accessKeySecret, conf);
    } else {
        this.ossClient = new OSSClientAgent(endpoint, accessKeyId, accessKeySecret, securityToken, conf);
    }
    this.numCopyThreads = conf.getInt("fs.oss.uploadPartCopy.thread.number", 10);
    this.numPutThreads = conf.getInt("fs.oss.uploadPart.thread.number", 5);
    this.maxSplitSize = conf.getInt("fs.oss.multipart.split.max.byte", 5 * 1024 * 1024);
    this.numSplits = conf.getInt("fs.oss.multipart.split.number", 10);
    this.maxSimpleCopySize = conf.getLong("fs.oss.copy.simple.max.byte", 64 * 1024 * 1024L);
    this.maxSimplePutSize = conf.getLong("fs.oss.put.simple.max.byte", 5 * 1024 * 1024);
}

From source file:com.cloudera.lib.service.hadoop.HadoopService.java

License:Open Source License

@Override
public <T> T execute(String user, final Configuration conf, final FileSystemExecutor<T> executor)
        throws HadoopException {
    Check.notEmpty(user, "user");
    Check.notNull(conf, "conf");
    Check.notNull(executor, "executor");
    if (conf.get(NAME_NODE_PROPERTY) == null || conf.getTrimmed(NAME_NODE_PROPERTY).length() == 0) {
        throw new HadoopException(HadoopException.ERROR.H06, NAME_NODE_PROPERTY);
    }/*from w  ww.  jav a 2  s .com*/
    try {
        validateNamenode(new URI(conf.get(NAME_NODE_PROPERTY)).getAuthority());
        UserGroupInformation ugi = getUGI(user);
        return ugi.doAs(new PrivilegedExceptionAction<T>() {
            public T run() throws Exception {
                Configuration namenodeConf = createNameNodeConf(conf);
                FileSystem fs = createFileSystem(namenodeConf);
                Instrumentation instrumentation = getServer().get(Instrumentation.class);
                Instrumentation.Cron cron = instrumentation.createCron();
                try {
                    checkNameNodeHealth(fs);
                    cron.start();
                    return executor.execute(fs);
                } finally {
                    cron.stop();
                    instrumentation.addCron(INSTRUMENTATION_GROUP, executor.getClass().getSimpleName(), cron);
                    closeFileSystem(fs);
                }
            }
        });
    } catch (HadoopException ex) {
        throw ex;
    } catch (Exception ex) {
        throw new HadoopException(HadoopException.ERROR.H03, ex);
    }
}

From source file:com.cloudera.lib.service.hadoop.HadoopService.java

License:Open Source License

@Override
public <T> T execute(String user, final Configuration conf, final JobClientExecutor<T> executor)
        throws HadoopException {
    Check.notEmpty(user, "user");
    Check.notNull(conf, "conf");
    Check.notNull(executor, "executor");
    if (conf.get(JOB_TRACKER_PROPERTY) == null || conf.getTrimmed(JOB_TRACKER_PROPERTY).length() == 0) {
        throw new HadoopException(HadoopException.ERROR.H06, JOB_TRACKER_PROPERTY);
    }/*ww w  . j a v a 2s  .  co  m*/
    if (conf.get(NAME_NODE_PROPERTY) == null || conf.getTrimmed(NAME_NODE_PROPERTY).length() == 0) {
        throw new HadoopException(HadoopException.ERROR.H06, NAME_NODE_PROPERTY);
    }
    try {
        validateJobtracker(new URI(conf.get(JOB_TRACKER_PROPERTY)).getAuthority());
        validateNamenode(new URI(conf.get(NAME_NODE_PROPERTY)).getAuthority());
        UserGroupInformation ugi = getUGI(user);
        return ugi.doAs(new PrivilegedExceptionAction<T>() {
            public T run() throws Exception {
                JobConf jobtrackerConf = createJobTrackerConf(conf);
                Configuration namenodeConf = createNameNodeConf(conf);
                JobClient jobClient = createJobClient(jobtrackerConf);
                try {
                    checkJobTrackerHealth(jobClient);
                    FileSystem fs = createFileSystem(namenodeConf);
                    Instrumentation instrumentation = getServer().get(Instrumentation.class);
                    Instrumentation.Cron cron = instrumentation.createCron();
                    try {
                        checkNameNodeHealth(fs);
                        cron.start();
                        return executor.execute(jobClient, fs);
                    } finally {
                        cron.stop();
                        instrumentation.addCron(INSTRUMENTATION_GROUP, executor.getClass().getSimpleName(),
                                cron);
                        closeFileSystem(fs);
                    }
                } finally {
                    closeJobClient(jobClient);
                }
            }
        });
    } catch (HadoopException ex) {
        throw ex;
    } catch (Exception ex) {
        throw new HadoopException(HadoopException.ERROR.H04, ex);
    }
}

From source file:org.apache.mahout.classifier.df.mapreduce.OversamplingBuilder.java

License:Apache License

public static String getNegClass(Configuration conf) {
    //return conf.getInt("mahout.rf.negclass", -1);    
    return conf.getTrimmed("mahout.rf.negclass");
}

From source file:org.apache.mahout.classifier.df.mapreduce.UndersamplingBuilder.java

License:Apache License

public static String getPosClass(Configuration conf) {
    return conf.getTrimmed("mahout.rf.posclass");
}

From source file:org.apache.oozie.action.hadoop.GitMain.java

License:Apache License

/**
 * Calls helper function to verify name not null and throw an exception if so.
 * Otherwise, returns actionConf value// w w  w .  j ava  2s .  co m
 * @param name - actionConf value to return
 */
String checkAndGetTrimmed(final Configuration actionConf, String name) {
    Objects.requireNonNull(actionConf.getTrimmed(name),
            () -> String.format("Action Configuration does not have [%s] property", name));
    return actionConf.getTrimmed(name);
}

From source file:org.apache.tez.dag.api.HistoryLogLevel.java

License:Apache License

public static HistoryLogLevel getLogLevel(Configuration conf, HistoryLogLevel defaultValue) {
    String logLevel = conf.getTrimmed(TezConfiguration.TEZ_HISTORY_LOGGING_LOGLEVEL);
    if (logLevel == null) {
        return defaultValue;
    }/*  w  w w .  j  a  va 2  s.c o  m*/
    return valueOf(logLevel.toUpperCase(Locale.ENGLISH));
}

From source file:org.apache.tez.dag.utils.TaskSpecificLaunchCmdOption.java

License:Apache License

public TaskSpecificLaunchCmdOption(Configuration conf) {
    this.tsLaunchCmdOpts = conf.getTrimmed(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS);
    this.tsLogParams = TezClientUtils
            .parseLogParams(conf.getTrimmed(TezConfiguration.TEZ_TASK_SPECIFIC_LOG_LEVEL));

    if (shouldParseSpecificTaskList()) {
        this.tasksMap = getSpecificTasks(conf);
    } else {//from  w w  w.  j a va 2 s  .c om
        this.tasksMap = null;
    }
}

From source file:org.apache.tez.dag.utils.TaskSpecificLaunchCmdOption.java

License:Apache License

/**
 * <pre>// ww w  .j a  v  a2s.  c  om
 * Get the set of tasks in the job, which needs task specific launch command arguments to be
 * added. Example formats are
 * v[0,1,2] - Add additional options to subset of tasks in a vertex
 * v[1,2,3];v2[5,6,7] - Add additional options in tasks in multiple vertices
 * v[1:5,20,30];v2[2:5,60,7] - To support range of tasks in vertices. Partial
 * ranges are not supported (e.g v[:5],v2[2:]).
 * v[] - For all tasks in a vertex
 * </pre>
 *
 * @param conf
 * @return a map from the vertex name to a BitSet representing tasks to be instruemented. null if
 *         the provided configuration is empty or invalid
 */
private Map<String, BitSet> getSpecificTasks(Configuration conf) {
    String specificTaskList = conf.getTrimmed(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS_LIST);
    if (!Strings.isNullOrEmpty(specificTaskList) && isValid(specificTaskList)) {
        final Map<String, BitSet> resultSet = new HashMap<String, BitSet>();
        Matcher matcher = TASKS_REGEX.matcher(specificTaskList);
        while (matcher.find()) {
            String vertexName = matcher.group(1).trim();
            BitSet taskSet = parseTasks(matcher.group(2).trim());
            resultSet.put(vertexName, taskSet);
        }
        LOG.info("Specific tasks with additional launch-cmd options=" + resultSet);
        return resultSet;
    } else {
        return null;
    }
}