Example usage for org.apache.hadoop.conf Configuration getTrimmed

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration getTrimmed.

Prototype

public String getTrimmed(String name, String defaultValue) 

Source Link

Document

Get the value of the name property as a trimmed String, defaultValue if no such property exists.

Usage

From source file:com.aliyun.fs.oss.AliyunOSSTestUtils.java

License:Apache License

/**
 * Create the test filesystem./* w ww .jav  a  2 s. c  o  m*/
 *
 * If the test.fs.oss.name property is not set,
 * tests will fail.
 *
 * @param conf configuration
 * @return the FS
 * @throws IOException
 */
public static NativeOssFileSystem createTestFileSystem(Configuration conf) throws IOException {
    String fsname = conf.getTrimmed(TestAliyunOSSFileSystemContract.TEST_FS_OSS_NAME, "");

    boolean liveTest = StringUtils.isNotEmpty(fsname);
    URI testURI = null;
    if (liveTest) {
        testURI = URI.create(fsname);
        liveTest = testURI.getScheme().equals("oss");
    }

    if (!liveTest) {
        throw new AssumptionViolatedException(
                "No test filesystem in " + TestAliyunOSSFileSystemContract.TEST_FS_OSS_NAME);
    }
    conf.set("mapreduce.job.run-local", "true");
    conf.set("fs.oss.buffer.dirs", "/tmp");
    NativeOssFileSystem ossfs = new NativeOssFileSystem();
    ossfs.initialize(testURI, conf);
    return ossfs;
}

---

From source file:com.cloudera.impala.service.JniFrontend.java

License:Apache License

/**
 * Return an empty string if short circuit read is properly enabled. If not, return an
 * error string describing the issues./*from  w w  w  .j  a v  a  2  s.c o m*/
 */
private String checkShortCircuitRead(Configuration conf) {
    StringBuilder output = new StringBuilder();
    String errorMessage = "ERROR: short-circuit local reads is disabled because\n";
    String prefix = "  - ";
    StringBuilder errorCause = new StringBuilder();

    // dfs.domain.socket.path must be set properly
    String domainSocketPath = conf.getTrimmed(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY,
            DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_DEFAULT);
    if (domainSocketPath.isEmpty()) {
        errorCause.append(prefix);
        errorCause.append(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY);
        errorCause.append(" is not configured.\n");
    } else {
        // The socket path parent directory must be readable and executable.
        File socketFile = new File(domainSocketPath);
        File socketDir = socketFile.getParentFile();
        if (socketDir == null || !socketDir.canRead() || !socketDir.canExecute()) {
            errorCause.append(prefix);
            errorCause.append("Impala cannot read or execute the parent directory of ");
            errorCause.append(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY);
            errorCause.append("\n");
        }
    }

    // dfs.client.read.shortcircuit must be set to true.
    if (!conf.getBoolean(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY,
            DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_DEFAULT)) {
        errorCause.append(prefix);
        errorCause.append(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY);
        errorCause.append(" is not enabled.\n");
    }

    // dfs.client.use.legacy.blockreader.local must be set to false
    if (conf.getBoolean(DFSConfigKeys.DFS_CLIENT_USE_LEGACY_BLOCKREADERLOCAL,
            DFSConfigKeys.DFS_CLIENT_USE_LEGACY_BLOCKREADERLOCAL_DEFAULT)) {
        errorCause.append(prefix);
        errorCause.append(DFSConfigKeys.DFS_CLIENT_USE_LEGACY_BLOCKREADERLOCAL);
        errorCause.append(" should not be enabled.\n");
    }

    if (errorCause.length() > 0) {
        output.append(errorMessage);
        output.append(errorCause);
    }

    return output.toString();
}

---

From source file:org.apache.impala.service.JniFrontend.java

License:Apache License

/**
 * Returns an error message if short circuit reads are enabled but misconfigured.
 * Otherwise, returns an empty string,/*from w w  w. j  a v a  2  s .  c o m*/
 */
private String checkShortCircuitRead(Configuration conf) {
    if (!conf.getBoolean(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY,
            DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_DEFAULT)) {
        LOG.info("Short-circuit reads are not enabled.");
        return "";
    }

    StringBuilder output = new StringBuilder();
    String errorMessage = "Invalid short-circuit reads configuration:\n";
    String prefix = "  - ";
    StringBuilder errorCause = new StringBuilder();

    // dfs.domain.socket.path must be set properly
    String domainSocketPath = conf.getTrimmed(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY,
            DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_DEFAULT);
    if (domainSocketPath.isEmpty()) {
        errorCause.append(prefix);
        errorCause.append(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY);
        errorCause.append(" is not configured.\n");
    } else {
        // The socket path parent directory must be readable and executable.
        File socketFile = new File(domainSocketPath);
        File socketDir = socketFile.getParentFile();
        if (socketDir == null || !socketDir.canRead() || !socketDir.canExecute()) {
            errorCause.append(prefix);
            errorCause.append("Impala cannot read or execute the parent directory of ");
            errorCause.append(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY);
            errorCause.append("\n");
        }
    }

    // dfs.client.use.legacy.blockreader.local must be set to false
    if (conf.getBoolean(DFSConfigKeys.DFS_CLIENT_USE_LEGACY_BLOCKREADERLOCAL,
            DFSConfigKeys.DFS_CLIENT_USE_LEGACY_BLOCKREADERLOCAL_DEFAULT)) {
        errorCause.append(prefix);
        errorCause.append(DFSConfigKeys.DFS_CLIENT_USE_LEGACY_BLOCKREADERLOCAL);
        errorCause.append(" should not be enabled.\n");
    }

    if (errorCause.length() > 0) {
        output.append(errorMessage);
        output.append(errorCause);
    }

    return output.toString();
}

---

From source file:org.apache.slider.server.appmaster.management.MetricsBindingService.java

License:Apache License

@Override
protected void serviceStart() throws Exception {
    super.serviceStart();

    StringBuilder summary = new StringBuilder();
    Configuration conf = getConfig();

    summary.append("Reporting to JMX");
    // always start the JMX binding
    JmxReporter jmxReporter;//  www. j  a v a 2s . c  o m
    jmxReporter = JmxReporter.forRegistry(metrics).build();
    jmxReporter.start();
    addService(new ClosingService<>(jmxReporter));

    // Ganglia
    if (conf.getBoolean(METRICS_GANGLIA_ENABLED, false)) {
        log.warn("Ganglia integration is not implemented");
        /*
              // This is all disabled due to transitive dependencies on an LGPL library
              com.codahale.metrics.ganglia.GangliaReporter gangliaReporter;
              String host = conf.getTrimmed(METRICS_GANGLIA_HOST, "");
              int port = conf.getInt(METRICS_GANGLIA_PORT, DEFAULT_GANGLIA_PORT);
              int interval = conf.getInt(METRICS_GANGLIA_REPORT_INTERVAL, 60);
              int ttl = 1;
              info.ganglia.gmetric4j.gmetric.GMetric.UDPAddressingMode
                  mcast = info.ganglia.gmetric4j.gmetric.GMetric.UDPAddressingMode.getModeForAddress(host);
              boolean ganglia31 = conf.getBoolean(METRICS_GANGLIA_VERSION_31, true);
                
              final info.ganglia.gmetric4j.gmetric.GMetric ganglia =
                  new info.ganglia.gmetric4j.gmetric.GMetric(
                      host,
                      port,
                      mcast,
                      ttl,
                      ganglia31);
              gangliaReporter = com.codahale.metrics.ganglia.GangliaReporter.forRegistry(metrics)
                               .convertRatesTo(TimeUnit.SECONDS)
                               .convertDurationsTo(
                                   TimeUnit.MILLISECONDS)
                               .build(ganglia);
              gangliaReporter.start(interval, TimeUnit.SECONDS);
              addService(new ClosingService<ScheduledReporter>(gangliaReporter));
              summary.append(String.format(", Ganglia at %s:%d interval=%d",
                  host, port, interval));
              */
    }

    // Logging
    if (conf.getBoolean(METRICS_LOGGING_ENABLED, false)) {
        ScheduledReporter reporter;
        String logName = conf.getTrimmed(METRICS_LOGGING_LOG, METRICS_DEFAULT_LOG);
        int interval = conf.getInt(METRICS_LOGGING_LOG_INTERVAL, METRICS_DEFAULT_LOG_INTERVAL);
        reporter = Slf4jReporter.forRegistry(metrics).convertRatesTo(TimeUnit.SECONDS)
                .outputTo(LoggerFactory.getLogger(logName)).convertDurationsTo(TimeUnit.MILLISECONDS).build();
        reporter.start(interval, TimeUnit.MINUTES);
        addService(new ClosingService<>(reporter));
        summary.append(String.format(", SLF4J to log %s interval=%d", logName, interval));
    }
    reportingDetails = summary.toString();
    log.info(reportingDetails);
}

---

From source file:org.apache.tez.dag.utils.JavaProfilerOptions.java

License:Apache License

public JavaProfilerOptions(Configuration conf) {
    jvmProfilingOpts = conf.getTrimmed(TezConfiguration.TEZ_PROFILE_JVM_OPTS, "");

    if (!Strings.isNullOrEmpty(jvmProfilingOpts)) {
        this.tasksToProfileMap = getTasksToProfile(conf);
        if (!tasksToProfileMap.isEmpty() && jvmProfilingOpts.isEmpty()) {
            LOG.warn(TezConfiguration.TEZ_PROFILE_JVM_OPTS + " should be specified for profiling");
        }/*from   w w w .j  a va  2 s.c  o  m*/
    }
}

---

From source file:org.apache.tez.dag.utils.JavaProfilerOptions.java

License:Apache License

/**
 * <pre>/*  w  w  w  .jav  a  2s.  com*/
 * Get the set of tasks to be profiled in the job. Example formats are
 * v[0,1,2] - To profile subset of tasks in a vertex
 * v[1,2,3];v2[5,6,7] - To profile multiple vertices
 * v[1:5,20,30];v2[2:5,60,7] - To support range of tasks in vertices. Partial
 * ranges are not supported (e.g v[:5],v2[2:]).
 * v[] - To profile all tasks in a vertex
 * </pre>
 *
 * @param conf
 * @return Map<String, BitSet>
 */
private Map<String, BitSet> getTasksToProfile(Configuration conf) {
    String tasksToProfile = conf.getTrimmed(TezConfiguration.TEZ_PROFILE_TASK_LIST, "");
    final Map<String, BitSet> resultSet = new HashMap<String, BitSet>();
    if (tasksToProfile.isEmpty() || !isValid(tasksToProfile)) {
        return resultSet; // empty set
    }
    Matcher matcher = TASKS_TO_PROFILE_REGEX.matcher(tasksToProfile);
    while (matcher.find()) {
        String vertexName = matcher.group(1).trim();
        BitSet profiledTaskSet = parseTasksToProfile(matcher.group(2).trim());
        resultSet.put(vertexName, profiledTaskSet);
    }
    LOG.info("Tasks to profile info=" + resultSet);
    return resultSet;
}

---