Example usage for org.apache.hadoop.conf Configuration get

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration get.

Prototype

public String get(String name)

Source Link

Document

Get the value of the name property, null if no such property exists.

Usage

From source file:com.asakusafw.bridge.hadoop.ConfigurationEditor.java

License:Apache License

/**
 * Returns the {@link StageInfo} object from the Hadoop configuration.
 * @param conf the target Hadoop configuration
 * @return the restored {@link StageInfo} object which is put using
 *     {@link #putStageInfo(Configuration, StageInfo)}, or {@code null} if it does not exist
 *///w  w  w.j a va 2s.c  om
public static StageInfo findStageInfo(Configuration conf) {
    String serialized = conf.get(KEY_STAGE_INFO);
    if (serialized == null) {
        return null;
    }
    return StageInfo.deserialize(serialized);
}

From source file:com.asakusafw.bridge.hadoop.ConfigurationEditorTest.java

License:Apache License

/**
 * test for merge.// w ww.  jav  a2  s  . co m
 */
@Test
public void merge() {
    Map<String, String> entries = new LinkedHashMap<>();
    entries.put("a", "A");
    entries.put("b", "B");
    entries.put("c", "C");

    Configuration conf = new Configuration(false);
    ConfigurationEditor.merge(conf, entries);

    for (Map.Entry<String, String> entry : entries.entrySet()) {
        assertThat(conf.get(entry.getKey()), is(entry.getValue()));
    }
}

From source file:com.asakusafw.bridge.hadoop.directio.DirectFileInputFormat.java

License:Apache License

private static String extract(Configuration conf, String key, boolean mandatory, boolean resolve) {
    String value = conf.get(key);
    if (value == null) {
        if (mandatory) {
            throw new IllegalStateException(MessageFormat.format("missing mandatory configuration: {0}", key));
        }//from  ww w .j  av a2  s.c om
        return null;
    }
    if (resolve) {
        StageInfo info = getStageInfo(conf);
        try {
            value = info.resolveUserVariables(value);
        } catch (IllegalArgumentException e) {
            throw new IllegalStateException(
                    MessageFormat.format("failed to resolve configuration: {0}={1}", key, value), e);
        }
    }
    return value;
}

From source file:com.asakusafw.runtime.directio.hadoop.HadoopDataSourceUtil.java

License:Apache License

/**
 * Returns the local temporary directory.
 * @param localFileSystem the local file system
 * @return the output path (must be on local fs), or {@code null} if not defined
 * @throws IOException if failed to compute the path
 * @throws IllegalArgumentException if some parameters were {@code null}
 *///w w  w. j  av  a  2  s.com
public static Path getLocalTemporaryDirectory(LocalFileSystem localFileSystem) throws IOException {
    if (localFileSystem == null) {
        throw new IllegalArgumentException("localFileSystem must not be null"); //$NON-NLS-1$
    }
    Configuration conf = localFileSystem.getConf();
    if (conf == null) {
        return null;
    }
    String path = conf.get(KEY_LOCAL_TEMPDIR);
    if (path == null) {
        return null;
    }
    LocalFileSystem fs = FileSystem.getLocal(conf);
    Path result = fs.makeQualified(new Path(path));
    return result;
}

From source file:com.asakusafw.runtime.mapreduce.simple.SimpleJobRunner.java

License:Apache License

private KeyValueSorter.Options getSorterOptions(Configuration configuration) {
    long bufferSize = configuration.getLong(KEY_BUFFER_SIZE, -1);
    if (bufferSize < 0) {
        bufferSize = DEFAULT_BUFFER_SIZE;
    } else {/*from w w  w.j  av a 2s .  c o m*/
        bufferSize = Math.max(MIN_BUFFER_SIZE, Math.min(MAX_BUFFER_SIZE, bufferSize));
    }
    File temporaryDirectory = null;
    String tempdirString = configuration.get(KEY_TEMPORARY_LOCATION);
    if (tempdirString != null) {
        temporaryDirectory = new File(tempdirString);
        if (temporaryDirectory.mkdirs() == false && temporaryDirectory.isDirectory() == false) {
            LOG.warn(MessageFormat.format("failed to prepare shuffle temporary directory: {0}={1}",
                    KEY_TEMPORARY_LOCATION, temporaryDirectory));
        }
    }
    boolean compress = configuration.getBoolean(KEY_COMPRESS_BLOCK, DEFAULT_COMPRESS_BLOCK);
    KeyValueSorter.Options options = new KeyValueSorter.Options().withBufferSize((int) bufferSize)
            .withTemporaryDirectory(temporaryDirectory).withCompressBlock(compress);
    return options;
}

From source file:com.asakusafw.runtime.stage.AbstractCleanupStageClient.java

License:Apache License

private VariableTable getPathParser(Configuration configuration) {
    assert configuration != null;
    VariableTable variables = new VariableTable(RedefineStrategy.IGNORE);
    variables.defineVariable(VAR_USER, getUser());
    variables.defineVariable(VAR_DEFINITION_ID, getDefinitionId());
    variables.defineVariable(VAR_STAGE_ID, getStageId());
    variables.defineVariable(VAR_BATCH_ID, getBatchId());
    variables.defineVariable(VAR_FLOW_ID, getFlowId());
    variables.defineVariable(VAR_EXECUTION_ID, getExecutionId());
    String arguments = configuration.get(PROP_ASAKUSA_BATCH_ARGS);
    if (arguments == null) {
        LOG.warn(/*  w  w  w.j av a  2  s .c  o  m*/
                MessageFormat.format("A mandatory property \"{0}\" does not defined", PROP_ASAKUSA_BATCH_ARGS));
    } else {
        variables.defineVariables(arguments);
    }
    return variables;
}

From source file:com.asakusafw.runtime.stage.AbstractStageClient.java

License:Apache License

private VariableTable getPathParser(Configuration configuration) {
    assert configuration != null;
    VariableTable variables = new VariableTable(RedefineStrategy.IGNORE);
    variables.defineVariable(VAR_USER, getUser());
    variables.defineVariable(VAR_DEFINITION_ID, getDefinitionId());
    variables.defineVariable(VAR_STAGE_ID, getStageId());
    variables.defineVariable(VAR_BATCH_ID, getBatchId());
    variables.defineVariable(VAR_FLOW_ID, getFlowId());
    variables.defineVariable(VAR_EXECUTION_ID, getExecutionId());
    String arguments = configuration.get(PROP_ASAKUSA_BATCH_ARGS);
    if (arguments == null) {
        LOG.warn(MessageFormat.format("Missing configuration \"{0}\" (batch arguments)",
                PROP_ASAKUSA_BATCH_ARGS));
    } else {//from   w w w  . java  2 s.  c o  m
        variables.defineVariables(arguments);
    }

    // replace variables
    configuration.set(PROP_ASAKUSA_BATCH_ARGS, variables.toSerialString());
    return variables;
}

From source file:com.asakusafw.runtime.stage.inprocess.InProcessStageConfiguratorTest.java

License:Apache License

/**
 * simple case.//from  ww  w .j ava 2  s  .co  m
 * @throws Exception if failed
 */
@Test
public void simple() throws Exception {
    Job job = newJob();
    Configuration conf = job.getConfiguration();

    conf.setLong(KEY_LIMIT, 100);
    StageInputFormat.setSplitCombinerClass(job, IdentitySplitCombiner.class);

    new Mock(100).configure(job);
    assertThat(conf.get(StageConstants.PROP_JOB_RUNNER), is(notNullValue()));
    assertThat(StageResourceDriver.getAccessMode(job), is(StageResourceDriver.AccessMode.DIRECT));
    assertThat(StageInputFormat.getSplitCombinerClass(job), is((Object) ExtremeSplitCombiner.class));
}

From source file:com.asakusafw.runtime.stage.inprocess.InProcessStageConfiguratorTest.java

License:Apache License

/**
 * disabled./*from  w  ww  .ja v  a  2 s .c  o  m*/
 * @throws Exception if failed
 */
@Test
public void disabled() throws Exception {
    Job job = newJob();
    Configuration conf = job.getConfiguration();

    Assume.assumeThat(conf.get(KEY_LIMIT), is(nullValue()));
    StageInputFormat.setSplitCombinerClass(job, IdentitySplitCombiner.class);

    new Mock(100).configure(job);
    assertThat(conf.get(StageConstants.PROP_JOB_RUNNER), is(nullValue()));
    assertThat(StageResourceDriver.getAccessMode(job), is(not(StageResourceDriver.AccessMode.DIRECT)));
    assertThat(StageInputFormat.getSplitCombinerClass(job), is(not((Object) ExtremeSplitCombiner.class)));
}

From source file:com.asakusafw.runtime.stage.inprocess.InProcessStageConfiguratorTest.java

License:Apache License

/**
 * input size is exceeded.//  w  w  w  . java2 s .c o  m
 * @throws Exception if failed
 */
@Test
public void large() throws Exception {
    Job job = newJob();
    Configuration conf = job.getConfiguration();

    conf.setLong(KEY_LIMIT, 100);
    new Mock(101).configure(job);
    assertThat(conf.get(StageConstants.PROP_JOB_RUNNER), is(nullValue()));
}