Example usage for org.apache.hadoop.conf Configuration set

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration set.

Prototype

public void set(String name, String value) 

Source Link

Document

Set the value of the name property.

Usage

From source file:com.asakusafw.runtime.stage.AbstractStageClient.java

License:Apache License

@Override
protected int execute(String[] args) throws Exception {
    Configuration conf = getConf();
    conf.set(StageConstants.PROP_BATCH_ID, getBatchId());
    conf.set(StageConstants.PROP_FLOW_ID, getFlowId());
    LOG.info(MessageFormat.format("Initializing Job: batchId={0}, flowId={1}, executionId={2}, stageId={3}",
            getBatchId(), getFlowId(), getExecutionId(), getStageId()));
    Job job = createJob(conf);/*from   ww w.  j  a v a  2s  .c  o m*/
    return submit(job);
}

---

From source file:com.asakusafw.runtime.stage.AbstractStageClient.java

License:Apache License

private VariableTable getPathParser(Configuration configuration) {
    assert configuration != null;
    VariableTable variables = new VariableTable(RedefineStrategy.IGNORE);
    variables.defineVariable(VAR_USER, getUser());
    variables.defineVariable(VAR_DEFINITION_ID, getDefinitionId());
    variables.defineVariable(VAR_STAGE_ID, getStageId());
    variables.defineVariable(VAR_BATCH_ID, getBatchId());
    variables.defineVariable(VAR_FLOW_ID, getFlowId());
    variables.defineVariable(VAR_EXECUTION_ID, getExecutionId());
    String arguments = configuration.get(PROP_ASAKUSA_BATCH_ARGS);
    if (arguments == null) {
        LOG.warn(MessageFormat.format("Missing configuration \"{0}\" (batch arguments)",
                PROP_ASAKUSA_BATCH_ARGS));
    } else {//from  w  w w .  ja v  a 2s  . com
        variables.defineVariables(arguments);
    }

    // replace variables
    configuration.set(PROP_ASAKUSA_BATCH_ARGS, variables.toSerialString());
    return variables;
}

---

From source file:com.asakusafw.runtime.stage.configurator.AutoLocalStageConfigurator.java

License:Apache License

private void localize(Job job) {
    Configuration conf = job.getConfiguration();

    // reset job-tracker
    conf.set(KEY_JOBTRACKER, DEFAULT_JOBTRACKER);

    // replace local directories
    String tmpDir = conf.get(KEY_TEMPORARY_DIRECTORY, "");
    if (tmpDir.isEmpty()) {
        String name = System.getProperty("user.name", "asakusa");
        tmpDir = String.format("/tmp/hadoop-%s/autolocal", name);
    } else if (tmpDir.length() > 1 && tmpDir.endsWith("/")) {
        tmpDir = tmpDir.substring(0, tmpDir.length() - 1);
    }/*from  w  ww .j  av  a 2  s  .  c o m*/
    if (conf.getBoolean(KEY_DIRECTORY_QUALIFIER, true)) {
        String qualifier = UUID.randomUUID().toString();
        tmpDir = String.format("%s/%s", tmpDir, qualifier);
    }
    LOG.info(MessageFormat.format("Substituting temporary dir: job={0}, target={1}", job.getJobName(), tmpDir));
    conf.set(KEY_LOCAL_DIR, tmpDir + "/mapred/local");
    conf.set(KEY_STAGING_DIR, tmpDir + "/mapred/staging");
}

---

From source file:com.asakusafw.runtime.stage.inprocess.InProcessStageConfigurator.java

License:Apache License

private void install(Job job) {
    Configuration conf = job.getConfiguration();
    int prefixLength = KEY_PREFIX_REPLACE.length();
    for (Map.Entry<String, String> entry : conf.getValByRegex(PATTERN_KEY_REPLACE.pattern()).entrySet()) {
        assert entry.getKey().length() >= prefixLength;
        String key = entry.getKey().substring(prefixLength);
        if (key.isEmpty()) {
            continue;
        }// w w  w  . j a v  a 2s .  co m
        String value = entry.getValue();
        if (LOG.isDebugEnabled()) {
            LOG.debug(MessageFormat.format("activate in-process configuration: {0}=\"{1}\"->\"{2}\"", //$NON-NLS-1$
                    key, conf.get(key, ""), //$NON-NLS-1$
                    value));
        }
        conf.set(key, value);
    }
    conf.set(StageConstants.PROP_JOB_RUNNER, SimpleJobRunner.class.getName());
    StageResourceDriver.setAccessMode(job, StageResourceDriver.AccessMode.DIRECT);
    StageInputFormat.setSplitCombinerClass(job, ExtremeSplitCombiner.class);
}

---

From source file:com.asakusafw.runtime.stage.inprocess.InProcessStageConfiguratorTest.java

License:Apache License

/**
 * activate alternative properties./* ww  w .j  a va2 s  .  c om*/
 * @throws Exception if failed
 */
@Test
public void activate_properties() throws Exception {
    Job job = newJob();
    Configuration conf = job.getConfiguration();

    conf.setLong(KEY_LIMIT, 100);
    conf.set(KEY_PREFIX_REPLACE + "com.example.testing", "YES!");
    new Mock(100).configure(job);
    assertThat(conf.get("com.example.testing"), is("YES!"));
}

---

From source file:com.asakusafw.runtime.stage.inprocess.InProcessStageConfiguratorTest.java

License:Apache License

/**
 * activate alternative properties.//ww w. j  a v a2s  .  c  o  m
 * @throws Exception if failed
 */
@Test
public void activate_properties_skip() throws Exception {
    Job job = newJob();
    Configuration conf = job.getConfiguration();

    conf.setLong(KEY_LIMIT, 100);
    conf.set(KEY_PREFIX_REPLACE + "com.example.testing", "YES!");
    new Mock(1000).configure(job);
    assertThat(conf.get("com.example.testing"), is(not("YES!")));
}

---

From source file:com.asakusafw.runtime.stage.optimizer.LibraryCopySuppressionConfigurator.java

License:Apache License

@Override
public void configure(Job job) throws IOException, InterruptedException {
    Configuration conf = job.getConfiguration();
    if (conf.getBoolean(KEY_ENABLED, DEFAULT_ENABLED) == false) {
        return;/* w  w  w  .ja v  a 2 s .c o  m*/
    }
    // activates only if application launcher is used
    if (conf.getBoolean(ApplicationLauncher.KEY_LAUNCHER_USED, false) == false) {
        return;
    }
    if (JobCompatibility.isLocalMode(job) == false) {
        return;
    }
    String libraries = conf.get(KEY_CONF_LIBRARIES);
    if (libraries == null || libraries.isEmpty()) {
        return;
    }
    Set<String> loaded = new HashSet<>();
    ClassLoader loader = conf.getClassLoader();
    if (loader instanceof URLClassLoader) {
        for (URL url : ((URLClassLoader) loader).getURLs()) {
            try {
                loaded.add(url.toURI().toString());
            } catch (URISyntaxException e) {
                LOG.warn(MessageFormat.format("Failed to analyze classpath: {0}", url));
            }
        }
    }
    if (loaded.isEmpty()) {
        return;
    }
    StringBuilder result = new StringBuilder();
    for (String library : libraries.split(",")) { //$NON-NLS-1$
        if (loaded.contains(library)) {
            if (LOG.isDebugEnabled()) {
                LOG.debug(MessageFormat.format("Keep library: {0}", library)); //$NON-NLS-1$
            }
        } else {
            if (LOG.isDebugEnabled()) {
                LOG.debug(MessageFormat.format("Suppress library: {0}", library)); //$NON-NLS-1$
            }
            if (result.length() != 0) {
                result.append(',');
            }
            result.append(library);
        }
    }
    if (result.length() > 0) {
        conf.set(KEY_CONF_LIBRARIES, result.toString());
    } else {
        if (CONFIGURATION_UNSET != null) {
            try {
                CONFIGURATION_UNSET.invoke(conf, KEY_CONF_LIBRARIES);
                return;
            } catch (Exception e) {
                LOG.warn(MessageFormat.format("Failed to invoke {0}", CONFIGURATION_UNSET), e);
            }
        }
        String newLibraries = selectLibraries(libraries);
        conf.set(KEY_CONF_LIBRARIES, newLibraries);
    }
}

---

From source file:com.asakusafw.runtime.stage.output.BridgeOutputFormat.java

License:Apache License

private static void save(Configuration conf, List<OutputSpec> specs) {
    assert conf != null;
    assert specs != null;
    for (OutputSpec spec : specs) {
        if (spec.resolved) {
            throw new IllegalStateException();
        }/*from  w  w w  .jav a 2 s.  co  m*/
    }
    ByteArrayOutputStream sink = new ByteArrayOutputStream();
    try (DataOutputStream output = new DataOutputStream(new GZIPOutputStream(new Base64OutputStream(sink)))) {
        WritableUtils.writeVLong(output, SERIAL_VERSION);
        WritableUtils.writeVInt(output, specs.size());
        for (OutputSpec spec : specs) {
            WritableUtils.writeString(output, spec.basePath);
            WritableUtils.writeVInt(output, spec.deletePatterns.size());
            for (String pattern : spec.deletePatterns) {
                WritableUtils.writeString(output, pattern);
            }
        }
    } catch (IOException e) {
        throw new IllegalStateException(e);
    }
    conf.set(KEY, new String(sink.toByteArray(), ASCII));
}

---

From source file:com.asakusafw.runtime.util.hadoop.ConfigurationProviderTest.java

License:Apache License

private File putConf(String path) {
    Configuration c = new Configuration(false);
    c.set("testing.conf", "added");
    File file = create(path);//  w  w  w.j  av  a 2  s.c  om
    try (OutputStream s = new FileOutputStream(file)) {
        c.writeXml(s);
    } catch (IOException e) {
        throw new AssertionError(e);
    }
    return file;
}

---

From source file:com.asakusafw.testdriver.hadoop.ConfigurationFactory.java

License:Apache License

@Override
protected void configure(Configuration configuration) {
    if (preferences.getLocalFileSystemClassName() != null) {
        configuration.set("fs.file.impl", preferences.getLocalFileSystemClassName()); //$NON-NLS-1$
        configuration.setBoolean("fs.fs.impl.disable.cache", true); //$NON-NLS-1$
    }/*  w w w.  jav a2s.c  o m*/
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »