Example usage for org.apache.hadoop.conf Configuration writeXml

List of usage examples for org.apache.hadoop.conf Configuration writeXml

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.conf.*
  5. Configuration
  6. writeXml

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration writeXml.

Prototype

public void writeXml(Writer out) throws IOException

Source Link

Usage

From source file:org.apache.tez.mapreduce.examples.GroupByOrderByMRRTest.java

License:Apache License

private static DAG createDAG(Configuration conf, Map<String, LocalResource> commonLocalResources,
        Path stagingDir, String inputPath, String outputPath, boolean useMRSettings) throws Exception {

    Configuration mapStageConf = new JobConf(conf);
    mapStageConf.set(MRJobConfig.MAP_CLASS_ATTR, MyMapper.class.getName());

    MRHelpers.translateMRConfToTez(mapStageConf);

    Configuration iReduceStageConf = new JobConf(conf);
    // TODO replace with auto-reduce parallelism
    iReduceStageConf.setInt(MRJobConfig.NUM_REDUCES, 2);
    iReduceStageConf.set(MRJobConfig.REDUCE_CLASS_ATTR, MyGroupByReducer.class.getName());
    iReduceStageConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS, Text.class.getName());
    iReduceStageConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS, IntWritable.class.getName());
    iReduceStageConf.setBoolean("mapred.mapper.new-api", true);
    MRHelpers.translateMRConfToTez(iReduceStageConf);

    Configuration finalReduceConf = new JobConf(conf);
    finalReduceConf.setInt(MRJobConfig.NUM_REDUCES, 1);
    finalReduceConf.set(MRJobConfig.REDUCE_CLASS_ATTR, MyOrderByNoOpReducer.class.getName());
    finalReduceConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS, IntWritable.class.getName());
    finalReduceConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS, Text.class.getName());
    MRHelpers.translateMRConfToTez(finalReduceConf);

    MRHelpers.configureMRApiUsage(mapStageConf);
    MRHelpers.configureMRApiUsage(iReduceStageConf);
    MRHelpers.configureMRApiUsage(finalReduceConf);

    List<Vertex> vertices = new ArrayList<Vertex>();

    ByteArrayOutputStream outputStream = new ByteArrayOutputStream(4096);
    mapStageConf.writeXml(outputStream);
    String mapStageHistoryText = new String(outputStream.toByteArray(), "UTF-8");
    mapStageConf.set(MRJobConfig.INPUT_FORMAT_CLASS_ATTR, TextInputFormat.class.getName());
    mapStageConf.set(FileInputFormat.INPUT_DIR, inputPath);
    mapStageConf.setBoolean("mapred.mapper.new-api", true);
    DataSourceDescriptor dsd = MRInputHelpers.configureMRInputWithLegacySplitGeneration(mapStageConf,
            stagingDir, true);/*w w  w . j  a  v a  2 s  .  co  m*/

    Vertex mapVertex;
    ProcessorDescriptor mapProcessorDescriptor = ProcessorDescriptor.create(MapProcessor.class.getName())
            .setUserPayload(TezUtils.createUserPayloadFromConf(mapStageConf))
            .setHistoryText(mapStageHistoryText);
    if (!useMRSettings) {
        mapVertex = Vertex.create("initialmap", mapProcessorDescriptor);
    } else {
        mapVertex = Vertex.create("initialmap", mapProcessorDescriptor, -1,
                MRHelpers.getResourceForMRMapper(mapStageConf));
        mapVertex.setTaskLaunchCmdOpts(MRHelpers.getJavaOptsForMRMapper(mapStageConf));
    }
    mapVertex.addTaskLocalFiles(commonLocalResources).addDataSource("MRInput", dsd);
    vertices.add(mapVertex);

    ByteArrayOutputStream iROutputStream = new ByteArrayOutputStream(4096);
    iReduceStageConf.writeXml(iROutputStream);
    String iReduceStageHistoryText = new String(iROutputStream.toByteArray(), "UTF-8");

    ProcessorDescriptor iReduceProcessorDescriptor = ProcessorDescriptor.create(ReduceProcessor.class.getName())
            .setUserPayload(TezUtils.createUserPayloadFromConf(iReduceStageConf))
            .setHistoryText(iReduceStageHistoryText);

    Vertex intermediateVertex;
    if (!useMRSettings) {
        intermediateVertex = Vertex.create("ireduce1", iReduceProcessorDescriptor, 1);
    } else {
        intermediateVertex = Vertex.create("ireduce1", iReduceProcessorDescriptor, 1,
                MRHelpers.getResourceForMRReducer(iReduceStageConf));
        intermediateVertex.setTaskLaunchCmdOpts(MRHelpers.getJavaOptsForMRReducer(iReduceStageConf));
    }
    intermediateVertex.addTaskLocalFiles(commonLocalResources);
    vertices.add(intermediateVertex);

    ByteArrayOutputStream finalReduceOutputStream = new ByteArrayOutputStream(4096);
    finalReduceConf.writeXml(finalReduceOutputStream);
    String finalReduceStageHistoryText = new String(finalReduceOutputStream.toByteArray(), "UTF-8");
    UserPayload finalReducePayload = TezUtils.createUserPayloadFromConf(finalReduceConf);
    Vertex finalReduceVertex;

    ProcessorDescriptor finalReduceProcessorDescriptor = ProcessorDescriptor
            .create(ReduceProcessor.class.getName()).setUserPayload(finalReducePayload)
            .setHistoryText(finalReduceStageHistoryText);
    if (!useMRSettings) {
        finalReduceVertex = Vertex.create("finalreduce", finalReduceProcessorDescriptor, 1);
    } else {
        finalReduceVertex = Vertex.create("finalreduce", finalReduceProcessorDescriptor, 1,
                MRHelpers.getResourceForMRReducer(finalReduceConf));
        finalReduceVertex.setTaskLaunchCmdOpts(MRHelpers.getJavaOptsForMRReducer(finalReduceConf));
    }
    finalReduceVertex.addTaskLocalFiles(commonLocalResources);
    finalReduceVertex.addDataSink("MROutput",
            MROutputLegacy.createConfigBuilder(finalReduceConf, TextOutputFormat.class, outputPath).build());
    vertices.add(finalReduceVertex);

    DAG dag = DAG.create("groupbyorderbymrrtest");
    for (Vertex v : vertices) {
        dag.addVertex(v);
    }

    OrderedPartitionedKVEdgeConfig edgeConf1 = OrderedPartitionedKVEdgeConfig
            .newBuilder(Text.class.getName(), IntWritable.class.getName(), HashPartitioner.class.getName())
            .setFromConfiguration(conf).configureInput().useLegacyInput().done().build();
    dag.addEdge(Edge.create(dag.getVertex("initialmap"), dag.getVertex("ireduce1"),
            edgeConf1.createDefaultEdgeProperty()));

    OrderedPartitionedKVEdgeConfig edgeConf2 = OrderedPartitionedKVEdgeConfig
            .newBuilder(IntWritable.class.getName(), Text.class.getName(), HashPartitioner.class.getName())
            .setFromConfiguration(conf).configureInput().useLegacyInput().done().build();
    dag.addEdge(Edge.create(dag.getVertex("ireduce1"), dag.getVertex("finalreduce"),
            edgeConf2.createDefaultEdgeProperty()));

    return dag;
}

From source file:org.apache.tez.mapreduce.examples.TestOrderedWordCount.java

License:Apache License

@VisibleForTesting
public DAG createDAG(FileSystem fs, Configuration conf, Map<String, LocalResource> commonLocalResources,
        Path stagingDir, int dagIndex, String inputPath, String outputPath, boolean generateSplitsInClient,
        boolean useMRSettings, int intermediateNumReduceTasks) throws Exception {

    Configuration mapStageConf = new JobConf(conf);
    mapStageConf.set(MRJobConfig.MAP_CLASS_ATTR, TokenizerMapper.class.getName());

    MRHelpers.translateMRConfToTez(mapStageConf);

    Configuration iReduceStageConf = new JobConf(conf);
    // TODO replace with auto-reduce parallelism
    iReduceStageConf.setInt(MRJobConfig.NUM_REDUCES, 2);
    iReduceStageConf.set(MRJobConfig.REDUCE_CLASS_ATTR, IntSumReducer.class.getName());
    iReduceStageConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS, Text.class.getName());
    iReduceStageConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS, IntWritable.class.getName());
    iReduceStageConf.setBoolean("mapred.mapper.new-api", true);
    MRHelpers.translateMRConfToTez(iReduceStageConf);

    Configuration finalReduceConf = new JobConf(conf);
    finalReduceConf.setInt(MRJobConfig.NUM_REDUCES, 1);
    finalReduceConf.set(MRJobConfig.REDUCE_CLASS_ATTR, MyOrderByNoOpReducer.class.getName());
    finalReduceConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS, IntWritable.class.getName());
    finalReduceConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS, Text.class.getName());
    MRHelpers.translateMRConfToTez(finalReduceConf);

    MRHelpers.configureMRApiUsage(mapStageConf);
    MRHelpers.configureMRApiUsage(iReduceStageConf);
    MRHelpers.configureMRApiUsage(finalReduceConf);

    List<Vertex> vertices = new ArrayList<Vertex>();

    ByteArrayOutputStream outputStream = new ByteArrayOutputStream(4096);
    mapStageConf.writeXml(outputStream);
    String mapStageHistoryText = new String(outputStream.toByteArray(), "UTF-8");
    DataSourceDescriptor dsd;/* ww w. j  a  v a 2 s  .c o m*/
    if (generateSplitsInClient) {
        mapStageConf.set(MRJobConfig.INPUT_FORMAT_CLASS_ATTR, TextInputFormat.class.getName());
        mapStageConf.set(FileInputFormat.INPUT_DIR, inputPath);
        mapStageConf.setBoolean("mapred.mapper.new-api", true);
        dsd = MRInputHelpers.configureMRInputWithLegacySplitGeneration(mapStageConf, stagingDir, true);
    } else {
        dsd = MRInputLegacy.createConfigBuilder(mapStageConf, TextInputFormat.class, inputPath).build();
    }

    Map<String, String> mapEnv = Maps.newHashMap();
    MRHelpers.updateEnvBasedOnMRTaskEnv(mapStageConf, mapEnv, true);
    Map<String, String> reduceEnv = Maps.newHashMap();
    MRHelpers.updateEnvBasedOnMRTaskEnv(mapStageConf, reduceEnv, false);

    Vertex mapVertex;
    ProcessorDescriptor mapProcessorDescriptor = ProcessorDescriptor.create(MapProcessor.class.getName())
            .setUserPayload(TezUtils.createUserPayloadFromConf(mapStageConf))
            .setHistoryText(mapStageHistoryText);
    if (!useMRSettings) {
        mapVertex = Vertex.create("initialmap", mapProcessorDescriptor);
    } else {
        mapVertex = Vertex.create("initialmap", mapProcessorDescriptor, -1,
                MRHelpers.getResourceForMRMapper(mapStageConf));
        mapVertex.setTaskLaunchCmdOpts(MRHelpers.getJavaOptsForMRMapper(mapStageConf));
        mapVertex.setTaskEnvironment(mapEnv);
    }
    mapVertex.addTaskLocalFiles(commonLocalResources).addDataSource("MRInput", dsd);
    vertices.add(mapVertex);

    ByteArrayOutputStream iROutputStream = new ByteArrayOutputStream(4096);
    iReduceStageConf.writeXml(iROutputStream);
    String iReduceStageHistoryText = new String(iROutputStream.toByteArray(), "UTF-8");

    ProcessorDescriptor iReduceProcessorDescriptor = ProcessorDescriptor.create(ReduceProcessor.class.getName())
            .setUserPayload(TezUtils.createUserPayloadFromConf(iReduceStageConf))
            .setHistoryText(iReduceStageHistoryText);

    Vertex intermediateVertex;
    if (!useMRSettings) {
        intermediateVertex = Vertex.create("intermediate_reducer", iReduceProcessorDescriptor,
                intermediateNumReduceTasks);
    } else {
        intermediateVertex = Vertex.create("intermediate_reducer", iReduceProcessorDescriptor,
                intermediateNumReduceTasks, MRHelpers.getResourceForMRReducer(iReduceStageConf));
        intermediateVertex.setTaskLaunchCmdOpts(MRHelpers.getJavaOptsForMRReducer(iReduceStageConf));
        intermediateVertex.setTaskEnvironment(reduceEnv);
    }
    intermediateVertex.addTaskLocalFiles(commonLocalResources);
    vertices.add(intermediateVertex);

    ByteArrayOutputStream finalReduceOutputStream = new ByteArrayOutputStream(4096);
    finalReduceConf.writeXml(finalReduceOutputStream);
    String finalReduceStageHistoryText = new String(finalReduceOutputStream.toByteArray(), "UTF-8");
    UserPayload finalReducePayload = TezUtils.createUserPayloadFromConf(finalReduceConf);
    Vertex finalReduceVertex;

    ProcessorDescriptor finalReduceProcessorDescriptor = ProcessorDescriptor
            .create(ReduceProcessor.class.getName()).setUserPayload(finalReducePayload)
            .setHistoryText(finalReduceStageHistoryText);
    if (!useMRSettings) {
        finalReduceVertex = Vertex.create("finalreduce", finalReduceProcessorDescriptor, 1);
    } else {
        finalReduceVertex = Vertex.create("finalreduce", finalReduceProcessorDescriptor, 1,
                MRHelpers.getResourceForMRReducer(finalReduceConf));
        finalReduceVertex.setTaskLaunchCmdOpts(MRHelpers.getJavaOptsForMRReducer(finalReduceConf));
        finalReduceVertex.setTaskEnvironment(reduceEnv);
    }
    finalReduceVertex.addTaskLocalFiles(commonLocalResources);
    finalReduceVertex.addDataSink("MROutput",
            MROutputLegacy.createConfigBuilder(finalReduceConf, TextOutputFormat.class, outputPath).build());
    vertices.add(finalReduceVertex);

    DAG dag = DAG.create("OrderedWordCount" + dagIndex);
    dag.setDAGInfo("{ \"context\": \"Tez\", \"description\": \"TestOrderedWordCount Job\" }");
    for (int i = 0; i < vertices.size(); ++i) {
        dag.addVertex(vertices.get(i));
    }

    OrderedPartitionedKVEdgeConfig edgeConf1 = OrderedPartitionedKVEdgeConfig
            .newBuilder(Text.class.getName(), IntWritable.class.getName(), HashPartitioner.class.getName())
            .setFromConfiguration(conf).configureInput().useLegacyInput().done().build();
    dag.addEdge(Edge.create(dag.getVertex("initialmap"), dag.getVertex("intermediate_reducer"),
            edgeConf1.createDefaultEdgeProperty()));

    OrderedPartitionedKVEdgeConfig edgeConf2 = OrderedPartitionedKVEdgeConfig
            .newBuilder(IntWritable.class.getName(), Text.class.getName(), HashPartitioner.class.getName())
            .setFromConfiguration(conf).configureInput().useLegacyInput().done().build();
    dag.addEdge(Edge.create(dag.getVertex("intermediate_reducer"), dag.getVertex("finalreduce"),
            edgeConf2.createDefaultEdgeProperty()));

    updateDAGACls(conf, dag, dagIndex);

    return dag;
}

From source file:org.apache.tez.mapreduce.hadoop.TestMRInputHelpers.java

License:Apache License

@BeforeClass
public static void setup() throws IOException {
    try {//from   w  ww.  j a  va 2  s. com
        conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TEST_ROOT_DIR);
        dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).format(true).racks(null).build();
        remoteFs = dfsCluster.getFileSystem();
    } catch (IOException io) {
        throw new RuntimeException("problem starting mini dfs cluster", io);
    }

    Configuration testConf = new YarnConfiguration(dfsCluster.getFileSystem().getConf());

    File testConfFile = new File(TEST_ROOT_DIR, "test.xml");
    try {
        testConfFile.createNewFile();
        testConf.writeXml(new FileOutputStream(testConfFile));
        testConfFile.deleteOnExit();
    } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
        throw new RuntimeException(e);
    }

    remoteFs.mkdirs(new Path("/tmp/input/"));
    remoteFs.mkdirs(new Path("/tmp/splitsDirNew/"));
    remoteFs.mkdirs(new Path("/tmp/splitsDirOld/"));
    testFilePath = remoteFs.makeQualified(new Path("/tmp/input/test.xml"));
    remoteFs.copyFromLocalFile(new Path(testConfFile.getAbsolutePath()), testFilePath);
    FileStatus fsStatus = remoteFs.getFileStatus(testFilePath);
    Assert.assertTrue(fsStatus.getLen() > 0);

    oldSplitsDir = remoteFs.makeQualified(new Path("/tmp/splitsDirOld/"));
    newSplitsDir = remoteFs.makeQualified(new Path("/tmp/splitsDirNew/"));
}

From source file:org.apache.tez.mapreduce.MiniMRRTezCluster.java

License:Apache License

@Override
public void serviceStart() throws Exception {
    super.serviceStart();
    File workDir = super.getTestWorkDir();
    Configuration conf = super.getConfig();

    confFilePath = new Path(workDir.getAbsolutePath(), YARN_CLUSTER_CONFIG);
    File confFile = new File(confFilePath.toString());
    try {/*from   www.ja va 2 s.  c  o m*/
        confFile.createNewFile();
        conf.writeXml(new FileOutputStream(confFile));
        confFile.deleteOnExit();
    } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
        throw new RuntimeException(e);
    }
    confFilePath = new Path(confFile.getAbsolutePath());
}

From source file:org.apache.tez.test.MiniTezCluster.java

License:Apache License

@Override
public void serviceStart() throws Exception {
    LOG.info("Starting MiniTezCluster");
    super.serviceStart();
    File workDir = super.getTestWorkDir();
    Configuration conf = super.getConfig();

    confFilePath = new Path(workDir.getAbsolutePath(), YARN_CLUSTER_CONFIG);
    File confFile = new File(confFilePath.toString());
    try {/*from  www. j a  va2  s  . c  o  m*/
        confFile.createNewFile();
        conf.writeXml(new FileOutputStream(confFile));
        confFile.deleteOnExit();
    } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
        throw new RuntimeException(e);
    }
    confFilePath = new Path(confFile.getAbsolutePath());
    conf.setStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, workDir.getAbsolutePath(),
            System.getProperty("java.class.path"));
    LOG.info("Setting yarn-site.xml via YARN-APP-CP at: "
            + conf.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH));
}

From source file:org.apache.tez.tests.MiniTezClusterWithTimeline.java

License:Apache License

@Override
public void serviceStart() throws Exception {
    LOG.info("Starting MiniTezClusterWithTimeline");
    super.serviceStart();
    File workDir = super.getTestWorkDir();
    Configuration conf = super.getConfig();

    confFilePath = new Path(workDir.getAbsolutePath(), YARN_CLUSTER_CONFIG);
    File confFile = new File(confFilePath.toString());
    try {/*from w  w w  .  j  a  va2 s  .  c  o  m*/
        confFile.createNewFile();
        conf.writeXml(new FileOutputStream(confFile));
        confFile.deleteOnExit();
    } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
        throw new RuntimeException(e);
    }
    confFilePath = new Path(confFile.getAbsolutePath());
    conf.setStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, workDir.getAbsolutePath(),
            System.getProperty("java.class.path"));
    LOG.info("Setting yarn-site.xml via YARN-APP-CP at: "
            + conf.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH));
}

From source file:org.apache.zeppelin.integration.MiniHadoopCluster.java

License:Apache License

protected void saveConfig(Configuration conf, String dest) throws IOException {
    Configuration redacted = new Configuration(conf);
    // This setting references a test class that is not available when using a real Spark
    // installation, so remove it from client configs.
    redacted.unset("net.topology.node.switch.mapping.impl");

    FileOutputStream out = new FileOutputStream(dest);
    try {/* w  w  w  .j  a  va2  s  .  co  m*/
        redacted.writeXml(out);
    } finally {
        out.close();
    }
    LOGGER.info("Save configuration to " + dest);
}

From source file:org.exem.flamingo.shared.util.ConfigurationUtils.java

License:Apache License

/**
 * {@link Configuration}? XML .//w  w  w . j a v a  2  s. co  m
 *
 * @param conf {@link Configuration}
 * @return XML
 */
public static String configurationToXml(Configuration conf) {
    try {
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        conf.writeXml(baos);
        return new String(baos.toByteArray());
    } catch (Exception e) {
        throw new ServiceException(" ? XML   .", e);
    }
}

From source file:org.gridgain.grid.ggfs.GridGgfsHadoopFileSystemAbstractSelfTest.java

License:Open Source License

/** {@inheritDoc} */
@Override//from   w  w w. j  av  a 2s. c  o  m
protected void beforeTestsStarted() throws Exception {
    Configuration secondaryConf = configuration(SECONDARY_AUTHORITY, true, true);

    secondaryConf.setInt("fs.ggfs.block.size", 1024);

    String path = U.getGridGainHome() + SECONDARY_CFG_PATH;

    File file = new File(path);

    try (FileOutputStream fos = new FileOutputStream(file)) {
        secondaryConf.writeXml(fos);
    }

    startNodes();
}

From source file:org.kiji.maven.plugins.hbase.StartMojo.java

License:Apache License

/**
 * Writes the specified configuration to the specified file.
 *
 * @param conf The configuration to write.
 * @param siteFile The file to write the configuration to.
 * @throws MojoExecutionException If there is an error writing the file.
 *//*from w w w .  j  a  va2 s .c om*/
private void writeSiteFile(Configuration conf, File siteFile) throws MojoExecutionException {
    // Create the parent directory for the site file if it does not already exist.
    createFileParentDir(siteFile);

    // Write the file.
    FileOutputStream fileOutputStream = null;
    try {
        fileOutputStream = new FileOutputStream(siteFile);
        conf.writeXml(fileOutputStream);
    } catch (IOException e) {
        throw new MojoExecutionException("Unable to write to site file: " + siteFile.getPath(), e);
    } finally {
        closeFileOutputStream(fileOutputStream);
    }
    getLog().info("Wrote " + siteFile.getPath() + ".");
}