Example usage for org.apache.hadoop.conf Configuration addResource

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration addResource.

Prototype

public void addResource(Configuration conf) 

Source Link

Document

Add a configuration resource.

Usage

From source file:co.cask.cdap.app.runtime.spark.SparkRuntimeContextProvider.java

License:Apache License

private static Configuration createHConf() throws MalformedURLException {
    Configuration hConf = new Configuration();
    hConf.clear();//from  w  w  w  .  j  a v a2  s.  co m
    hConf.addResource(new File(HCONF_FILE_NAME).toURI().toURL());
    return hConf;
}

---

From source file:co.cask.cdap.explore.service.ExploreServiceUtils.java

License:Apache License

/**
 * Change yarn-site.xml file, and return a temp copy of it to which are added
 * necessary options./*w  w w .  j  ava2 s .  c o  m*/
 */
private static File updateYarnConfFile(File confFile, File tempDir) {
    Configuration conf = new Configuration(false);
    try {
        conf.addResource(confFile.toURI().toURL());
    } catch (MalformedURLException e) {
        LOG.error("File {} is malformed.", confFile, e);
        throw Throwables.propagate(e);
    }

    String yarnAppClassPath = conf.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            Joiner.on(",").join(YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH));

    // add the pwd/* at the beginning of classpath. so user's jar will take precedence and without this change,
    // job.jar will be at the beginning of the classpath, since job.jar has old guava version classes,
    // we want to add pwd/* before
    yarnAppClassPath = "$PWD/*," + yarnAppClassPath;

    conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH, yarnAppClassPath);

    File newYarnConfFile = new File(tempDir, "yarn-site.xml");
    try (FileOutputStream os = new FileOutputStream(newYarnConfFile)) {
        conf.writeXml(os);
    } catch (IOException e) {
        LOG.error("Problem creating and writing to temporary yarn-conf.xml conf file at {}", newYarnConfFile,
                e);
        throw Throwables.propagate(e);
    }

    return newYarnConfFile;
}

---

From source file:co.cask.cdap.explore.service.ExploreServiceUtils.java

License:Apache License

/**
 * Change mapred-site.xml file, and return a temp copy of it to which are added
 * necessary options./*from  w w  w.  j a v a  2 s.  c o  m*/
 */
private static File updateMapredConfFile(File confFile, File tempDir) {
    Configuration conf = new Configuration(false);
    try {
        conf.addResource(confFile.toURI().toURL());
    } catch (MalformedURLException e) {
        LOG.error("File {} is malformed.", confFile, e);
        throw Throwables.propagate(e);
    }

    String mrAppClassPath = conf.get(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH,
            MRJobConfig.DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH);

    // Add the pwd/* at the beginning of classpath. Without this change, old jars from mr framework classpath
    // get into classpath.
    mrAppClassPath = "$PWD/*," + mrAppClassPath;

    conf.set(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH, mrAppClassPath);

    File newMapredConfFile = new File(tempDir, "mapred-site.xml");
    try (FileOutputStream os = new FileOutputStream(newMapredConfFile)) {
        conf.writeXml(os);
    } catch (IOException e) {
        LOG.error("Problem creating and writing to temporary mapred-site.xml conf file at {}",
                newMapredConfFile, e);
        throw Throwables.propagate(e);
    }

    return newMapredConfFile;
}

---

From source file:co.cask.cdap.explore.service.ExploreServiceUtils.java

License:Apache License

/**
 * Change hive-site.xml file, and return a temp copy of it to which are added
 * necessary options.//ww w.  j  ava  2s  .  c  o  m
 */
private static File updateHiveConfFile(File confFile, File tempDir) {
    Configuration conf = new Configuration(false);
    try {
        conf.addResource(confFile.toURI().toURL());
    } catch (MalformedURLException e) {
        LOG.error("File {} is malformed.", confFile, e);
        throw Throwables.propagate(e);
    }

    // we prefer jars at container's root directory before job.jar,
    // we edit the YARN_APPLICATION_CLASSPATH in yarn-site.xml using
    // co.cask.cdap.explore.service.ExploreServiceUtils.updateYarnConfFile and
    // setting the MAPREDUCE_JOB_CLASSLOADER and MAPREDUCE_JOB_USER_CLASSPATH_FIRST to false will put
    // YARN_APPLICATION_CLASSPATH before job.jar for container's classpath.
    conf.setBoolean(Job.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false);
    conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, false);

    String sparkHome = System.getenv(Constants.SPARK_HOME);
    if (sparkHome != null) {
        LOG.debug("Setting spark.home in hive conf to {}", sparkHome);
        conf.set("spark.home", sparkHome);
    }

    File newHiveConfFile = new File(tempDir, "hive-site.xml");

    try (FileOutputStream os = new FileOutputStream(newHiveConfFile)) {
        conf.writeXml(os);
    } catch (IOException e) {
        LOG.error("Problem creating temporary hive-site.xml conf file at {}", newHiveConfFile, e);
        throw Throwables.propagate(e);
    }
    return newHiveConfFile;
}

---

From source file:co.cask.cdap.explore.service.ExploreServiceUtilsTest.java

License:Apache License

@Test
public void hijackConfFileTest() throws Exception {
    Configuration conf = new Configuration(false);
    conf.set("foo", "bar");
    Assert.assertEquals(1, conf.size());

    File tempDir = tmpFolder.newFolder();

    File confFile = tmpFolder.newFile("hive-site.xml");

    try (FileOutputStream os = new FileOutputStream(confFile)) {
        conf.writeXml(os);//  w w w  . j av  a2 s  . c  o m
    }

    File newConfFile = ExploreServiceUtils.updateConfFileForExplore(confFile, tempDir);

    conf = new Configuration(false);
    conf.addResource(newConfFile.toURI().toURL());

    Assert.assertEquals(3, conf.size());
    Assert.assertEquals("false", conf.get(Job.MAPREDUCE_JOB_USER_CLASSPATH_FIRST));
    Assert.assertEquals("false", conf.get(Job.MAPREDUCE_JOB_CLASSLOADER));
    Assert.assertEquals("bar", conf.get("foo"));

    // check yarn-site changes
    confFile = tmpFolder.newFile("yarn-site.xml");
    conf = new YarnConfiguration();

    try (FileOutputStream os = new FileOutputStream(confFile)) {
        conf.writeXml(os);
    }

    String yarnApplicationClassPath = "$PWD/*," + conf.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            Joiner.on(",").join(YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH));

    newConfFile = ExploreServiceUtils.updateConfFileForExplore(confFile, tempDir);

    conf = new Configuration(false);
    conf.addResource(newConfFile.toURI().toURL());

    Assert.assertEquals(yarnApplicationClassPath, conf.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH));

    // check mapred-site changes
    confFile = tmpFolder.newFile("mapred-site.xml");
    conf = new YarnConfiguration();

    try (FileOutputStream os = new FileOutputStream(confFile)) {
        conf.writeXml(os);
    }

    String mapredApplicationClassPath = "$PWD/*," + conf.get(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH,
            MRJobConfig.DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH);

    newConfFile = ExploreServiceUtils.updateConfFileForExplore(confFile, tempDir);

    conf = new Configuration(false);
    conf.addResource(newConfFile.toURI().toURL());

    Assert.assertEquals(mapredApplicationClassPath, conf.get(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH));

    // Ensure conf files that are not hive-site.xml/mapred-site.xml/yarn-site.xml are unchanged
    confFile = tmpFolder.newFile("core-site.xml");
    Assert.assertEquals(confFile, ExploreServiceUtils.updateConfFileForExplore(confFile, tempDir));
}

---

From source file:co.cask.cdap.gateway.handlers.ConfigServiceTest.java

License:Apache License

@Test
public void testConfig() {
    String cConfResourceString = "<configuration>\n" + "\n" + "  <property>\n"
            + "    <name>stream.zz.threshold</name>\n" + "    <value>1</value>\n"
            + "    <description>Some description</description>\n" + "  </property>\n" + "\n"
            + "</configuration>";
    ReaderInputStream cConfResource = new ReaderInputStream(new StringReader(cConfResourceString));
    CConfiguration cConf = CConfiguration.create(cConfResource);

    ConfigEntry cConfEntry = new ConfigEntry("stream.zz.threshold", "1", cConfResource.toString());

    // hConf//from   www . ja  v a  2  s .  c o m
    Configuration hConf = new Configuration();
    String hConfResourceString = "<configuration>\n" + "\n" + "  <property>\n"
            + "    <name>stream.notification.threshold</name>\n" + "    <value>3</value>\n"
            + "    <description>Some description</description>\n" + "  </property>\n" + "\n"
            + "</configuration>";
    ReaderInputStream hConfResource = new ReaderInputStream(new StringReader(hConfResourceString));
    hConf.addResource(hConfResource);

    ConfigEntry hConfEntry = new ConfigEntry("stream.notification.threshold", "3", hConfResource.toString());

    // test
    ConfigService configService = new ConfigService(cConf, hConf);
    List<ConfigEntry> cConfEntries = configService.getCConf();
    Assert.assertTrue(cConfEntries.contains(cConfEntry));

    List<ConfigEntry> hConfEntries = configService.getHConf();
    Assert.assertTrue(hConfEntries.contains(hConfEntry));
}

---

From source file:co.cask.cdap.hive.context.HConfCodec.java

License:Apache License

@Override
public Configuration decode(byte[] data) throws IOException {
    if (data == null) {
        return HBaseConfiguration.create();
    }/*from   w ww  .ja  va2 s  .com*/

    ByteArrayInputStream bin = new ByteArrayInputStream(data);
    Configuration hConfiguration = new Configuration();
    hConfiguration.addResource(bin);
    return hConfiguration;
}

---

From source file:co.cask.cdap.internal.app.runtime.spark.AbstractSparkContext.java

License:Apache License

/**
 * Adds the supplied {@link Configuration} file as an resource
 * This configuration is needed to read/write {@link Dataset} using {@link DataSetInputFormat}/{@link
 * DataSetOutputFormat} by {@link JavaSparkContext#readFromDataset(String, Class, Class)} or
 * {@link ScalaSparkContext#readFromDataset(String, Class, Class)}
 * This function requires that the hConf.xml file containing {@link Configuration} is present in the job jar.
 *///from  w w w  .j ava 2  s.c o  m
private Configuration loadHConf() {
    // TODO: Inject through Guice in Distributed mode, see CDAP-3
    Configuration hConf = new Configuration();
    hConf.clear();

    URL url = Thread.currentThread().getContextClassLoader()
            .getResource(SparkRuntimeService.SPARK_HCONF_FILENAME);
    if (url == null) {
        LOG.error("Unable to find Hadoop Configuration file {} in the submitted jar.",
                SparkRuntimeService.SPARK_HCONF_FILENAME);
        throw new RuntimeException(
                "Hadoop Configuration file not found in the supplied jar. Please include Hadoop "
                        + "Configuration file with name " + SparkRuntimeService.SPARK_HCONF_FILENAME);
    }
    hConf.addResource(url);
    return hConf;
}

---

From source file:co.cask.cdap.logging.run.LogSaverTwillRunnable.java

License:Apache License

@Override
public void initialize(TwillContext context) {
    super.initialize(context);

    completion = SettableFuture.create();
    name = context.getSpecification().getName();
    Map<String, String> configs = context.getSpecification().getConfigs();

    LOG.info("Initialize runnable: " + name);
    try {/*from w ww. j av a 2  s .c o  m*/
        // Load configuration
        Configuration hConf = new Configuration();
        hConf.clear();
        hConf.addResource(new File(configs.get("hConf")).toURI().toURL());

        UserGroupInformation.setConfiguration(hConf);

        CConfiguration cConf = CConfiguration.create(new File(configs.get("cConf")));

        cConf.set(Constants.LogSaver.ADDRESS, context.getHost().getCanonicalHostName());

        // Initialize ZK client
        String zookeeper = cConf.get(Constants.Zookeeper.QUORUM);
        if (zookeeper == null) {
            LOG.error("No ZooKeeper quorum provided.");
            throw new IllegalStateException("No ZooKeeper quorum provided.");
        }

        Injector injector = createGuiceInjector(cConf, hConf);
        zkClientService = injector.getInstance(ZKClientService.class);
        kafkaClientService = injector.getInstance(KafkaClientService.class);
        logSaverService = injector.getInstance(KafkaLogSaverService.class);

        int numPartitions = Integer.parseInt(
                cConf.get(LoggingConfiguration.NUM_PARTITIONS, LoggingConfiguration.DEFAULT_NUM_PARTITIONS));
        LOG.info("Num partitions = {}", numPartitions);

        logSaverStatusService = injector.getInstance(LogSaverStatusService.class);
        metricsCollectionService = injector.getInstance(MetricsCollectionService.class);
        LOG.info("Runnable initialized: " + name);
    } catch (Throwable t) {
        LOG.error(t.getMessage(), t);
        throw Throwables.propagate(t);
    }
}

---

From source file:co.cask.cdap.StandaloneMain.java

License:Apache License

public static StandaloneMain create(CConfiguration cConf, Configuration hConf) {
    // This is needed to use LocalJobRunner with fixes (we have it in app-fabric).
    // For the modified local job runner
    hConf.addResource("mapred-site-local.xml");
    hConf.reloadConfiguration();// w ww  . j  a va 2  s .c o  m
    // Due to incredibly stupid design of Limits class, once it is initialized, it keeps its settings. We
    // want to make sure it uses our settings in this hConf, so we have to force it initialize here before
    // someone else initializes it.
    Limits.init(hConf);

    File localDataDir = new File(cConf.get(Constants.CFG_LOCAL_DATA_DIR));
    hConf.set(Constants.CFG_LOCAL_DATA_DIR, localDataDir.getAbsolutePath());
    hConf.set(Constants.AppFabric.OUTPUT_DIR, cConf.get(Constants.AppFabric.OUTPUT_DIR));
    hConf.set("hadoop.tmp.dir",
            new File(localDataDir, cConf.get(Constants.AppFabric.TEMP_DIR)).getAbsolutePath());

    // Windows specific requirements
    if (OSDetector.isWindows()) {
        // not set anywhere by the project, expected to be set from IDEs if running from the project instead of sdk
        // hadoop.dll is at cdap-unit-test\src\main\resources\hadoop.dll for some reason
        String hadoopDLLPath = System.getProperty("hadoop.dll.path");
        if (hadoopDLLPath != null) {
            System.load(hadoopDLLPath);
        } else {
            // this is where it is when the standalone sdk is built
            String userDir = System.getProperty("user.dir");
            System.load(Joiner.on(File.separator).join(userDir, "lib", "native", "hadoop.dll"));
        }
    }

    //Run dataset service on random port
    List<Module> modules = createPersistentModules(cConf, hConf);

    return new StandaloneMain(modules, cConf);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »