Example usage for org.apache.hadoop.conf Configuration addResource

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration addResource.

Prototype

public void addResource(Configuration conf) 

Source Link

Document

Add a configuration resource.

Usage

From source file:org.apache.nutch.plugin.TestPluginSystem.java

License:Apache License

@Test
public void testRepositoryCache() {
    Configuration config = NutchConfiguration.create();
    PluginRepository repo = PluginRepository.get(config);
    JobConf job = new NutchJobConf(config);
    PluginRepository repo1 = PluginRepository.get(job);
    assertTrue(repo == repo1);/*from w  w w  .java  2 s .com*/
    // now construct a config without UUID
    config = new Configuration();
    config.addResource("nutch-default.xml");
    config.addResource("nutch-site.xml");
    repo = PluginRepository.get(config);
    job = new NutchJobConf(config);
    repo1 = PluginRepository.get(job);
    assertTrue(repo1 != repo);
}

---

From source file:org.apache.nutch.util.CrawlTestUtil.java

License:Apache License

/**
 * For now we need to manually construct our Configuration, because we need to
 * override the default one and it is currently not possible to use
 * dynamically set values./*from   ww  w.ja  v a2  s .  c o  m*/
 * 
 * @return
 */
public static Configuration createConfiguration() {
    Configuration conf = new Configuration();
    conf.addResource("nutch-default.xml");
    conf.addResource("nutch-site.xml");
    conf.addResource("crawl-tests.xml");
    return conf;
}

---

From source file:org.apache.oozie.action.hadoop.GitMain.java

License:Apache License

@SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", justification = "File created without user input")
private Configuration prepareActionConf() {
    final Configuration actionConf = new Configuration(false);

    final String actionXml = System.getProperty(OOZIE_ACTION_CONF);
    if (actionXml == null) {
        throw new RuntimeException("Missing Java System Property [" + OOZIE_ACTION_CONF + "]");
    }/*from www  . ja  va  2 s.  c  o  m*/
    if (!new File(actionXml).exists()) {
        throw new RuntimeException("Action Configuration XML file [" + actionXml + "] does not exist");
    }

    actionConf.addResource(new Path("file:///", actionXml));
    return actionConf;
}

---

From source file:org.apache.oozie.action.hadoop.Hive2Main.java

License:Apache License

private static Configuration initActionConf() {
    // Loading action conf prepared by Oozie
    Configuration actionConf = new Configuration(false);

    String actionXml = System.getProperty("oozie.action.conf.xml");

    if (actionXml == null) {
        throw new RuntimeException("Missing Java System Property [oozie.action.conf.xml]");
    }/*from w w w  .  j a va  2 s . c  o m*/
    if (!new File(actionXml).exists()) {
        throw new RuntimeException("Action Configuration XML file [" + actionXml + "] does not exist");
    } else {
        System.out.println("Using action configuration file " + actionXml);
    }

    actionConf.addResource(new Path("file:///", actionXml));
    setYarnTag(actionConf);

    // Propagate delegation related props from launcher job to Hive job
    String delegationToken = getFilePathFromEnv("HADOOP_TOKEN_FILE_LOCATION");
    if (delegationToken != null) {
        actionConf.set("mapreduce.job.credentials.binary", delegationToken);
        actionConf.set("tez.credentials.path", delegationToken);
        System.out.println("------------------------");
        System.out.println("Setting env property for mapreduce.job.credentials.binary to: " + delegationToken);
        System.out.println("------------------------");
        System.setProperty("mapreduce.job.credentials.binary", delegationToken);
    } else {
        System.out.println("Non-Kerberos execution");
    }

    // See https://issues.apache.org/jira/browse/HIVE-1411
    actionConf.set("datanucleus.plugin.pluginRegistryBundleCheck", "LOG");

    return actionConf;
}

---

From source file:org.apache.oozie.action.hadoop.HiveMain.java

License:Apache License

private static Configuration initActionConf() throws java.io.IOException {
    // Loading action conf prepared by Oozie
    Configuration hiveConf = new Configuration(false);

    String actionXml = System.getProperty("oozie.action.conf.xml");

    if (actionXml == null) {
        throw new RuntimeException("Missing Java System Property [oozie.action.conf.xml]");
    }//from w ww .  j a  v a 2s.  co  m
    if (!new File(actionXml).exists()) {
        throw new RuntimeException("Action Configuration XML file [" + actionXml + "] does not exist");
    } else {
        System.out.println("Using action configuration file " + actionXml);
    }

    hiveConf.addResource(new Path("file:///", actionXml));

    setYarnTag(hiveConf);

    // Propagate delegation related props from launcher job to Hive job
    String delegationToken = getFilePathFromEnv("HADOOP_TOKEN_FILE_LOCATION");
    if (delegationToken != null) {
        hiveConf.set("mapreduce.job.credentials.binary", delegationToken);
        hiveConf.set("tez.credentials.path", delegationToken);
        System.out.println("------------------------");
        System.out.println("Setting env property for mapreduce.job.credentials.binary to: " + delegationToken);
        System.out.println("------------------------");
        System.setProperty("mapreduce.job.credentials.binary", delegationToken);
        System.out.println("------------------------");
        System.out.println("Setting env property for tez.credentials.path to: " + delegationToken);
        System.out.println("------------------------");
        System.setProperty("tez.credentials.path", delegationToken);
    } else {
        System.out.println("Non-Kerberos execution");
    }

    // Have to explicitly unset this property or Hive will not set it.
    // Reset if Oozie-defaults are set. Otherwise, honour user-selected job-name.
    if (hiveConf.get("mapred.job.name", "").startsWith("oozie:action:")) {
        hiveConf.set("mapred.job.name", "");
    }

    // See https://issues.apache.org/jira/browse/HIVE-1411
    hiveConf.set("datanucleus.plugin.pluginRegistryBundleCheck", "LOG");

    // to force hive to use the jobclient to submit the job, never using HADOOPBIN (to do localmode)
    hiveConf.setBoolean("hive.exec.mode.local.auto", false);

    String pwd = new java.io.File("").getCanonicalPath();
    hiveConf.set("hive.querylog.location", pwd + File.separator + "hivetmp" + File.separator + "querylog");
    hiveConf.set("hive.exec.local.scratchdir",
            pwd + File.separator + "hivetmp" + File.separator + "scratchdir");

    return hiveConf;
}

---

From source file:org.apache.oozie.action.hadoop.LauncherAM.java

License:Apache License

private void executePrepare(ErrorHolder errorHolder) throws Exception {
    try {//from ww w  . j  a v a 2  s  .c o m
        System.out.println("\nStarting the execution of prepare actions");
        String prepareXML = launcherConf.get(ACTION_PREPARE_XML);
        if (prepareXML != null) {
            if (prepareXML.length() != 0) {
                Configuration actionConf = new Configuration(launcherConf);
                actionConf.addResource(ACTION_CONF_XML);
                prepareHandler.prepareAction(prepareXML, actionConf);
            } else {
                System.out.println("There are no prepare actions to execute.");
            }
        }
        System.out.println("Completed the execution of prepare actions successfully");
    } catch (Exception ex) {
        errorHolder.setErrorMessage("Prepare execution in the Launcher AM has failed");
        errorHolder.setErrorCause(ex);
        throw ex;
    }
}

---

From source file:org.apache.oozie.action.hadoop.LauncherMain.java

License:Apache License

/**
 * Read action configuration passes through action xml file.
 *
 * @return action  Configuration/*from   w ww  .  java 2  s. c o  m*/
 * @throws IOException
 */
public static Configuration loadActionConf() throws IOException {
    // loading action conf prepared by Oozie
    Configuration actionConf = new Configuration(false);

    String actionXml = System.getProperty("oozie.action.conf.xml");

    if (actionXml == null) {
        throw new RuntimeException("Missing Java System Property [oozie.action.conf.xml]");
    }
    if (!new File(actionXml).exists()) {
        throw new RuntimeException("Action Configuration XML file [" + actionXml + "] does not exist");
    }

    actionConf.addResource(new Path("file:///", actionXml));
    return actionConf;
}

---

From source file:org.apache.oozie.action.hadoop.LauncherMapper.java

License:Apache License

private void executePrepare() throws IOException, LauncherException {
    String prepareXML = getJobConf().get(ACTION_PREPARE_XML);
    if (prepareXML != null) {
        if (!prepareXML.equals("")) {
            Configuration actionConf = new Configuration(getJobConf());
            String actionXml = System.getProperty("oozie.action.conf.xml");
            actionConf.addResource(new Path("file:///", actionXml));
            PrepareActionsDriver.doOperations(prepareXML, actionConf);
        } else {/*w  w w.j  a  v  a  2  s . c  o m*/
            System.out.println("There are no prepare actions to execute.");
        }
    }
}

---

From source file:org.apache.oozie.action.hadoop.LocalFsOperations.java

License:Apache License

/**
 * Reads the launcher configuration "launcher.xml"
 * @return Configuration object//from w ww . ja va 2 s.c om
 */
public Configuration readLauncherConf() {
    File confFile = new File(LauncherAM.LAUNCHER_JOB_CONF_XML);
    Configuration conf = new Configuration(false);
    conf.addResource(new org.apache.hadoop.fs.Path(confFile.getAbsolutePath()));
    return conf;
}

---

From source file:org.apache.oozie.action.hadoop.MapReduceMain.java

License:Apache License

protected void run(String[] args) throws Exception {
    System.out.println();/*from w w  w .j  a  va 2s. c o m*/
    System.out.println("Oozie Map-Reduce action configuration");
    System.out.println("=======================");

    // loading action conf prepared by Oozie
    Configuration actionConf = new Configuration(false);
    actionConf.addResource(new Path("file:///", System.getProperty("oozie.action.conf.xml")));
    setYarnTag(actionConf);

    JobConf jobConf = new JobConf();
    addActionConf(jobConf, actionConf);
    LauncherMainHadoopUtils.killChildYarnJobs(jobConf);

    // Run a config class if given to update the job conf
    runConfigClass(jobConf);

    logMasking("Map-Reduce job configuration:", new HashSet<String>(), jobConf);

    File idFile = new File(
            System.getProperty(LauncherMapper.ACTION_PREFIX + LauncherMapper.ACTION_DATA_NEW_ID));
    System.out.println("Submitting Oozie action Map-Reduce job");
    System.out.println();
    // submitting job
    RunningJob runningJob = submitJob(jobConf);

    String jobId = runningJob.getID().toString();
    writeJobIdFile(idFile, jobId);

    System.out.println("=======================");
    System.out.println();
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »