Example usage for org.apache.hadoop.conf Configuration addResource

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration addResource.

Prototype

public void addResource(Configuration conf) 

Source Link

Document

Add a configuration resource.

Usage

From source file:com.twitter.ambrose.model.hadoop.MapReduceHelper.java

License:Apache License

/**
 * Get the configurations at the beginning of the job flow, it will contain information
 * about the map/reduce plan and decoded pig script.
 * @param runningJob/*from   ww  w. j  ava  2s . c  om*/
 * @return Properties - configuration properties of the job
 */
public void setJobConfFromFile(MapReduceJob job, JobClient jobClient) {
    Properties jobConfProperties = new Properties();
    try {
        RunningJob runningJob = jobClient.getJob(JobID.forName(job.getId()));
        if (runningJob == null) {
            log.warn("Couldn't find job status for jobId: " + job.getId());
        }

        log.info("RunningJob Configuration File location: " + runningJob.getJobFile());
        Path path = new Path(runningJob.getJobFile());

        Configuration conf = new Configuration(false);
        FileSystem fileSystem = FileSystem.get(new Configuration());
        InputStream inputStream = fileSystem.open(path);
        conf.addResource(inputStream);

        Iterator<Map.Entry<String, String>> iter = conf.iterator();
        while (iter.hasNext()) {
            Map.Entry<String, String> entry = iter.next();
            jobConfProperties.put(entry.getKey(), entry.getValue());
        }
    } catch (FileNotFoundException e) {
        log.warn("Configuration file not found for old jobsflows.");
    } catch (Exception e) {
        log.warn("Error occurred when retrieving configuration info." + e.getMessage());
    }
    job.setConfiguration(jobConfProperties);
}

---

From source file:com.twitter.ambrose.service.impl.hraven.HRavenStatsWriteService.java

License:Apache License

static Configuration initHBaseConfiguration(Configuration jobConf) {
    // setup hraven hbase connection information
    Configuration hbaseConf = null;
    String hravenDir = System.getenv(HRAVEN_HBASE_CONF_DIR_ENV);
    if (hravenDir != null && !hravenDir.isEmpty()) {
        hbaseConf = new Configuration(false);
        Path hbaseConfFile = new Path(hravenDir, HRAVEN_HBASE_CONF_FILE);
        LOG.info("Loading hRaven HBase configuration from " + hbaseConfFile.toString());
        hbaseConf.addResource(hbaseConfFile);
    } else {//  w w w.  j  a  v  a  2s  .co m
        hbaseConf = HBaseConfiguration.create();
        // override any hRaven connection properties from the job
        String hbaseZKQuorum = jobConf.get(HRAVEN_ZOOKEEPER_QUORUM);
        if (hbaseZKQuorum != null && !hbaseZKQuorum.isEmpty()) {
            try {
                LOG.info("Applying hRaven Zookeeper quorum information from job conf: " + hbaseZKQuorum);
                ZKUtil.applyClusterKeyToConf(hbaseConf, hbaseZKQuorum);
            } catch (IOException ioe) {
                throw new RuntimeException("Invalid cluster configuration for " + HRAVEN_ZOOKEEPER_QUORUM
                        + " (\"" + hbaseZKQuorum + "\")");
            }
        } else if (LOG.isDebugEnabled()) {
            LOG.debug("No cluster configuration found for " + HRAVEN_ZOOKEEPER_QUORUM
                    + ", continuing with default");
        }
    }

    return hbaseConf;
}

---

From source file:com.twitter.elephanttwin.util.HdfsUtils.java

License:Apache License

public static Configuration getHdfsConfiguration(String hdfsConfigPath) throws IOException {
    Configuration conf = new Configuration(true);
    conf.addResource(new Path(hdfsConfigPath));
    conf.reloadConfiguration();/*w ww .j av  a2s.  co m*/
    return conf;
}

---

From source file:com.twitter.hraven.datasource.JobHistoryRawService.java

License:Apache License

/**
 * @param result//from  ww  w .j  a v a 2s  .c  o m
 *          from the {@link Scan} from
 *          {@link #getHistoryRawTableScan(String, String, String, boolean, boolean, boolean)}
 * @return the configuration part.
 * @throws MissingColumnInResultException
 *           when the result does not contain {@link Constants#RAW_FAM},
 *           {@link Constants#JOBCONF_COL}.
 */
public Configuration createConfigurationFromResult(Result result) throws MissingColumnInResultException {

    if (result == null) {
        throw new IllegalArgumentException("Cannot create InputStream from null");
    }

    KeyValue keyValue = result.getColumnLatest(Constants.RAW_FAM_BYTES, Constants.JOBCONF_COL_BYTES);

    // Create a jobConf from the raw input
    Configuration jobConf = new Configuration(false);

    byte[] jobConfRawBytes = null;
    if (keyValue != null) {
        jobConfRawBytes = keyValue.getValue();
    }
    if (jobConfRawBytes == null || jobConfRawBytes.length == 0) {
        throw new MissingColumnInResultException(Constants.RAW_FAM_BYTES, Constants.JOBCONF_COL_BYTES);
    }

    InputStream in = new ByteArrayInputStream(jobConfRawBytes);
    jobConf.addResource(in);

    // Configuration property loading is lazy, so we need to force a load from the input stream
    try {
        int size = jobConf.size();
        if (LOG.isDebugEnabled()) {
            LOG.info("Loaded " + size + " job configuration properties from result");
        }
    } catch (Exception e) {
        throw new ProcessingException(
                "Invalid configuration from result " + Bytes.toStringBinary(result.getRow()), e);
    }

    return jobConf;
}

---

From source file:com.twitter.hraven.datasource.TestJobHistoryService.java

License:Apache License

@Test
public void testSetHravenQueueName() throws FileNotFoundException {

    final String JOB_CONF_FILE_NAME = "src/test/resources/job_1329348432655_0001_conf.xml";

    Configuration jobConf = new Configuration();
    jobConf.addResource(new FileInputStream(JOB_CONF_FILE_NAME));

    String USERNAME = "user";
    JobKey jobKey = new JobKey("cluster1", USERNAME, "Sleep", 1, "job_1329348432655_0001");
    byte[] jobKeyBytes = new JobKeyConverter().toBytes(jobKey);
    Put jobPut = new Put(jobKeyBytes);
    byte[] jobConfColumnPrefix = Bytes.toBytes(Constants.JOB_CONF_COLUMN_PREFIX + Constants.SEP);

    assertEquals(jobPut.size(), 0);/*from   www .  java2 s. c o m*/

    // check queuename matches user name since the conf has
    // value "default" as the queuename
    JobHistoryService.setHravenQueueNamePut(jobConf, jobPut, jobKey, jobConfColumnPrefix);
    assertEquals(jobPut.size(), 1);
    byte[] column = Bytes.add(jobConfColumnPrefix, Constants.HRAVEN_QUEUE_BYTES);
    assertFoundOnce(column, jobPut, 1, USERNAME);

    // populate the jobConf with all types of queue name parameters
    String expH2QName = "hadoop2queue";
    String expH1PoolName = "fairpool";
    String capacityH1QName = "capacity1aueue";
    jobConf.set(Constants.QUEUENAME_HADOOP2, expH2QName);
    jobConf.set(Constants.FAIR_SCHEDULER_POOLNAME_HADOOP1, expH1PoolName);
    jobConf.set(Constants.CAPACITY_SCHEDULER_QUEUENAME_HADOOP1, capacityH1QName);

    // now check queuename is correctly set as hadoop2 queue name
    // even when the fairscheduler and capacity scheduler are set
    jobPut = new Put(jobKeyBytes);
    assertEquals(jobPut.size(), 0);
    JobHistoryService.setHravenQueueNamePut(jobConf, jobPut, jobKey, jobConfColumnPrefix);
    assertEquals(jobPut.size(), 1);
    assertFoundOnce(column, jobPut, 1, expH2QName);

    // now unset hadoop2 queuename, expect fairscheduler name to be used as queuename
    jobConf.set(Constants.QUEUENAME_HADOOP2, "");
    jobPut = new Put(jobKeyBytes);
    assertEquals(jobPut.size(), 0);
    JobHistoryService.setHravenQueueNamePut(jobConf, jobPut, jobKey, jobConfColumnPrefix);
    assertEquals(jobPut.size(), 1);
    assertFoundOnce(column, jobPut, 1, expH1PoolName);

    // now unset fairscheduler name, expect capacity scheduler to be used as queuename
    jobConf.set(Constants.FAIR_SCHEDULER_POOLNAME_HADOOP1, "");
    jobPut = new Put(jobKeyBytes);
    assertEquals(jobPut.size(), 0);
    JobHistoryService.setHravenQueueNamePut(jobConf, jobPut, jobKey, jobConfColumnPrefix);
    assertEquals(jobPut.size(), 1);
    assertFoundOnce(column, jobPut, 1, capacityH1QName);

    // now unset capacity scheduler, expect default_queue to be used as queuename
    jobConf.set(Constants.CAPACITY_SCHEDULER_QUEUENAME_HADOOP1, "");
    jobPut = new Put(jobKeyBytes);
    assertEquals(jobPut.size(), 0);
    JobHistoryService.setHravenQueueNamePut(jobConf, jobPut, jobKey, jobConfColumnPrefix);
    assertEquals(jobPut.size(), 1);
    assertFoundOnce(column, jobPut, 1, Constants.DEFAULT_QUEUENAME);

}

---

From source file:com.twitter.hraven.etl.TestJobHistoryFileParserHadoop1.java

License:Apache License

@Test
public void testMegaByteMillis() throws IOException {

    final String JOB_HISTORY_FILE_NAME = "src/test/resources/job_201311192236_3583_1386370578196_user1_Sleep+job";

    File jobHistoryfile = new File(JOB_HISTORY_FILE_NAME);
    byte[] contents = Files.toByteArray(jobHistoryfile);
    final String JOB_CONF_FILENAME = "src/test/resources/job_1329348432655_0001_conf.xml";
    Configuration jobConf = new Configuration();
    jobConf.addResource(new Path(JOB_CONF_FILENAME));

    JobHistoryFileParser historyFileParser = JobHistoryFileParserFactory.createJobHistoryFileParser(contents,
            jobConf);/*ww w. j av  a 2 s .c  om*/
    assertNotNull(historyFileParser);

    // confirm that we get back an object that can parse hadoop 1.0 files
    assertTrue(historyFileParser instanceof JobHistoryFileParserHadoop1);

    JobKey jobKey = new JobKey("cluster1", "user1", "Sleep", 1, "job_201311192236_3583");
    historyFileParser.parse(contents, jobKey);

    List<Put> jobPuts = historyFileParser.getJobPuts();
    assertNotNull(jobPuts);

    Long mbMillis = historyFileParser.getMegaByteMillis();
    Long expValue = 2981062L;
    assertEquals(expValue, mbMillis);
}

---

From source file:com.twitter.hraven.etl.TestJobHistoryFileParserHadoop2.java

License:Apache License

@Test
public void testCreateJobHistoryFileParserCorrectCreation() throws IOException {

    final String JOB_HISTORY_FILE_NAME = "src/test/resources/job_1329348432655_0001-1329348443227-user-Sleep+job-1329348468601-10-1-SUCCEEDED-default.jhist";

    File jobHistoryfile = new File(JOB_HISTORY_FILE_NAME);
    byte[] contents = Files.toByteArray(jobHistoryfile);
    // now load the conf file and check
    final String JOB_CONF_FILE_NAME = "src/test/resources/job_1329348432655_0001_conf.xml";
    Configuration jobConf = new Configuration();
    jobConf.addResource(new Path(JOB_CONF_FILE_NAME));

    JobHistoryFileParser historyFileParser = JobHistoryFileParserFactory.createJobHistoryFileParser(contents,
            jobConf);/*from   ww w. j  av a2s.c o  m*/
    assertNotNull(historyFileParser);

    // confirm that we get back an object that can parse hadoop 2.0 files
    assertTrue(historyFileParser instanceof JobHistoryFileParserHadoop2);

    JobKey jobKey = new JobKey("cluster1", "user", "Sleep", 1, "job_1329348432655_0001");
    historyFileParser.parse(contents, jobKey);

    List<Put> jobPuts = historyFileParser.getJobPuts();
    assertEquals(6, jobPuts.size());

    JobKeyConverter jobKeyConv = new JobKeyConverter();
    assertEquals("cluster1!user!Sleep!1!job_1329348432655_0001",
            jobKeyConv.fromBytes(jobPuts.get(0).getRow()).toString());

    // check hadoop version
    boolean foundVersion2 = false;
    for (Put p : jobPuts) {
        List<KeyValue> kv2 = p.get(Constants.INFO_FAM_BYTES,
                Bytes.toBytes(JobHistoryKeys.hadoopversion.toString()));
        if (kv2.size() == 0) {
            // we are interested in hadoop version put only
            // hence continue
            continue;
        }
        assertEquals(1, kv2.size());
        Map<byte[], List<KeyValue>> d = p.getFamilyMap();
        for (List<KeyValue> lkv : d.values()) {
            for (KeyValue kv : lkv) {
                // ensure we have a hadoop2 version as the value
                assertEquals(Bytes.toString(kv.getValue()), HadoopVersion.TWO.toString());

                // ensure we don't see the same put twice
                assertFalse(foundVersion2);
                // now set this to true
                foundVersion2 = true;
            }
        }
    }
    // ensure that we got the hadoop2 version put
    assertTrue(foundVersion2);

    // check job status
    boolean foundJobStatus = false;
    for (Put p : jobPuts) {
        List<KeyValue> kv2 = p.get(Constants.INFO_FAM_BYTES,
                Bytes.toBytes(JobHistoryKeys.JOB_STATUS.toString().toLowerCase()));
        if (kv2.size() == 0) {
            // we are interested in JobStatus put only
            // hence continue
            continue;
        }
        assertEquals(1, kv2.size());

        for (KeyValue kv : kv2) {
            // ensure we have a job status value as the value
            assertEquals(Bytes.toString(kv.getValue()), JobHistoryFileParserHadoop2.JOB_STATUS_SUCCEEDED);

            // ensure we don't see the same put twice
            assertFalse(foundJobStatus);
            // now set this to true
            foundJobStatus = true;
        }
    }
    // ensure that we got the JobStatus put
    assertTrue(foundJobStatus);

    List<Put> taskPuts = historyFileParser.getTaskPuts();
    assertEquals(taskPuts.size(), 45);

    TaskKeyConverter taskKeyConv = new TaskKeyConverter();

    Set<String> putRowKeys = new HashSet<String>(Arrays.asList(
            "cluster1!user!Sleep!1!job_1329348432655_0001!AM_appattempt_1329348432655_0001_000001",
            "cluster1!user!Sleep!1!job_1329348432655_0001!r_000000",
            "cluster1!user!Sleep!1!job_1329348432655_0001!r_000000_0",
            "cluster1!user!Sleep!1!job_1329348432655_0001!m_000000",
            "cluster1!user!Sleep!1!job_1329348432655_0001!m_000000_0",
            "cluster1!user!Sleep!1!job_1329348432655_0001!m_000009",
            "cluster1!user!Sleep!1!job_1329348432655_0001!m_000009_0",
            "cluster1!user!Sleep!1!job_1329348432655_0001!m_000008",
            "cluster1!user!Sleep!1!job_1329348432655_0001!m_000008_0",
            "cluster1!user!Sleep!1!job_1329348432655_0001!m_000007",
            "cluster1!user!Sleep!1!job_1329348432655_0001!m_000007_0",
            "cluster1!user!Sleep!1!job_1329348432655_0001!m_000006",
            "cluster1!user!Sleep!1!job_1329348432655_0001!m_000006_0",
            "cluster1!user!Sleep!1!job_1329348432655_0001!m_000005",
            "cluster1!user!Sleep!1!job_1329348432655_0001!m_000005_0",
            "cluster1!user!Sleep!1!job_1329348432655_0001!m_000004",
            "cluster1!user!Sleep!1!job_1329348432655_0001!m_000004_0",
            "cluster1!user!Sleep!1!job_1329348432655_0001!m_000003",
            "cluster1!user!Sleep!1!job_1329348432655_0001!m_000003_0",
            "cluster1!user!Sleep!1!job_1329348432655_0001!m_000002",
            "cluster1!user!Sleep!1!job_1329348432655_0001!m_000002_0",
            "cluster1!user!Sleep!1!job_1329348432655_0001!m_000001",
            "cluster1!user!Sleep!1!job_1329348432655_0001!m_000001_0"));

    String tKey;
    for (Put p : taskPuts) {
        tKey = taskKeyConv.fromBytes(p.getRow()).toString();
        assertTrue(putRowKeys.contains(tKey));
    }

    // check post processing for megabytemillis
    // first with empty job conf
    Long mbMillis = historyFileParser.getMegaByteMillis();
    assertNotNull(mbMillis);
    Long expValue = 10390016L;
    assertEquals(expValue, mbMillis);
}

---

From source file:com.twitter.hraven.etl.TestJobHistoryFileParserHadoop2.java

License:Apache License

@Test
public void testMissingSlotMillis() throws IOException {
    final String JOB_HISTORY_FILE_NAME = "src/test/resources/job_1329348432999_0003-1329348443227-user-Sleep+job-1329348468601-10-1-SUCCEEDED-default.jhist";

    File jobHistoryfile = new File(JOB_HISTORY_FILE_NAME);
    byte[] contents = Files.toByteArray(jobHistoryfile);
    final String JOB_CONF_FILE_NAME = "src/test/resources/job_1329348432655_0001_conf.xml";
    Configuration jobConf = new Configuration();
    jobConf.addResource(new Path(JOB_CONF_FILE_NAME));

    JobHistoryFileParserHadoop2 historyFileParser = new JobHistoryFileParserHadoop2(jobConf);
    assertNotNull(historyFileParser);//from  ww  w .j  a  va 2  s.c om

    JobKey jobKey = new JobKey("cluster1", "user", "Sleep", 1, "job_1329348432655_0001");
    historyFileParser.parse(contents, jobKey);

    // this history file has only map slot millis no reduce millis
    Long mapMbMillis = historyFileParser.getMapMbMillis();
    assertNotNull(mapMbMillis);
    assertEquals(mapMbMillis, new Long(178169856L));
    Long reduceMbMillis = historyFileParser.getReduceMbMillis();
    assertNotNull(reduceMbMillis);
    assertEquals(reduceMbMillis, Constants.NOTFOUND_VALUE);

    Long mbMillis = historyFileParser.getMegaByteMillis();
    assertNotNull(mbMillis);
    Long expValue = 188559872L;
    assertEquals(expValue, mbMillis);
}

---

From source file:com.twitter.hraven.etl.TestJobHistoryFileParserHadoop2.java

License:Apache License

@Test
public void testVersion2_4() throws IOException {
    final String JOB_HISTORY_FILE_NAME = "src/test/resources/"
            + "job_1410289045532_259974-1411647985641-user35-SomeJobName-1411647999554-1-0-SUCCEEDED-root.someQueueName-1411647995323.jhist";
    File jobHistoryfile = new File(JOB_HISTORY_FILE_NAME);
    byte[] contents = Files.toByteArray(jobHistoryfile);
    final String JOB_CONF_FILE_NAME = "src/test/resources/job_1329348432655_0001_conf.xml";
    Configuration jobConf = new Configuration();
    jobConf.addResource(new Path(JOB_CONF_FILE_NAME));

    JobHistoryFileParser historyFileParser = JobHistoryFileParserFactory.createJobHistoryFileParser(contents,
            jobConf);/*w w  w .  ja  v  a 2 s  .c om*/
    assertNotNull(historyFileParser);

    // confirm that we get back an object that can parse hadoop 2.x files
    assertTrue(historyFileParser instanceof JobHistoryFileParserHadoop2);
    JobKey jobKey = new JobKey("cluster1", "user", "Sleep", 1, "job_1329348432655_0001");
    historyFileParser.parse(contents, jobKey);

    List<Put> jobPuts = historyFileParser.getJobPuts();

    // now confirm that 2.4 constructs have been parsed
    // check that queue name is found, since
    // it's part of the JobQueueChange object in the history file
    boolean foundQueue = false;
    for (Put p : jobPuts) {
        List<KeyValue> kv2 = p.get(Constants.INFO_FAM_BYTES,
                Bytes.toBytes(JobHistoryKeys.JOB_QUEUE.toString().toLowerCase()));
        if (kv2.size() == 0) {
            // we are interested in queue put only
            // hence continue
            continue;
        } else {
            for (KeyValue kv : kv2) {
                // ensure we have a hadoop2 version as the value
                assertEquals(Bytes.toString(kv.getValue()), "root.someQueueName");
                // ensure we don't see the same put twice
                assertFalse(foundQueue);
                // now set this to true
                foundQueue = true;
            }
            break;
        }

    }
    // ensure that we got the hadoop2 version put
    assertTrue(foundQueue);
}

---

From source file:com.twitter.hraven.hadoopJobMonitor.AppStatusChecker.java

License:Apache License

private AppConfiguraiton getAppConf(ApplicationReport appReport) throws ConfigurationAccessException {
    ApplicationId appId = appReport.getApplicationId();
    AppConfiguraiton appConf = appConfCache.get(appId);
    if (appConf != null)
        return appConf;
    String xmlUrlStr = buildXmlUrl(appReport);
    URL xmlUrl;/* w  w  w.j av a2 s .  c om*/
    try {
        xmlUrl = new URL(xmlUrlStr);
        Configuration origAppConf = new Configuration(false);
        origAppConf.addResource(xmlUrl);
        appConf = new AppConfiguraiton(origAppConf, vConf);
        appConfCache.put(appId, appConf);
    } catch (MalformedURLException e) {
        LOG.warn(e);
        throw new AppConfiguraiton.ConfigurationAccessException(e);
    }
    return appConf;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »