Example usage for org.apache.hadoop.conf Configuration addResource

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration addResource.

Prototype

public void addResource(Configuration conf)

Source Link

Document

Add a configuration resource.

Usage

From source file:com.datatorrent.demos.mroperator.MapOperator.java

License:Open Source License

@Override
public void setup(OperatorContext context) {
    if (context != null) {
        operatorId = context.getId();/* www .  j a v a 2s. c om*/
    }
    reporter = new ReporterImpl(ReporterType.Mapper, new Counters());
    outputCollector = new OutputCollectorImpl<K2, V2>();
    Configuration conf = new Configuration();
    try {
        inputFormat = inputFormatClass.newInstance();
        SerializationFactory serializationFactory = new SerializationFactory(conf);
        Deserializer keyDesiralizer = serializationFactory.getDeserializer(inputSplitClass);
        keyDesiralizer.open(new ByteArrayInputStream(outstream.toByteArray()));
        inputSplit = (InputSplit) keyDesiralizer.deserialize(null);
        ((ReporterImpl) reporter).setInputSplit(inputSplit);
        reader = inputFormat.getRecordReader(inputSplit, new JobConf(conf), reporter);
    } catch (Exception e) {
        logger.info("failed to initialize inputformat obj {}", inputFormat);
        throw new RuntimeException(e);
    }
    InputStream stream = null;
    if (configFile != null && configFile.length() > 0) {
        stream = ClassLoader.getSystemResourceAsStream("/" + configFile);
        if (stream == null) {
            stream = ClassLoader.getSystemResourceAsStream(configFile);
        }
    }
    if (stream != null) {
        conf.addResource(stream);
    }
    jobConf = new JobConf(conf);
    if (mapClass != null) {
        try {
            mapObject = mapClass.newInstance();
        } catch (Exception e) {
            logger.info("can't instantiate object {}", e.getMessage());
        }

        mapObject.configure(jobConf);
    }
    if (combineClass != null) {
        try {
            combineObject = combineClass.newInstance();
        } catch (Exception e) {
            logger.info("can't instantiate object {}", e.getMessage());
        }
        combineObject.configure(jobConf);
    }
}

From source file:com.datatorrent.demos.mroperator.ReduceOperator.java

License:Open Source License

@Override
public void setup(OperatorContext context) {
    reporter = new ReporterImpl(ReporterType.Reducer, new Counters());
    if (context != null) {
        operatorId = context.getId();//  w w w . ja v  a  2 s.co m
    }
    cacheObject = new HashMap<K1, List<V1>>();
    outputCollector = new OutputCollectorImpl<K2, V2>();
    if (reduceClass != null) {
        try {
            reduceObj = reduceClass.newInstance();
        } catch (Exception e) {
            logger.info("can't instantiate object {}", e.getMessage());
            throw new RuntimeException(e);
        }
        Configuration conf = new Configuration();
        InputStream stream = null;
        if (configFile != null && configFile.length() > 0) {
            logger.info("system /{}", configFile);
            stream = ClassLoader.getSystemResourceAsStream("/" + configFile);
            if (stream == null) {
                logger.info("system {}", configFile);
                stream = ClassLoader.getSystemResourceAsStream(configFile);
            }
        }
        if (stream != null) {
            logger.info("found our stream... so adding it");
            conf.addResource(stream);
        }
        reduceObj.configure(new JobConf(conf));
    }

}

From source file:com.datatorrent.demos.pi.ApplicationTest.java

License:Open Source License

@Test
public void testSomeMethod() throws Exception {
    LocalMode lma = LocalMode.newInstance();
    Configuration conf = new Configuration(false);
    conf.addResource("dt-site-pi.xml");
    lma.prepareDAG(new Application(), conf);
    LocalMode.Controller lc = lma.getController();
    lc.run(10000);/*from  www.  j  av  a2  s  .c o m*/

}

From source file:com.datatorrent.demos.pi.CalculatorTest.java

License:Open Source License

@Test
public void testSomeMethod() throws Exception {
    LocalMode lma = LocalMode.newInstance();
    Configuration conf = new Configuration(false);
    conf.addResource("dt-site-pilibrary.xml");
    lma.prepareDAG(new Calculator(), conf);
    LocalMode.Controller lc = lma.getController();
    lc.run(10000);//from  w ww.  j  a va2 s. c o  m
}

From source file:com.datatorrent.demos.twitter.TwitterTopCounterWithFilterTest.java

License:Open Source License

/**
 * This test requires twitter authentication setup and is skipped by default
 * (see {@link TwitterSampleInput})./* w  w  w  .j a  v  a2 s.com*/
 *
 * @throws Exception
 */
@Test
public void testApplication() throws Exception {
    TwitterTopCounterWithFilterApplication app = new TwitterTopCounterWithFilterApplication();
    Configuration conf = new Configuration(false);
    conf.addResource("dt-site-topURLsWithFilter.xml");
    LocalMode lma = LocalMode.newInstance();
    lma.prepareDAG(app, conf);

    LocalMode.Controller lc = lma.getController();
    lc.run(120000);

}

From source file:com.datatorrent.demos.twitter.TwitterTopMediaTypesTest.java

License:Open Source License

/**
 * This test requires twitter authentication setup and is skipped by default
 * (see {@link TwitterSampleInput})./*from w w w . java  2 s . c o  m*/
 *
 * @throws Exception
 */
@Test
public void testApplication() throws Exception {
    TwitterTopMediaTypesApplication app = new TwitterTopMediaTypesApplication();
    Configuration conf = new Configuration(false);
    conf.addResource("dt-site-topMediaTypes.xml");
    LocalMode lma = LocalMode.newInstance();
    lma.prepareDAG(app, conf);
    LocalMode.Controller lc = lma.getController();
    lc.run(120000);
}

From source file:com.datatorrent.demos.twitter.TwitterTopUsersTest.java

License:Open Source License

/**
 * This test requires twitter authentication setup and is skipped by default
 * (see {@link TwitterSampleInput}).// w  ww  .ja  v a 2s . c  om
 *
 * @throws Exception
 */
@Test
public void testApplication() throws Exception {
    TwitterTopUsersApplication app = new TwitterTopUsersApplication();
    Configuration conf = new Configuration(false);
    conf.addResource("dt-site-topUsers.xml");
    LocalMode lma = LocalMode.newInstance();
    lma.prepareDAG(app, conf);
    LocalMode.Controller lc = lma.getController();
    lc.run(120000);
}

From source file:com.datatorrent.demos.twitter.TwitterTopWordsTest.java

License:Open Source License

/**
 * This test requires twitter authentication setup and is skipped by default
 * (see {@link TwitterSampleInput}).//  w  w  w .java  2  s  . c  o  m
 *
 * @throws Exception
 */
@Test
public void testApplication() throws Exception {
    TwitterTopWordsApplication app = new TwitterTopWordsApplication();
    Configuration conf = new Configuration(false);
    conf.addResource("dt-site-rollingtopwords.xml");
    LocalMode lma = LocalMode.newInstance();
    lma.prepareDAG(app, conf);
    LocalMode.Controller lc = lma.getController();
    lc.run(120000);
}

From source file:com.datatorrent.demos.yahoofinance.YahooFinanceChartApplicationTest.java

License:Open Source License

/**
 * This will run for ever./*w  w  w . j a  v  a 2  s .c om*/
 *
 * @throws Exception
 */
@Test
public void testApplication() throws Exception {
    LocalMode lma = LocalMode.newInstance();
    Configuration conf = new Configuration(false);
    conf.addResource("dt-site-chart.xml");
    lma.prepareDAG(new YahooFinanceApplicationWithChart(), conf);
    LocalMode.Controller lc = lma.getController();
    lc.run(60000);
}

From source file:com.datatorrent.lib.db.jdbc.JdbcInputOperatorApplicationTest.java

License:Apache License

public void testApplication(StreamingApplication streamingApplication) throws Exception {
    try {/*from w  w  w  . ja va2s .  c  o  m*/
        LocalMode lma = LocalMode.newInstance();
        Configuration conf = new Configuration(false);
        conf.addResource(this.getClass().getResourceAsStream("/JdbcProperties.xml"));
        lma.prepareDAG(streamingApplication, conf);
        LocalMode.Controller lc = lma.getController();
        lc.setHeartbeatMonitoringEnabled(false);
        ((StramLocalCluster) lc).setExitCondition(new Callable<Boolean>() {
            @Override
            public Boolean call() throws Exception {
                return TupleCount == 10;
            }
        });
        lc.run(10000);// runs for 10 seconds and quits
        Assert.assertEquals("rows in db", TupleCount, getNumOfRowsinTable(TABLE_POJO_NAME));
    } catch (ConstraintViolationException e) {
        Assert.fail("constraint violations: " + e.getConstraintViolations());
    }

}