Example usage for org.apache.hadoop.conf Configuration addResource

List of usage examples for org.apache.hadoop.conf Configuration addResource

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.conf.*
  5. Configuration
  6. addResource

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration addResource.

Prototype

public void addResource(Configuration conf)

Source Link

Document

Add a configuration resource.

Usage

From source file:com.emc.greenplum.gpdb.hdfsconnector.TestConnectorUtil.java

License:Open Source License

@Test
public void test_unicode() {
    int hadoopPort = cluster.getNameNodePort();

    Configuration conf = new Configuration();
    conf.addResource("hdfs-site.xml");

    try {//from w w  w  . ja  v  a2s. c om
        URI uri = new URI("gphdfs://localhost:" + hadoopPort + "/tmp/alertlog.avro");
        ConnectorUtil.setHadoopFSURI(conf, uri, "gphdfs");

        ByteArrayOutputStream bout = new ByteArrayOutputStream();
        AvroFileReader aReader = new AvroFileReader(conf, 0, 1, uri.getPath(), null, null, false, false, bout);

        aReader.readAvroFormat();

        byte[] barray = bout.toByteArray();

        int line = 0;
        DataInputStream din = new DataInputStream(new ByteArrayInputStream(barray));
        while (din.available() != 0) {
            GPDBWritable writable = new GPDBWritable();
            writable.readFields(din);
            line++;
        }

        assertEquals(line, 1943);
    } catch (IOException e) {
        fail(e.getMessage());
    } catch (URISyntaxException e) {
        fail(e.getMessage());
    }
}

From source file:com.emc.greenplum.gpdb.hdfsconnector.TestConnectorUtil.java

License:Open Source License

@Test
public void test_parquet_short() {
    int hadoopPort = cluster.getNameNodePort();

    Configuration conf = new Configuration();
    conf.addResource("hdfs-site.xml");

    try {// w  ww  . j a  v a  2  s  . c  o m
        URI uri = new URI("gphdfs://localhost:" + hadoopPort + "/tmp/short.parquet");
        ConnectorUtil.setHadoopFSURI(conf, uri, "gphdfs");

        ByteArrayOutputStream bout = new ByteArrayOutputStream();
        List<ColumnSchema> columns = new ArrayList<ColumnSchema>();
        columns.add(new ColumnSchema("c1", GPDBWritable.SMALLINT, 1, 0, ','));
        columns.add(new ColumnSchema("c2", GPDBWritable.SMALLINT, 1, 0, ','));
        columns.add(new ColumnSchema("c2", GPDBWritable.BIGINT, 1, 0, ','));
        GpdbParquetFileReader pReader = new GpdbParquetFileReader(conf, 0, 1, uri.getPath(), columns, false,
                false, bout);
        pReader.readParquetFormat();

        byte[] barray = bout.toByteArray();
        DataInputStream din = new DataInputStream(new ByteArrayInputStream(barray));
        GPDBWritable writable = new GPDBWritable();
        writable.readFields(din);
        short c1 = writable.getShort(0);
        assertEquals(c1, 123);
        short c2 = writable.getShort(1);
        assertEquals(c2, 1234);
        long c3 = writable.getLong(2);
        assertEquals(c3, 12345);

        assertEquals(din.available(), 0);
    } catch (IOException e) {
        fail(e.getMessage());
    } catch (URISyntaxException e) {
        fail(e.getMessage());
    }
}

From source file:com.emc.greenplum.gpdb.hdfsconnector.TestConnectorUtil.java

License:Open Source License

@Test
public void test_avro_short() {
    int hadoopPort = cluster.getNameNodePort();

    Configuration conf = new Configuration();
    conf.addResource("hdfs-site.xml");

    try {/*from   w  ww  .ja  v a2s. c om*/
        URI uri = new URI("gphdfs://localhost:" + hadoopPort + "/tmp/short.avro");
        ConnectorUtil.setHadoopFSURI(conf, uri, "gphdfs");

        ByteArrayOutputStream bout = new ByteArrayOutputStream();
        List<ColumnSchema> columns = new ArrayList<ColumnSchema>();
        columns.add(new ColumnSchema("c1", GPDBWritable.SMALLINT, 1, 1, ','));
        AvroFileReader aReader = new AvroFileReader(conf, 0, 1, uri.getPath(), columns, null, false, false,
                bout);

        aReader.readAvroFormat();

        byte[] barray = bout.toByteArray();
        DataInputStream din = new DataInputStream(new ByteArrayInputStream(barray));
        GPDBWritable writable = new GPDBWritable();
        writable.readFields(din);
        short c1 = writable.getShort(0);
        assertEquals(c1, 123);

        assertEquals(din.available(), 0);
    } catch (IOException e) {
        fail(e.getMessage());
    } catch (URISyntaxException e) {
        fail(e.getMessage());
    }
}

From source file:com.example.dedup.ApplicationTest.java

@Test
public void testApplication() throws IOException, Exception {
    try {//from   w  ww  .  j  ava 2s.com
        LocalMode lma = LocalMode.newInstance();
        Configuration conf = new Configuration(false);
        conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
        lma.prepareDAG(new Application(), conf);
        LocalMode.Controller lc = lma.getController();
        lc.runAsync();
        Thread.sleep(10 * 1000);
        lc.shutdown();
    } catch (ConstraintViolationException e) {
        Assert.fail("constraint violations: " + e.getConstraintViolations());
    }
}

From source file:com.example.droosapps.ApplicationTest.java

public void testApplication() throws IOException, Exception {
    try {/*from w ww .  j a  v  a  2 s.  c  o m*/
        LocalMode lma = LocalMode.newInstance();
        Configuration conf = new Configuration(false);
        conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
        lma.prepareDAG(new Application(), conf);
        LocalMode.Controller lc = lma.getController();
        lc.run(10000); // runs for 10 seconds and quits
    } catch (ConstraintViolationException e) {
        //      Assert.fail("constraint violations: " + e.getConstraintViolations());
    }
}

From source file:com.example.hdht.sample.ApplicationTest.java

@Test
public void testApplication() throws IOException, Exception {
    try {/*from  ww w .  j  ava 2  s  .  c  o  m*/
        LocalMode lma = LocalMode.newInstance();
        Configuration conf = new Configuration(false);
        conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
        lma.prepareDAG(new QueryHDHTApplication(), conf);
        LocalMode.Controller lc = lma.getController();
        lc.run(10000); // runs for 10 seconds and quits
    } catch (ConstraintViolationException e) {
        Assert.fail("constraint violations: " + e.getConstraintViolations());
    }
}

From source file:com.example.jmsSqs.ApplicationTest.java

private Configuration getConfig() {
    Configuration conf = new Configuration(false);

    // read config values from the properties.xml file
    conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));

    // one can also set or override values in code as below
    String pre = "dt.operator.fileOut.prop.";
    conf.set(pre + "filePath", FILE_DIR);

    pre = "dt.operator.sqsIn.prop.";
    // set the subject here since it is dynamically generated
    conf.set(pre + "subject", currentQueueName);
    // set the endpoint to point to our mock server
    conf.set(pre + "aws.endpoint", "http://localhost:9324");

    return conf;/*from   w  w w  .  j  a v a2  s .com*/
}

From source file:com.example.kafkatohdfs.ApplicationTest.java

@Test
public void testApplication() throws IOException, Exception {
    try {//from w  w  w  .j  a  v a  2 s . c o  m
        LocalMode lma = LocalMode.newInstance();
        Configuration conf = new Configuration(false);
        conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
        lma.prepareDAG(new Kafka2HDFSApplication(), conf);
        LocalMode.Controller lc = lma.getController();
        lc.run(10000); // runs for 10 seconds and quits
    } catch (ConstraintViolationException e) {
        Assert.fail("constraint violations: " + e.getConstraintViolations());
    }
}

From source file:com.example.MyApex.SanExerciseApplicationTest.java

@Test
public void testApplication() throws IOException, Exception {
    try {/*  w w  w.j  a v a  2s . c  o m*/
        LocalMode lma = LocalMode.newInstance();
        Configuration conf = new Configuration(false);
        conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
        lma.prepareDAG(new SanExerciseApplication(), conf);
        LocalMode.Controller lc = lma.getController();
        lc.run(20000); // runs for 20 seconds and quits
    } catch (ConstraintViolationException e) {
        Assert.fail("constraint violations: " + e.getConstraintViolations());
    }
}

From source file:com.example.myapexapp.ApplicationTest.java

private void go(final boolean useUnifier) throws Exception {
    try {/*ww w.j  a v a  2  s  .  com*/
        LocalMode lma = LocalMode.newInstance();
        Configuration conf = new Configuration(false);
        conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
        conf.setBoolean("dt.application.MyFirstApplication.operator.console.prop.saveTuples", true);
        if (useUnifier) {
            conf.setBoolean("dt.application.MyFirstApplication.operator.range.prop.useUnifier", true);
        }
        lma.prepareDAG(new Application(), conf);
        ToConsole console = (ToConsole) lma.getDAG().getOperatorMeta("console").getOperator();
        LocalMode.Controller lc = lma.getController();
        lc.runAsync(); // runs for 10 seconds and quits

        // wait for tuples to show up
        while (!check(useUnifier, console)) {
            System.out.println("Sleeping ....");
            Thread.sleep(500);
        }

    } catch (ConstraintViolationException e) {
        Assert.fail("constraint violations: " + e.getConstraintViolations());
    }
}