Example usage for org.apache.hadoop.conf Configuration addResource

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration addResource.

Prototype

public void addResource(Configuration conf)

Source Link

Document

Add a configuration resource.

Usage

From source file:org.apache.nifi.processors.hive.TestPutHive3Streaming.java

License:Apache License

@Before
public void setUp() throws Exception {

    final String avroSchema = IOUtils.toString(new FileInputStream("src/test/resources/user.avsc"),
            StandardCharsets.UTF_8);
    schema = new Schema.Parser().parse(avroSchema);

    Configuration testConf = new Configuration();
    testConf.addResource(new Path(TEST_CONF_PATH));

    // needed for calls to UserGroupInformation.setConfiguration() to work when passing in
    // config with Kerberos authentication enabled
    System.setProperty("java.security.krb5.realm", "nifi.com");
    System.setProperty("java.security.krb5.kdc", "nifi.kdc");

    ugi = null;/*from w w w .j a  va  2 s. com*/
    processor = new MockPutHive3Streaming();
    hiveConfigurator = mock(HiveConfigurator.class);
    hiveConf = new HiveConf();
    when(hiveConfigurator.getConfigurationFromFiles(anyString())).thenReturn(hiveConf);
    processor.hiveConfigurator = hiveConfigurator;

    // Delete any temp files from previous tests
    try {
        FileUtils.deleteDirectory(new File(TARGET_HIVE));
    } catch (IOException ioe) {
        // Do nothing, directory may not have existed
    }
}

From source file:org.apache.nifi.processors.kite.AbstractKiteProcessor.java

License:Apache License

protected static Configuration getConfiguration(String configFiles) {
    Configuration conf = DefaultConfiguration.get();

    if (configFiles == null || configFiles.isEmpty()) {
        return conf;
    }/*from   ww w  .  j  a  va 2  s  .  c  o m*/

    for (String file : COMMA.split(configFiles)) {
        // process each resource only once
        if (conf.getResource(file) == null) {
            // use Path instead of String to get the file from the FS
            conf.addResource(new Path(file));
        }
    }

    return conf;
}

From source file:org.apache.nifi.util.hive.HiveJdbcCommon.java

License:Apache License

public static Configuration getConfigurationFromFiles(final String configFiles) {
    final Configuration hiveConfig = new HiveConf();
    if (StringUtils.isNotBlank(configFiles)) {
        for (final String configFile : configFiles.split(",")) {
            hiveConfig.addResource(new Path(configFile.trim()));
        }// www .j av  a  2  s  .  co  m
    }
    return hiveConfig;
}

From source file:org.apache.nutch.admin.AdministrationApp.java

License:Apache License

/**
 * creates an instance object from a instance folder
 * //from   w  ww . j  a  v a  2 s.co m
 * @param defaultConf
 * @param folder
 * @return an instance representation of this folder
 * @throws IOException
 *           in case the folder is not a valid instance folder
 */
public static NutchInstance loadNutchInstance(Configuration defaultConf, Path folder) throws IOException {
    Path instanceConfFolder = new Path(folder, "conf");

    Configuration conf = NutchConfiguration.create();
    FileSystem fs = FileSystem.get(conf);
    if (fs.exists(instanceConfFolder) && fs.isDirectory(instanceConfFolder)) {
        Path instanceSiteConf = new Path(instanceConfFolder, "nutch-site.xml");
        if (fs.exists(instanceSiteConf)) {
            Configuration instanceConf = new Configuration(defaultConf);
            instanceConf.addResource(instanceSiteConf.makeQualified(fs));
            return new NutchInstance(folder.getName(), folder, instanceConf);
        }
    }
    throw new IOException("not a valid instance folder: " + folder);
}

From source file:org.apache.nutch.admin.GuiConfigUtil.java

License:Apache License

/**
 * Push nutch-(default|site).xml from a given  folder/conf 
 * into a configuration. /*from   w w  w .j a v a 2 s  . c om*/
 * 
 * @param configuration
 * @param folder
 */
private static void configure(Configuration configuration, Path folder) throws IOException {
    FileSystem fs = FileSystem.get(NutchConfiguration.create());
    Path confFolder = new Path(folder, "conf");

    if (fs.exists(confFolder)) {
        Path defaultConf = new Path(confFolder, "nutch-default.xml");
        if (fs.exists(defaultConf)) {
            configuration.addResource(defaultConf.makeQualified(fs));
        }

        Path siteConf = new Path(confFolder, "nutch-site.xml");
        if (fs.exists(siteConf)) {
            configuration.addResource(siteConf.makeQualified(fs));
        }
    }
}

From source file:org.apache.nutch.crawl.CrawlDBTestUtil.java

License:Apache License

/**
 * For now we need to manually construct our Configuration, because we need to
 * override the default one and it is currently not possible to use dynamically
 * set values.//from w  w w.j  a  va  2  s . c  om
 * 
 * @return
 */
public static Configuration createConfiguration() {
    Configuration conf = new Configuration();
    conf.addResource("nutch-default.xml");
    conf.addResource("crawl-tests.xml");
    return conf;
}

From source file:org.apache.nutch.indexer.TestIndexingFilters.java

License:Apache License

/**
 * Test behaviour when defined filter does not exist.
 * //  w w w. j  a v a2s. c  o  m
 * @throws IndexingException
 */
@Test
public void testNonExistingIndexingFilter() throws IndexingException {
    Configuration conf = NutchConfiguration.create();
    conf.addResource("nutch-default.xml");
    conf.addResource("crawl-tests.xml");

    String class1 = "NonExistingFilter";
    String class2 = "org.apache.nutch.indexer.basic.BasicIndexingFilter";
    conf.set(IndexingFilters.INDEXINGFILTER_ORDER, class1 + " " + class2);

    IndexingFilters filters = new IndexingFilters(conf);
    WebPage page = WebPage.newBuilder().build();
    page.setText(new Utf8("text"));
    page.setTitle(new Utf8("title"));
    filters.filter(new NutchDocument(), "http://www.example.com/", page);
}

From source file:org.apache.nutch.indexer.TestIndexingFilters.java

License:Apache License

/**
 * Test behaviour when NutchDOcument is null
 * //  ww w .  ja  v  a2 s .c  om
 * @throws IndexingException
 */
@Test
public void testNutchDocumentNullIndexingFilter() throws IndexingException {
    Configuration conf = NutchConfiguration.create();
    conf.addResource("nutch-default.xml");
    conf.addResource("crawl-tests.xml");

    IndexingFilters filters = new IndexingFilters(conf);
    WebPage page = WebPage.newBuilder().build();
    page.setText(new Utf8("text"));
    page.setTitle(new Utf8("title"));
    NutchDocument doc = filters.filter(null, "http://www.example.com/", page);

    assertNull(doc);
}

From source file:org.apache.nutch.indexer.TestIndexingFilters.java

License:Apache License

/**
 * Test behaviour when reset the index filter order will not take effect
 * // w w  w  .  j a v a  2  s.co m
 * @throws IndexingException
 */
@Test
public void testFilterCacheIndexingFilter() throws IndexingException {
    Configuration conf = NutchConfiguration.create();
    conf.addResource("nutch-default.xml");
    conf.addResource("crawl-tests.xml");

    String class1 = "org.apache.nutch.indexer.basic.BasicIndexingFilter";
    conf.set(IndexingFilters.INDEXINGFILTER_ORDER, class1);

    IndexingFilters filters1 = new IndexingFilters(conf);
    WebPage page = WebPage.newBuilder().build();
    page.setText(new Utf8("text"));
    page.setTitle(new Utf8("title"));
    NutchDocument fdoc1 = filters1.filter(new NutchDocument(), "http://www.example.com/", page);

    // add another index filter
    String class2 = "org.apache.nutch.indexer.metadata.MetadataIndexer";
    // set content metadata
    Metadata md = new Metadata();
    md.add("example", "data");
    // add MetadataIndxer filter
    conf.set(IndexingFilters.INDEXINGFILTER_ORDER, class1 + " " + class2);
    IndexingFilters filters2 = new IndexingFilters(conf);
    NutchDocument fdoc2 = filters2.filter(new NutchDocument(), "http://www.example.com/", page);
    assertEquals(fdoc1.getFieldNames().size(), fdoc2.getFieldNames().size());
}

From source file:org.apache.nutch.plugin.NutchConfiguration.java

License:Apache License

/**
 * Add the standard Nutch resources to {@link Configuration}.
 * //from w  w  w . j a  v  a 2  s .c  o m
 * @param conf               Configuration object to which
 *                           configuration is to be added.
 */
private static Configuration addNutchResources(Configuration conf) {
    conf.addResource("nutch-default.xml");
    conf.addResource("nutch-site.xml");
    return conf;
}