Example usage for org.apache.hadoop.conf Configuration addResource

List of usage examples for org.apache.hadoop.conf Configuration addResource

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.conf.*
  5. Configuration
  6. addResource

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration addResource.

Prototype

public void addResource(Configuration conf)

Source Link

Document

Add a configuration resource.

Usage

From source file:org.apache.ignite.igfs.IgniteHadoopFileSystemSecondaryModeSelfTest.java

License:Apache License

/**
 * Perform initial startup./*from   w w  w .j  ava 2s  . co m*/
 *
 * @throws Exception If failed.
 */
@SuppressWarnings("NullableProblems")
private void startUp() throws Exception {
    startUpSecondary();

    FileSystemConfiguration igfsCfg = new FileSystemConfiguration();

    igfsCfg.setDataCacheName("partitioned");
    igfsCfg.setMetaCacheName("replicated");
    igfsCfg.setName("igfs");
    igfsCfg.setBlockSize(512 * 1024);
    igfsCfg.setDefaultMode(mode);
    igfsCfg.setPathModes(pathModes);

    IgfsIpcEndpointConfiguration endpointCfg = new IgfsIpcEndpointConfiguration();

    endpointCfg.setType(IgfsIpcEndpointType.TCP);
    endpointCfg.setPort(10500);

    igfsCfg.setIpcEndpointConfiguration(endpointCfg);

    igfsCfg.setManagementPort(-1);
    igfsCfg.setSecondaryFileSystem(new IgniteHadoopIgfsSecondaryFileSystem(
            "igfs://igfs-secondary:igfs-grid-secondary@127.0.0.1:11500/",
            "modules/core/src/test/config/hadoop/core-site-loopback-secondary.xml"));

    CacheConfiguration cacheCfg = defaultCacheConfiguration();

    cacheCfg.setName("partitioned");
    cacheCfg.setCacheMode(PARTITIONED);
    cacheCfg.setNearConfiguration(null);
    cacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
    cacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(128));
    cacheCfg.setBackups(0);
    cacheCfg.setAtomicityMode(TRANSACTIONAL);

    CacheConfiguration metaCacheCfg = defaultCacheConfiguration();

    metaCacheCfg.setName("replicated");
    metaCacheCfg.setCacheMode(REPLICATED);
    metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
    metaCacheCfg.setAtomicityMode(TRANSACTIONAL);

    IgniteConfiguration cfg = new IgniteConfiguration();

    cfg.setGridName("igfs-grid");

    TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();

    discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true));

    cfg.setDiscoverySpi(discoSpi);
    cfg.setCacheConfiguration(metaCacheCfg, cacheCfg);
    cfg.setFileSystemConfiguration(igfsCfg);

    cfg.setLocalHost("127.0.0.1");

    G.start(cfg);

    Configuration fsCfg = new Configuration();

    fsCfg.addResource(U.resolveIgniteUrl("modules/core/src/test/config/hadoop/core-site-loopback.xml"));

    fsCfg.setBoolean("fs.igfs.impl.disable.cache", true);

    fs = (IgniteHadoopFileSystem) FileSystem.get(new URI("igfs://igfs:igfs-grid@/"), fsCfg);
}

From source file:org.apache.ignite.internal.igfs.hadoop.IgfsHadoopFileSystemWrapper.java

License:Apache License

/**
 * Constructor.//  ww w.j  a  va  2s .co m
 *
 * @param uri URI of file system.
 * @param cfgPath Additional path to Hadoop configuration.
 * @throws IgniteCheckedException In case of error.
 */
public IgfsHadoopFileSystemWrapper(@Nullable String uri, @Nullable String cfgPath)
        throws IgniteCheckedException {
    Configuration cfg = new Configuration();

    if (cfgPath != null)
        cfg.addResource(U.resolveIgniteUrl(cfgPath));

    try {
        fileSys = uri == null ? FileSystem.get(cfg) : FileSystem.get(new URI(uri), cfg);
    } catch (IOException | URISyntaxException e) {
        throw new IgniteCheckedException(e);
    }

    uri = fileSys.getUri().toString();

    if (!uri.endsWith("/"))
        uri += "/";

    props.put(SECONDARY_FS_CONFIG_PATH, cfgPath);
    props.put(SECONDARY_FS_URI, uri);
}

From source file:org.apache.ignite.internal.processors.hadoop.GridHadoopPopularWordsTest.java

License:Apache License

/**
 * Configures the Hadoop MapReduce job.// ww w .j  a v  a  2 s  .  com
 *
 * @return Instance of the Hadoop MapRed job.
 * @throws IOException If failed.
 */
private Job createConfigBasedHadoopJob() throws IOException {
    Job jobCfg = new Job();

    Configuration cfg = jobCfg.getConfiguration();

    // Use explicit configuration of distributed file system, if provided.
    if (DFS_CFG != null)
        cfg.addResource(U.resolveIgniteUrl(DFS_CFG));

    jobCfg.setJobName("HadoopPopularWordExample");
    jobCfg.setJarByClass(GridHadoopPopularWordsTest.class);
    jobCfg.setInputFormatClass(TextInputFormat.class);
    jobCfg.setOutputKeyClass(Text.class);
    jobCfg.setOutputValueClass(IntWritable.class);
    jobCfg.setMapperClass(TokenizingMapper.class);
    jobCfg.setReducerClass(TopNWordsReducer.class);

    FileInputFormat.setInputPaths(jobCfg, BOOKS_DFS_DIR);
    FileOutputFormat.setOutputPath(jobCfg, RESULT_DFS_DIR);

    // Local job tracker allows the only task per wave, but text input format
    // replaces it with the calculated value based on input split size option.
    if ("local".equals(cfg.get("mapred.job.tracker", "local"))) {
        // Split job into tasks using 32MB split size.
        FileInputFormat.setMinInputSplitSize(jobCfg, 32 * 1024 * 1024);
        FileInputFormat.setMaxInputSplitSize(jobCfg, Long.MAX_VALUE);
    }

    return jobCfg;
}

From source file:org.apache.ignite.internal.processors.hadoop.HadoopPopularWordsTest.java

License:Apache License

/**
 * Configures the Hadoop MapReduce job.//from w  w w  .  ja  va  2s . c o m
 *
 * @return Instance of the Hadoop MapRed job.
 * @throws IOException If failed.
 */
@SuppressWarnings("deprecation")
private Job createConfigBasedHadoopJob() throws IOException {
    Job jobCfg = new Job();

    Configuration cfg = jobCfg.getConfiguration();

    // Use explicit configuration of distributed file system, if provided.
    cfg.addResource(U.resolveIgniteUrl(DFS_CFG));

    jobCfg.setJobName("HadoopPopularWordExample");
    jobCfg.setJarByClass(HadoopPopularWordsTest.class);
    jobCfg.setInputFormatClass(TextInputFormat.class);
    jobCfg.setOutputKeyClass(Text.class);
    jobCfg.setOutputValueClass(IntWritable.class);
    jobCfg.setMapperClass(TokenizingMapper.class);
    jobCfg.setReducerClass(TopNWordsReducer.class);

    FileInputFormat.setInputPaths(jobCfg, BOOKS_DFS_DIR);
    FileOutputFormat.setOutputPath(jobCfg, RESULT_DFS_DIR);

    // Local job tracker allows the only task per wave, but text input format
    // replaces it with the calculated value based on input split size option.
    if ("local".equals(cfg.get("mapred.job.tracker", "local"))) {
        // Split job into tasks using 32MB split size.
        FileInputFormat.setMinInputSplitSize(jobCfg, 32 * 1024 * 1024);
        FileInputFormat.setMaxInputSplitSize(jobCfg, Long.MAX_VALUE);
    }

    return jobCfg;
}

From source file:org.apache.ignite.internal.processors.hadoop.impl.igfs.HadoopIgfsDualAbstractSelfTest.java

License:Apache License

/**
 * Check how prefetch override works./*from w ww  .ja  v a 2 s.  c o  m*/
 *
 * @throws Exception IF failed.
 */
public void testOpenPrefetchOverride() throws Exception {
    create(igfsSecondary, paths(DIR, SUBDIR), paths(FILE));

    // Write enough data to the secondary file system.
    final int blockSize = IGFS_BLOCK_SIZE;

    IgfsOutputStream out = igfsSecondary.append(FILE, false);

    int totalWritten = 0;

    while (totalWritten < blockSize * 2 + chunk.length) {
        out.write(chunk);

        totalWritten += chunk.length;
    }

    out.close();

    awaitFileClose(igfsSecondary, FILE);

    // Instantiate file system with overridden "seq reads before prefetch" property.
    Configuration cfg = new Configuration();

    cfg.addResource(U.resolveIgniteUrl(PRIMARY_CFG));

    int seqReads = SEQ_READS_BEFORE_PREFETCH + 1;

    cfg.setInt(String.format(PARAM_IGFS_SEQ_READS_BEFORE_PREFETCH, "igfs@"), seqReads);

    FileSystem fs = FileSystem.get(new URI(PRIMARY_URI), cfg);

    // Read the first two blocks.
    Path fsHome = new Path(PRIMARY_URI);
    Path dir = new Path(fsHome, DIR.name());
    Path subdir = new Path(dir, SUBDIR.name());
    Path file = new Path(subdir, FILE.name());

    FSDataInputStream fsIn = fs.open(file);

    final byte[] readBuf = new byte[blockSize * 2];

    fsIn.readFully(0, readBuf, 0, readBuf.length);

    // Wait for a while for prefetch to finish (if any).
    IgfsMetaManager meta = igfs.context().meta();

    IgfsEntryInfo info = meta.info(meta.fileId(FILE));

    IgfsBlockKey key = new IgfsBlockKey(info.id(), info.affinityKey(), info.evictExclude(), 2);

    IgniteCache<IgfsBlockKey, byte[]> dataCache = igfs.context().kernalContext().cache()
            .jcache(igfs.configuration().getDataCacheConfiguration().getName());

    for (int i = 0; i < 10; i++) {
        if (dataCache.containsKey(key))
            break;
        else
            U.sleep(100);
    }

    fsIn.close();

    // Remove the file from the secondary file system.
    igfsSecondary.delete(FILE, false);

    // Try reading the third block. Should fail.
    GridTestUtils.assertThrows(log, new Callable<Object>() {
        @Override
        public Object call() throws Exception {
            IgfsInputStream in0 = igfs.open(FILE);

            in0.seek(blockSize * 2);

            try {
                in0.read(readBuf);
            } finally {
                U.closeQuiet(in0);
            }

            return null;
        }
    }, IOException.class, "Failed to read data due to secondary file system exception: /dir/subdir/file");
}

From source file:org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemIpcCacheSelfTest.java

License:Apache License

/**
 * Test how IPC cache map works./* w ww .  j a va  2 s  . c om*/
 *
 * @throws Exception If failed.
 */
@SuppressWarnings("unchecked")
public void testIpcCache() throws Exception {
    Field cacheField = HadoopIgfsIpcIo.class.getDeclaredField("ipcCache");

    cacheField.setAccessible(true);

    Field activeCntField = HadoopIgfsIpcIo.class.getDeclaredField("activeCnt");

    activeCntField.setAccessible(true);

    Map<String, HadoopIgfsIpcIo> cache = (Map<String, HadoopIgfsIpcIo>) cacheField.get(null);

    cache.clear(); // avoid influence of previous tests in the same process.

    String name = "igfs:" + getTestIgniteInstanceName(0) + "@";

    Configuration cfg = new Configuration();

    cfg.addResource(U.resolveIgniteUrl(HADOOP_FS_CFG));
    cfg.setBoolean("fs.igfs.impl.disable.cache", true);
    cfg.setBoolean(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_NO_EMBED, name), true);

    // Ensure that existing IO is reused.
    FileSystem fs1 = FileSystem.get(new URI("igfs://" + name + "/"), cfg);

    assertEquals(1, cache.size());

    HadoopIgfsIpcIo io = null;

    System.out.println("CACHE: " + cache);

    for (String key : cache.keySet()) {
        if (key.contains("10500")) {
            io = cache.get(key);

            break;
        }
    }

    assert io != null;

    assertEquals(1, ((AtomicInteger) activeCntField.get(io)).get());

    // Ensure that when IO is used by multiple file systems and one of them is closed, IO is not stopped.
    FileSystem fs2 = FileSystem.get(new URI("igfs://" + name + "/abc"), cfg);

    assertEquals(1, cache.size());
    assertEquals(2, ((AtomicInteger) activeCntField.get(io)).get());

    fs2.close();

    assertEquals(1, cache.size());
    assertEquals(1, ((AtomicInteger) activeCntField.get(io)).get());

    Field stopField = HadoopIgfsIpcIo.class.getDeclaredField("stopping");

    stopField.setAccessible(true);

    assert !(Boolean) stopField.get(io);

    // Ensure that IO is stopped when nobody else is need it.
    fs1.close();

    assert cache.isEmpty();

    assert (Boolean) stopField.get(io);
}

From source file:org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemLoggerStateSelfTest.java

License:Apache License

/**
 * Instantiate new file system./*from  ww  w. j a va  2  s .  c  om*/
 *
 * @return New file system.
 * @throws Exception If failed.
 */
private IgniteHadoopFileSystem fileSystem() throws Exception {
    Configuration fsCfg = new Configuration();

    fsCfg.addResource(U.resolveIgniteUrl("modules/core/src/test/config/hadoop/core-site-loopback.xml"));

    fsCfg.setBoolean("fs.igfs.impl.disable.cache", true);

    if (logging)
        fsCfg.setBoolean(String.format(PARAM_IGFS_LOG_ENABLED, "igfs@"), logging);

    fsCfg.setStrings(String.format(PARAM_IGFS_LOG_DIR, "igfs@"), U.getIgniteHome());

    return (IgniteHadoopFileSystem) FileSystem.get(new URI("igfs://igfs@/"), fsCfg);
}

From source file:org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemSecondaryFileSystemInitializationSelfTest.java

License:Apache License

/**
 * Perform initial startup./*from   w w w .  ja  v  a 2 s . c o  m*/
 *
 * @param initDfltPathModes WHether to initialize default path modes.
 * @throws Exception If failed.
 */
@SuppressWarnings({ "NullableProblems", "unchecked" })
private void startUp(boolean initDfltPathModes) throws Exception {
    startUpSecondary();

    FileSystemConfiguration igfsCfg = new FileSystemConfiguration();

    igfsCfg.setName("igfs");
    igfsCfg.setBlockSize(512 * 1024);
    igfsCfg.setInitializeDefaultPathModes(initDfltPathModes);

    IgfsIpcEndpointConfiguration endpointCfg = new IgfsIpcEndpointConfiguration();

    endpointCfg.setType(IgfsIpcEndpointType.TCP);
    endpointCfg.setPort(10500);

    igfsCfg.setIpcEndpointConfiguration(endpointCfg);

    igfsCfg.setManagementPort(-1);
    igfsCfg.setSecondaryFileSystem(
            new IgniteHadoopIgfsSecondaryFileSystem("igfs://igfs-secondary@127.0.0.1:11500/",
                    "modules/core/src/test/config/hadoop/core-site-loopback-secondary.xml"));

    CacheConfiguration dataCacheCfg = defaultCacheConfiguration();

    dataCacheCfg.setCacheMode(PARTITIONED);
    dataCacheCfg.setNearConfiguration(null);
    dataCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
    dataCacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(128));
    dataCacheCfg.setBackups(0);
    dataCacheCfg.setAtomicityMode(TRANSACTIONAL);

    CacheConfiguration metaCacheCfg = defaultCacheConfiguration();

    metaCacheCfg.setCacheMode(REPLICATED);
    metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
    metaCacheCfg.setAtomicityMode(TRANSACTIONAL);

    igfsCfg.setDataCacheConfiguration(dataCacheCfg);
    igfsCfg.setMetaCacheConfiguration(metaCacheCfg);

    IgniteConfiguration cfg = new IgniteConfiguration();

    cfg.setIgniteInstanceName("igfs-grid");

    TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();

    discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true));

    cfg.setDiscoverySpi(discoSpi);
    cfg.setFileSystemConfiguration(igfsCfg);

    cfg.setLocalHost("127.0.0.1");

    G.start(cfg);

    Configuration fsCfg = new Configuration();

    fsCfg.addResource(U.resolveIgniteUrl("modules/core/src/test/config/hadoop/core-site-loopback.xml"));

    fsCfg.setBoolean("fs.igfs.impl.disable.cache", true);

    fs = (IgniteHadoopFileSystem) FileSystem.get(new URI("igfs://igfs@/"), fsCfg);
}

From source file:org.apache.ignite.loadtests.igfs.IgfsPerformanceBenchmark.java

License:Apache License

/** {@inheritDoc} */
private static FileSystem igfs(Path home, String cfgPath) throws IOException {
    Configuration cfg = new Configuration();

    cfg.addResource(U.resolveIgniteUrl(cfgPath));

    return FileSystem.get(home.toUri(), cfg);
}

From source file:org.apache.ivory.latedata.LateDataHandler.java

License:Apache License

public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    Path confPath = new Path("file:///" + System.getProperty("oozie.action.conf.xml"));

    LOG.info(confPath + " found ? " + confPath.getFileSystem(conf).exists(confPath));
    conf.addResource(confPath);
    ToolRunner.run(new Configuration(), new LateDataHandler(), args);
}