Example usage for org.apache.hadoop.conf Configuration setBoolean

List of usage examples for org.apache.hadoop.conf Configuration setBoolean

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.conf.*
  5. Configuration
  6. setBoolean

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration setBoolean.

Prototype

public void setBoolean(String name, boolean value)

Source Link

Document

Set the value of the name property to a boolean.

Usage

From source file:org.apache.ignite.igfs.IgniteHadoopFileSystemSecondaryModeSelfTest.java

License:Apache License

/**
 * Perform initial startup./*  ww w.j  a  va2s.c  om*/
 *
 * @throws Exception If failed.
 */
@SuppressWarnings("NullableProblems")
private void startUp() throws Exception {
    startUpSecondary();

    FileSystemConfiguration igfsCfg = new FileSystemConfiguration();

    igfsCfg.setDataCacheName("partitioned");
    igfsCfg.setMetaCacheName("replicated");
    igfsCfg.setName("igfs");
    igfsCfg.setBlockSize(512 * 1024);
    igfsCfg.setDefaultMode(mode);
    igfsCfg.setPathModes(pathModes);

    IgfsIpcEndpointConfiguration endpointCfg = new IgfsIpcEndpointConfiguration();

    endpointCfg.setType(IgfsIpcEndpointType.TCP);
    endpointCfg.setPort(10500);

    igfsCfg.setIpcEndpointConfiguration(endpointCfg);

    igfsCfg.setManagementPort(-1);
    igfsCfg.setSecondaryFileSystem(new IgniteHadoopIgfsSecondaryFileSystem(
            "igfs://igfs-secondary:igfs-grid-secondary@127.0.0.1:11500/",
            "modules/core/src/test/config/hadoop/core-site-loopback-secondary.xml"));

    CacheConfiguration cacheCfg = defaultCacheConfiguration();

    cacheCfg.setName("partitioned");
    cacheCfg.setCacheMode(PARTITIONED);
    cacheCfg.setNearConfiguration(null);
    cacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
    cacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(128));
    cacheCfg.setBackups(0);
    cacheCfg.setAtomicityMode(TRANSACTIONAL);

    CacheConfiguration metaCacheCfg = defaultCacheConfiguration();

    metaCacheCfg.setName("replicated");
    metaCacheCfg.setCacheMode(REPLICATED);
    metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
    metaCacheCfg.setAtomicityMode(TRANSACTIONAL);

    IgniteConfiguration cfg = new IgniteConfiguration();

    cfg.setGridName("igfs-grid");

    TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();

    discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true));

    cfg.setDiscoverySpi(discoSpi);
    cfg.setCacheConfiguration(metaCacheCfg, cacheCfg);
    cfg.setFileSystemConfiguration(igfsCfg);

    cfg.setLocalHost("127.0.0.1");

    G.start(cfg);

    Configuration fsCfg = new Configuration();

    fsCfg.addResource(U.resolveIgniteUrl("modules/core/src/test/config/hadoop/core-site-loopback.xml"));

    fsCfg.setBoolean("fs.igfs.impl.disable.cache", true);

    fs = (IgniteHadoopFileSystem) FileSystem.get(new URI("igfs://igfs:igfs-grid@/"), fsCfg);
}

From source file:org.apache.ignite.internal.processors.hadoop.fs.HadoopFileSystemCacheUtils.java

License:Apache License

/**
 * A common static factory method. Creates new HadoopLazyConcurrentMap.
 * @return a new HadoopLazyConcurrentMap.
 *//*  w  w w.j  ava 2 s. c  o  m*/
public static HadoopLazyConcurrentMap<FsCacheKey, FileSystem> createHadoopLazyConcurrentMap() {
    return new HadoopLazyConcurrentMap<>(new HadoopLazyConcurrentMap.ValueFactory<FsCacheKey, FileSystem>() {
        @Override
        public FileSystem createValue(FsCacheKey key) {
            try {
                assert key != null;

                // Explicitly disable FileSystem caching:
                URI uri = key.uri();

                String scheme = uri.getScheme();

                // Copy the configuration to avoid altering the external object.
                Configuration cfg = new Configuration(key.configuration());

                String prop = HadoopFileSystemsUtils.disableFsCachePropertyName(scheme);

                cfg.setBoolean(prop, true);

                return FileSystem.get(uri, cfg, key.user());
            } catch (IOException | InterruptedException ioe) {
                throw new IgniteException(ioe);
            }
        }
    });
}

From source file:org.apache.ignite.internal.processors.hadoop.GridHadoopTaskExecutionSelfTest.java

License:Apache License

/**
 * @throws Exception If failed.//from www  .j  a  v a2 s  . c  o  m
 */
public void testMapCombineRun() throws Exception {
    int lineCnt = 10001;
    String fileName = "/testFile";

    prepareFile(fileName, lineCnt);

    totalLineCnt.set(0);
    taskWorkDirs.clear();

    Configuration cfg = new Configuration();

    cfg.setStrings("fs.igfs.impl", IgfsHadoopFileSystem.class.getName());
    cfg.setBoolean(MAP_WRITE, true);

    Job job = Job.getInstance(cfg);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);

    job.setMapperClass(TestMapper.class);
    job.setCombinerClass(TestCombiner.class);
    job.setReducerClass(TestReducer.class);

    job.setNumReduceTasks(2);

    job.setInputFormatClass(TextInputFormat.class);

    FileInputFormat.setInputPaths(job, new Path("igfs://:" + getTestGridName(0) + "@/"));
    FileOutputFormat.setOutputPath(job, new Path("igfs://:" + getTestGridName(0) + "@/output"));

    job.setJarByClass(getClass());

    GridHadoopJobId jobId = new GridHadoopJobId(UUID.randomUUID(), 2);

    IgniteInternalFuture<?> fut = grid(0).hadoop().submit(jobId, createJobInfo(job.getConfiguration()));

    fut.get();

    assertEquals(lineCnt, totalLineCnt.get());

    assertEquals(34, taskWorkDirs.size());

    for (int g = 0; g < gridCount(); g++)
        grid(g).hadoop().finishFuture(jobId).get();
}

From source file:org.apache.ignite.internal.processors.hadoop.HadoopTaskExecutionSelfTest.java

License:Apache License

/**
 * @throws Exception If failed./*ww  w.  j a v  a 2 s .co m*/
 */
public void testMapCombineRun() throws Exception {
    int lineCnt = 10001;
    String fileName = "/testFile";

    prepareFile(fileName, lineCnt);

    totalLineCnt.set(0);
    taskWorkDirs.clear();

    Configuration cfg = new Configuration();

    cfg.setStrings("fs.igfs.impl", IgniteHadoopFileSystem.class.getName());
    cfg.setBoolean(MAP_WRITE, true);

    Job job = Job.getInstance(cfg);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);

    job.setMapperClass(TestMapper.class);
    job.setCombinerClass(TestCombiner.class);
    job.setReducerClass(TestReducer.class);

    job.setNumReduceTasks(2);

    job.setInputFormatClass(TextInputFormat.class);

    FileInputFormat.setInputPaths(job, new Path("igfs://:" + getTestGridName(0) + "@/"));
    FileOutputFormat.setOutputPath(job, new Path("igfs://:" + getTestGridName(0) + "@/output"));

    job.setJarByClass(getClass());

    HadoopJobId jobId = new HadoopJobId(UUID.randomUUID(), 2);

    IgniteInternalFuture<?> fut = grid(0).hadoop().submit(jobId, createJobInfo(job.getConfiguration()));

    fut.get();

    assertEquals(lineCnt, totalLineCnt.get());

    assertEquals(34, taskWorkDirs.size());

    for (int g = 0; g < gridCount(); g++)
        grid(g).hadoop().finishFuture(jobId).get();
}

From source file:org.apache.ignite.internal.processors.hadoop.impl.fs.HadoopFileSystemCacheUtils.java

License:Apache License

/**
 * A common static factory method. Creates new HadoopLazyConcurrentMap.
 * @return a new HadoopLazyConcurrentMap.
 *///  w  ww . j  a  va  2  s  .  com
public static HadoopLazyConcurrentMap<FsCacheKey, FileSystem> createHadoopLazyConcurrentMap() {
    return new HadoopLazyConcurrentMap<>(new HadoopLazyConcurrentMap.ValueFactory<FsCacheKey, FileSystem>() {
        @Override
        public FileSystem createValue(FsCacheKey key) throws IOException {
            try {
                assert key != null;

                // Explicitly disable FileSystem caching:
                URI uri = key.uri();

                String scheme = uri.getScheme();

                // Copy the configuration to avoid altering the external object.
                Configuration cfg = new Configuration(key.configuration());

                String prop = HadoopFileSystemsUtils.disableFsCachePropertyName(scheme);

                cfg.setBoolean(prop, true);

                return FileSystem.get(uri, cfg, key.user());
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();

                throw new IOException("Failed to create file system due to interrupt.", e);
            }
        }
    });
}

From source file:org.apache.ignite.internal.processors.hadoop.impl.HadoopTaskExecutionSelfTest.java

License:Apache License

/**
 * @throws Exception If failed.//from   www.  j a v a  2s  . c o  m
 */
public void testMapCombineRun() throws Exception {
    int lineCnt = 10001;
    String fileName = "/testFile";

    prepareFile(fileName, lineCnt);

    totalLineCnt.set(0);
    taskWorkDirs.clear();

    Configuration cfg = new Configuration();

    cfg.setStrings("fs.igfs.impl", IgniteHadoopFileSystem.class.getName());
    cfg.setBoolean(MAP_WRITE, true);

    Job job = Job.getInstance(cfg);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);

    job.setMapperClass(TestMapper.class);
    job.setCombinerClass(TestCombiner.class);
    job.setReducerClass(TestReducer.class);

    job.setNumReduceTasks(2);

    job.setInputFormatClass(TextInputFormat.class);

    FileInputFormat.setInputPaths(job, new Path("igfs://" + igfsName + "@/"));
    FileOutputFormat.setOutputPath(job, new Path("igfs://" + igfsName + "@/output/"));

    job.setJarByClass(getClass());

    HadoopJobId jobId = new HadoopJobId(UUID.randomUUID(), 2);

    IgniteInternalFuture<?> fut = grid(0).hadoop().submit(jobId, createJobInfo(job.getConfiguration()));

    fut.get();

    assertEquals(lineCnt, totalLineCnt.get());

    assertEquals(34, taskWorkDirs.size());

    for (int g = 0; g < gridCount(); g++)
        grid(g).hadoop().finishFuture(jobId).get();
}

From source file:org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemAbstractSelfTest.java

License:Apache License

/**
 * Test how IPC cache map works.//w  ww.  j  a  v a 2 s  .  c o  m
 *
 * @throws Exception If failed.
 */
public void testIpcCache() throws Exception {
    HadoopIgfsEx hadoop = GridTestUtils.getFieldValue(fs, "rmtClient", "delegateRef", "value", "hadoop");

    if (hadoop instanceof HadoopIgfsOutProc) {
        FileSystem fsOther = null;

        try {
            Field field = HadoopIgfsIpcIo.class.getDeclaredField("ipcCache");

            field.setAccessible(true);

            Map<String, HadoopIgfsIpcIo> cache = (Map<String, HadoopIgfsIpcIo>) field.get(null);

            Configuration cfg = configuration(PRIMARY_AUTHORITY, skipEmbed, skipLocShmem);

            // we disable caching in order to obtain new FileSystem instance.
            cfg.setBoolean("fs.igfs.impl.disable.cache", true);

            // Initial cache size.
            int initSize = cache.size();

            // Ensure that when IO is used by multiple file systems and one of them is closed, IO is not stopped.
            fsOther = FileSystem.get(new URI(PRIMARY_URI), cfg);

            assert fs != fsOther;

            assertEquals(initSize, cache.size());

            fsOther.close();

            assertEquals(initSize, cache.size());

            Field stopField = HadoopIgfsIpcIo.class.getDeclaredField("stopping");

            stopField.setAccessible(true);

            HadoopIgfsIpcIo io = null;

            for (Map.Entry<String, HadoopIgfsIpcIo> ioEntry : cache.entrySet()) {
                if (endpoint.contains(ioEntry.getKey())) {
                    io = ioEntry.getValue();

                    break;
                }
            }

            assert io != null;

            assert !(Boolean) stopField.get(io);

            // Ensure that IO is stopped when nobody else is need it.
            fs.close();

            assert initSize >= cache.size();

            assert (Boolean) stopField.get(io);
        } finally {
            U.closeQuiet(fsOther);
        }
    }
}

From source file:org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemAbstractSelfTest.java

License:Apache License

/**
 * Verifies that client reconnects after connection to the server has been lost (multithreaded mode).
 *
 * @throws Exception If error occurs./*from   w w  w.  j  av  a 2 s . c o m*/
 */
public void testClientReconnectMultithreaded() throws Exception {
    final ConcurrentLinkedQueue<FileSystem> q = new ConcurrentLinkedQueue<>();

    Configuration cfg = new Configuration();

    for (Map.Entry<String, String> entry : primaryFsCfg)
        cfg.set(entry.getKey(), entry.getValue());

    cfg.setBoolean("fs.igfs.impl.disable.cache", true);

    final int nClients = 1;

    // Initialize clients.
    for (int i = 0; i < nClients; i++)
        q.add(FileSystem.get(primaryFsUri, cfg));

    stopNodes();

    startNodes(); // Start server again.

    GridTestUtils.runMultiThreaded(new Callable<Object>() {
        @Override
        public Object call() throws Exception {
            FileSystem fs = q.poll();

            try {
                // Check that client is again operational.
                assertTrue(fs.mkdirs(new Path("/" + Thread.currentThread().getName())));

                return true;
            } finally {
                U.closeQuiet(fs);
            }
        }
    }, nClients, "test-client");
}

From source file:org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemAbstractSelfTest.java

License:Apache License

/**
 * Create configuration for test.//  w  w  w .j ava2s. c  o m
 *
 * @param authority Authority.
 * @param skipEmbed Whether to skip embedded mode.
 * @param skipLocShmem Whether to skip local shmem mode.
 * @return Configuration.
 */
protected Configuration configuration(String authority, boolean skipEmbed, boolean skipLocShmem) {
    Configuration cfg = new Configuration();

    cfg.set("fs.defaultFS", "igfs://" + authority + "/");
    cfg.set("fs.igfs.impl", IgniteHadoopFileSystem.class.getName());
    cfg.set("fs.AbstractFileSystem.igfs.impl",
            org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem.class.getName());

    cfg.setBoolean("fs.igfs.impl.disable.cache", true);

    if (skipEmbed)
        cfg.setBoolean(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_NO_EMBED, authority), true);

    if (skipLocShmem)
        cfg.setBoolean(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_NO_LOCAL_SHMEM, authority), true);

    return cfg;
}

From source file:org.apache.ignite.internal.processors.hadoop.impl.igfs.IgniteHadoopFileSystemAbstractSelfTest.java

License:Apache License

/**
 * Create configuration for test./*  w  ww.  j  a v a  2  s  . co m*/
 *
 * @param authority Authority.
 * @param skipEmbed Whether to skip embedded mode.
 * @param skipLocShmem Whether to skip local shmem mode.
 * @return Configuration.
 */
protected Configuration configurationSecondary(String authority) {
    Configuration cfg = new Configuration();

    cfg.set("fs.defaultFS", "igfs://" + authority + "/");
    cfg.set("fs.igfs.impl", IgniteHadoopFileSystem.class.getName());
    cfg.set("fs.AbstractFileSystem.igfs.impl",
            org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem.class.getName());

    cfg.setBoolean("fs.igfs.impl.disable.cache", true);

    cfg.setBoolean(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_NO_EMBED, authority), true);

    cfg.setBoolean(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_NO_LOCAL_SHMEM, authority), true);

    return cfg;
}