Example usage for org.apache.hadoop.conf Configuration setBoolean

List of usage examples for org.apache.hadoop.conf Configuration setBoolean

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.conf.*
  5. Configuration
  6. setBoolean

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration setBoolean.

Prototype

public void setBoolean(String name, boolean value)

Source Link

Document

Set the value of the name property to a boolean.

Usage

From source file:org.apache.hawq.pxf.plugins.hdfs.ChunkRecordReader.java

License:Apache License

/**
 * Constructs a ChunkRecordReader instance.
 *
 * @param job the job configuration/*from   ww w  .  j a  v  a2  s.c  o m*/
 * @param split contains the file name, begin byte of the split and the
 *            bytes length
 * @throws IOException if an I/O error occurs when accessing the file or
 *             creating input stream to read from it
 */
public ChunkRecordReader(Configuration job, FileSplit split) throws IOException {
    maxLineLength = job.getInt(MAX_LINE_LENGTH, Integer.MAX_VALUE);
    validateLength(maxLineLength);
    start = split.getStart();
    end = start + split.getLength();
    final Path file = split.getPath();
    compressionCodecs = new CompressionCodecFactory(job);
    codec = compressionCodecs.getCodec(file);

    // open the file and seek to the start of the split
    job.setBoolean(DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM_KEY, true);
    final FileSystem fs = file.getFileSystem(job);
    fs.setVerifyChecksum(false);
    fileIn = fs.open(file, ChunkReader.DEFAULT_BUFFER_SIZE);
    fileLength = getInputStream().getFileLength();
    if (isCompressedInput()) {
        decompressor = CodecPool.getDecompressor(codec);
        if (codec instanceof SplittableCompressionCodec) {
            final SplitCompressionInputStream cIn = ((SplittableCompressionCodec) codec).createInputStream(
                    fileIn, decompressor, start, end, SplittableCompressionCodec.READ_MODE.BYBLOCK);
            in = new ChunkReader(cIn);
            start = cIn.getAdjustedStart();
            end = cIn.getAdjustedEnd();
            filePosition = cIn; // take pos from compressed stream
        } else {
            in = new ChunkReader(codec.createInputStream(fileIn, decompressor));
            filePosition = fileIn;
        }
    } else {
        fileIn.seek(start);
        in = new ChunkReader(fileIn);
        filePosition = fileIn;
    }
    /*
     * If this is not the first split, we always throw away first record
     * because we always (except the last split) read one extra line in
     * next() method.
     */
    if (start != 0) {
        start += in.readLine(new ChunkWritable(), maxBytesToConsume(start));
    }
    this.pos = start;
}

From source file:org.apache.hcatalog.hbase.ImportSequenceFile.java

License:Apache License

/**
 * Method to run the Importer MapReduce Job. Normally will be called by another MR job
 * during OutputCommitter.commitJob().// w w w  .  ja  v  a2  s .co m
 * @param parentContext JobContext of the parent job
 * @param tableName name of table to bulk load data into
 * @param InputDir path of SequenceFile formatted data to read
 * @param scratchDir temporary path for the Importer MR job to build the HFiles which will be imported
 * @return
 */
static boolean runJob(JobContext parentContext, String tableName, Path InputDir, Path scratchDir) {
    Configuration parentConf = parentContext.getConfiguration();
    Configuration conf = new Configuration();
    for (Map.Entry<String, String> el : parentConf) {
        if (el.getKey().startsWith("hbase."))
            conf.set(el.getKey(), el.getValue());
        if (el.getKey().startsWith("mapred.cache.archives"))
            conf.set(el.getKey(), el.getValue());
    }

    //Inherit jar dependencies added to distributed cache loaded by parent job
    conf.set("mapred.job.classpath.archives", parentConf.get("mapred.job.classpath.archives", ""));
    conf.set("mapreduce.job.cache.archives.visibilities",
            parentConf.get("mapreduce.job.cache.archives.visibilities", ""));

    //Temporary fix until hbase security is ready
    //We need the written HFile to be world readable so
    //hbase regionserver user has the privileges to perform a hdfs move
    if (parentConf.getBoolean("hadoop.security.authorization", false)) {
        FsPermission.setUMask(conf, FsPermission.valueOf("----------"));
    }

    conf.set(HBaseConstants.PROPERTY_OUTPUT_TABLE_NAME_KEY, tableName);
    conf.setBoolean(JobContext.JOB_CANCEL_DELEGATION_TOKEN, false);

    boolean localMode = "local".equals(conf.get("mapred.job.tracker"));

    boolean success = false;
    try {
        FileSystem fs = FileSystem.get(parentConf);
        Path workDir = new Path(new Job(parentConf).getWorkingDirectory(), IMPORTER_WORK_DIR);
        if (!fs.mkdirs(workDir))
            throw new IOException("Importer work directory already exists: " + workDir);
        Job job = createSubmittableJob(conf, tableName, InputDir, scratchDir, localMode);
        job.setWorkingDirectory(workDir);
        job.getCredentials().addAll(parentContext.getCredentials());
        success = job.waitForCompletion(true);
        fs.delete(workDir, true);
        //We only cleanup on success because failure might've been caused by existence of target directory
        if (localMode && success) {
            new ImporterOutputFormat().getOutputCommitter(
                    org.apache.hadoop.mapred.HCatMapRedUtil.createTaskAttemptContext(conf, new TaskAttemptID()))
                    .commitJob(job);
        }
    } catch (InterruptedException e) {
        LOG.error("ImportSequenceFile Failed", e);
    } catch (ClassNotFoundException e) {
        LOG.error("ImportSequenceFile Failed", e);
    } catch (IOException e) {
        LOG.error("ImportSequenceFile Failed", e);
    }
    return success;
}

From source file:org.apache.hive.jdbc.TestActivePassiveHA.java

License:Apache License

private static void setHAConfigs(Configuration conf) {
    conf.setBoolean(ConfVars.HIVE_SERVER2_SUPPORT_DYNAMIC_SERVICE_DISCOVERY.varname, true);
    conf.set(ConfVars.HIVE_ZOOKEEPER_QUORUM.varname, zkServer.getConnectString());
    conf.setBoolean(ConfVars.HIVE_SERVER2_ACTIVE_PASSIVE_HA_ENABLE.varname, true);
    conf.set(ConfVars.HIVE_SERVER2_ACTIVE_PASSIVE_HA_REGISTRY_NAMESPACE.varname, zkHANamespace);
    conf.setTimeDuration(ConfVars.HIVE_ZOOKEEPER_CONNECTION_TIMEOUT.varname, 2, TimeUnit.SECONDS);
    conf.setTimeDuration(ConfVars.HIVE_ZOOKEEPER_CONNECTION_BASESLEEPTIME.varname, 100, TimeUnit.MILLISECONDS);
    conf.setInt(ConfVars.HIVE_ZOOKEEPER_CONNECTION_MAX_RETRIES.varname, 1);
}

From source file:org.apache.ignite.igfs.hadoop.v1.IgfsHadoopFileSystem.java

License:Apache License

/** {@inheritDoc} */
@Override// w  w  w  .  j a v  a  2  s .c o m
public void initialize(URI name, Configuration cfg) throws IOException {
    enterBusy();

    try {
        if (rmtClient != null)
            throw new IOException("File system is already initialized: " + rmtClient);

        A.notNull(name, "name");
        A.notNull(cfg, "cfg");

        super.initialize(name, cfg);

        setConf(cfg);

        String disableCacheName = String.format("fs.%s.impl.disable.cache", name.getScheme());

        cacheEnabled = !cfg.getBoolean(disableCacheName, false);

        mgmt = cfg.getBoolean(IGFS_MANAGEMENT, false);

        if (!IGFS_SCHEME.equals(name.getScheme()))
            throw new IOException("Illegal file system URI [expected=" + IGFS_SCHEME
                    + "://[name]/[optional_path], actual=" + name + ']');

        uri = name;

        uriAuthority = uri.getAuthority();

        setUser(cfg.get(MRJobConfig.USER_NAME, DFLT_USER_NAME));

        // Override sequential reads before prefetch if needed.
        seqReadsBeforePrefetch = parameter(cfg, PARAM_IGFS_SEQ_READS_BEFORE_PREFETCH, uriAuthority, 0);

        if (seqReadsBeforePrefetch > 0)
            seqReadsBeforePrefetchOverride = true;

        // In GG replication factor is controlled by data cache affinity.
        // We use replication factor to force the whole file to be stored on local node.
        dfltReplication = (short) cfg.getInt("dfs.replication", 3);

        // Get file colocation control flag.
        colocateFileWrites = parameter(cfg, PARAM_IGFS_COLOCATED_WRITES, uriAuthority, false);
        preferLocFileWrites = cfg.getBoolean(PARAM_IGFS_PREFER_LOCAL_WRITES, false);

        // Get log directory.
        String logDirCfg = parameter(cfg, PARAM_IGFS_LOG_DIR, uriAuthority, DFLT_IGFS_LOG_DIR);

        File logDirFile = U.resolveIgnitePath(logDirCfg);

        String logDir = logDirFile != null ? logDirFile.getAbsolutePath() : null;

        rmtClient = new IgfsHadoopWrapper(uriAuthority, logDir, cfg, LOG);

        // Handshake.
        IgfsHandshakeResponse handshake = rmtClient.handshake(logDir);

        igfsGrpBlockSize = handshake.blockSize();

        IgfsPaths paths = handshake.secondaryPaths();

        // Initialize client logger.
        Boolean logEnabled = parameter(cfg, PARAM_IGFS_LOG_ENABLED, uriAuthority, false);

        if (handshake.sampling() != null ? handshake.sampling() : logEnabled) {
            // Initiate client logger.
            if (logDir == null)
                throw new IOException("Failed to resolve log directory: " + logDirCfg);

            Integer batchSize = parameter(cfg, PARAM_IGFS_LOG_BATCH_SIZE, uriAuthority,
                    DFLT_IGFS_LOG_BATCH_SIZE);

            clientLog = IgfsLogger.logger(uriAuthority, handshake.igfsName(), logDir, batchSize);
        } else
            clientLog = IgfsLogger.disabledLogger();

        modeRslvr = new IgfsModeResolver(paths.defaultMode(), paths.pathModes());

        boolean initSecondary = paths.defaultMode() == PROXY;

        if (paths.pathModes() != null && !paths.pathModes().isEmpty()) {
            for (T2<IgfsPath, IgfsMode> pathMode : paths.pathModes()) {
                IgfsMode mode = pathMode.getValue();

                initSecondary |= mode == PROXY;
            }
        }

        if (initSecondary) {
            Map<String, String> props = paths.properties();

            String secUri = props.get(IgfsHadoopFileSystemWrapper.SECONDARY_FS_URI);
            String secConfPath = props.get(IgfsHadoopFileSystemWrapper.SECONDARY_FS_CONFIG_PATH);

            if (secConfPath == null)
                throw new IOException("Failed to connect to the secondary file system because configuration "
                        + "path is not provided.");

            if (secUri == null)
                throw new IOException(
                        "Failed to connect to the secondary file system because URI is not " + "provided.");

            try {
                secondaryUri = new URI(secUri);

                URL secondaryCfgUrl = U.resolveIgniteUrl(secConfPath);

                Configuration conf = new Configuration();

                if (secondaryCfgUrl != null)
                    conf.addResource(secondaryCfgUrl);

                String prop = String.format("fs.%s.impl.disable.cache", secondaryUri.getScheme());

                conf.setBoolean(prop, true);

                secondaryFs = FileSystem.get(secondaryUri, conf);
            } catch (URISyntaxException ignore) {
                if (!mgmt)
                    throw new IOException("Failed to resolve secondary file system URI: " + secUri);
                else
                    LOG.warn(
                            "Visor failed to create secondary file system (operations on paths with PROXY mode "
                                    + "will have no effect).");
            } catch (IOException e) {
                if (!mgmt)
                    throw new IOException("Failed to connect to the secondary file system: " + secUri, e);
                else
                    LOG.warn(
                            "Visor failed to create secondary file system (operations on paths with PROXY mode "
                                    + "will have no effect): " + e.getMessage());
            }
        }
    } finally {
        leaveBusy();
    }
}

From source file:org.apache.ignite.igfs.hadoop.v2.IgfsHadoopFileSystem.java

License:Apache License

/**
 * @param name URI passed to constructor.
 * @param cfg Configuration passed to constructor.
 * @throws IOException If initialization failed.
 *///w w  w .  j  ava2 s .  c o  m
private void initialize(URI name, Configuration cfg) throws IOException {
    enterBusy();

    try {
        if (rmtClient != null)
            throw new IOException("File system is already initialized: " + rmtClient);

        A.notNull(name, "name");
        A.notNull(cfg, "cfg");

        if (!IGFS_SCHEME.equals(name.getScheme()))
            throw new IOException("Illegal file system URI [expected=" + IGFS_SCHEME
                    + "://[name]/[optional_path], actual=" + name + ']');

        uriAuthority = name.getAuthority();

        // Override sequential reads before prefetch if needed.
        seqReadsBeforePrefetch = parameter(cfg, PARAM_IGFS_SEQ_READS_BEFORE_PREFETCH, uriAuthority, 0);

        if (seqReadsBeforePrefetch > 0)
            seqReadsBeforePrefetchOverride = true;

        // In GG replication factor is controlled by data cache affinity.
        // We use replication factor to force the whole file to be stored on local node.
        dfltReplication = (short) cfg.getInt("dfs.replication", 3);

        // Get file colocation control flag.
        colocateFileWrites = parameter(cfg, PARAM_IGFS_COLOCATED_WRITES, uriAuthority, false);
        preferLocFileWrites = cfg.getBoolean(PARAM_IGFS_PREFER_LOCAL_WRITES, false);

        // Get log directory.
        String logDirCfg = parameter(cfg, PARAM_IGFS_LOG_DIR, uriAuthority, DFLT_IGFS_LOG_DIR);

        File logDirFile = U.resolveIgnitePath(logDirCfg);

        String logDir = logDirFile != null ? logDirFile.getAbsolutePath() : null;

        rmtClient = new IgfsHadoopWrapper(uriAuthority, logDir, cfg, LOG);

        // Handshake.
        IgfsHandshakeResponse handshake = rmtClient.handshake(logDir);

        grpBlockSize = handshake.blockSize();

        IgfsPaths paths = handshake.secondaryPaths();

        Boolean logEnabled = parameter(cfg, PARAM_IGFS_LOG_ENABLED, uriAuthority, false);

        if (handshake.sampling() != null ? handshake.sampling() : logEnabled) {
            // Initiate client logger.
            if (logDir == null)
                throw new IOException("Failed to resolve log directory: " + logDirCfg);

            Integer batchSize = parameter(cfg, PARAM_IGFS_LOG_BATCH_SIZE, uriAuthority,
                    DFLT_IGFS_LOG_BATCH_SIZE);

            clientLog = IgfsLogger.logger(uriAuthority, handshake.igfsName(), logDir, batchSize);
        } else
            clientLog = IgfsLogger.disabledLogger();

        modeRslvr = new IgfsModeResolver(paths.defaultMode(), paths.pathModes());

        boolean initSecondary = paths.defaultMode() == PROXY;

        if (paths.pathModes() != null) {
            for (T2<IgfsPath, IgfsMode> pathMode : paths.pathModes()) {
                IgfsMode mode = pathMode.getValue();

                initSecondary |= mode == PROXY;
            }
        }

        if (initSecondary) {
            Map<String, String> props = paths.properties();

            String secUri = props.get(IgfsHadoopFileSystemWrapper.SECONDARY_FS_URI);
            String secConfPath = props.get(IgfsHadoopFileSystemWrapper.SECONDARY_FS_CONFIG_PATH);

            if (secConfPath == null)
                throw new IOException("Failed to connect to the secondary file system because configuration "
                        + "path is not provided.");

            if (secUri == null)
                throw new IOException(
                        "Failed to connect to the secondary file system because URI is not " + "provided.");

            if (secConfPath == null)
                throw new IOException("Failed to connect to the secondary file system because configuration "
                        + "path is not provided.");

            if (secUri == null)
                throw new IOException(
                        "Failed to connect to the secondary file system because URI is not " + "provided.");

            try {
                secondaryUri = new URI(secUri);

                URL secondaryCfgUrl = U.resolveIgniteUrl(secConfPath);

                if (secondaryCfgUrl == null)
                    throw new IOException("Failed to resolve secondary file system config URL: " + secConfPath);

                Configuration conf = new Configuration();

                conf.addResource(secondaryCfgUrl);

                String prop = String.format("fs.%s.impl.disable.cache", secondaryUri.getScheme());

                conf.setBoolean(prop, true);

                secondaryFs = AbstractFileSystem.get(secondaryUri, conf);
            } catch (URISyntaxException ignore) {
                throw new IOException("Failed to resolve secondary file system URI: " + secUri);
            } catch (IOException e) {
                throw new IOException("Failed to connect to the secondary file system: " + secUri, e);
            }
        }
    } finally {
        leaveBusy();
    }
}

From source file:org.apache.ignite.igfs.HadoopIgfs20FileSystemAbstractSelfTest.java

License:Apache License

/**
 * Verifies that client reconnects after connection to the server has been lost (multithreaded mode).
 *
 * @throws Exception If error occurs./*from ww w. j  a  va 2s  .  co  m*/
 */
public void testClientReconnectMultithreaded() throws Exception {
    final ConcurrentLinkedQueue<FileSystem> q = new ConcurrentLinkedQueue<>();

    Configuration cfg = new Configuration();

    for (Map.Entry<String, String> entry : primaryFsCfg)
        cfg.set(entry.getKey(), entry.getValue());

    cfg.setBoolean("fs.igfs.impl.disable.cache", true);

    final int nClients = 16;

    // Initialize clients.
    for (int i = 0; i < nClients; i++)
        q.add(FileSystem.get(primaryFsUri, cfg));

    G.stopAll(true); // Stop the server.

    startNodes(); // Start server again.

    GridTestUtils.runMultiThreaded(new Callable<Object>() {
        @Override
        public Object call() throws Exception {
            FileSystem fs = q.poll();

            try {
                // Check that client is again operational.
                assertTrue(fs.mkdirs(new Path("igfs:///" + Thread.currentThread().getName())));

                return true;
            } finally {
                U.closeQuiet(fs);
            }
        }
    }, nClients, "test-client");
}

From source file:org.apache.ignite.igfs.HadoopSecondaryFileSystemConfigurationTest.java

License:Apache License

/**
 * Create configuration for test.// w ww  . j  a v a  2 s  .  co  m
 *
 * @param skipEmbed Whether to skip embedded mode.
 * @param skipLocShmem Whether to skip local shmem mode.
 * @return Configuration.
 */
static Configuration configuration(String scheme, String authority, boolean skipEmbed, boolean skipLocShmem) {
    final Configuration cfg = new Configuration();

    if (scheme != null && authority != null)
        cfg.set("fs.defaultFS", scheme + "://" + authority + "/");

    setImplClasses(cfg);

    if (authority != null) {
        if (skipEmbed)
            cfg.setBoolean(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_NO_EMBED, authority), true);

        if (skipLocShmem)
            cfg.setBoolean(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_NO_LOCAL_SHMEM, authority), true);
    }

    return cfg;
}

From source file:org.apache.ignite.igfs.IgfsHadoopFileSystemAbstractSelfTest.java

License:Apache License

/**
 * Test how IPC cache map works./*w w w  .  jav  a2 s  . c om*/
 *
 * @throws Exception If failed.
 */
public void testIpcCache() throws Exception {
    IgfsHadoopEx hadoop = GridTestUtils.getFieldValue(fs, "rmtClient", "delegateRef", "value", "hadoop");

    if (hadoop instanceof IgfsHadoopOutProc) {
        FileSystem fsOther = null;

        try {
            Field field = IgfsHadoopIpcIo.class.getDeclaredField("ipcCache");

            field.setAccessible(true);

            Map<String, IgfsHadoopIpcIo> cache = (Map<String, IgfsHadoopIpcIo>) field.get(null);

            Configuration cfg = configuration(PRIMARY_AUTHORITY, skipEmbed, skipLocShmem);

            // we disable caching in order to obtain new FileSystem instance.
            cfg.setBoolean("fs.igfs.impl.disable.cache", true);

            // Initial cache size.
            int initSize = cache.size();

            // Ensure that when IO is used by multiple file systems and one of them is closed, IO is not stopped.
            fsOther = FileSystem.get(new URI(PRIMARY_URI), cfg);

            assert fs != fsOther;

            assertEquals(initSize, cache.size());

            fsOther.close();

            assertEquals(initSize, cache.size());

            Field stopField = IgfsHadoopIpcIo.class.getDeclaredField("stopping");

            stopField.setAccessible(true);

            IgfsHadoopIpcIo io = null;

            for (Map.Entry<String, IgfsHadoopIpcIo> ioEntry : cache.entrySet()) {
                if (endpoint.contains(ioEntry.getKey())) {
                    io = ioEntry.getValue();

                    break;
                }
            }

            assert io != null;

            assert !(Boolean) stopField.get(io);

            // Ensure that IO is stopped when nobody else is need it.
            fs.close();

            assertEquals(initSize - 1, cache.size());

            assert (Boolean) stopField.get(io);
        } finally {
            U.closeQuiet(fsOther);
        }
    }
}

From source file:org.apache.ignite.igfs.IgfsHadoopFileSystemAbstractSelfTest.java

License:Apache License

/**
 * Verifies that client reconnects after connection to the server has been lost (multithreaded mode).
 *
 * @throws Exception If error occurs./*from  ww  w . j  a  v a  2 s . co m*/
 */
public void testClientReconnectMultithreaded() throws Exception {
    final ConcurrentLinkedQueue<FileSystem> q = new ConcurrentLinkedQueue<>();

    Configuration cfg = new Configuration();

    for (Map.Entry<String, String> entry : primaryFsCfg)
        cfg.set(entry.getKey(), entry.getValue());

    cfg.setBoolean("fs.igfs.impl.disable.cache", true);

    final int nClients = 1;

    // Initialize clients.
    for (int i = 0; i < nClients; i++)
        q.add(FileSystem.get(primaryFsUri, cfg));

    G.stopAll(true); // Stop the server.

    startNodes(); // Start server again.

    GridTestUtils.runMultiThreaded(new Callable<Object>() {
        @Override
        public Object call() throws Exception {
            FileSystem fs = q.poll();

            try {
                // Check that client is again operational.
                assertTrue(fs.mkdirs(new Path("/" + Thread.currentThread().getName())));

                return true;
            } finally {
                U.closeQuiet(fs);
            }
        }
    }, nClients, "test-client");
}

From source file:org.apache.ignite.igfs.IgfsHadoopFileSystemAbstractSelfTest.java

License:Apache License

/**
 * Create configuration for test.//from w ww  .  ja v  a  2s  .c  o  m
 *
 * @param authority Authority.
 * @param skipEmbed Whether to skip embedded mode.
 * @param skipLocShmem Whether to skip local shmem mode.
 * @return Configuration.
 */
private static Configuration configuration(String authority, boolean skipEmbed, boolean skipLocShmem) {
    Configuration cfg = new Configuration();

    cfg.set("fs.defaultFS", "igfs://" + authority + "/");
    cfg.set("fs.igfs.impl", IgfsHadoopFileSystem.class.getName());
    cfg.set("fs.AbstractFileSystem.igfs.impl",
            org.apache.ignite.igfs.hadoop.v2.IgfsHadoopFileSystem.class.getName());

    cfg.setBoolean("fs.igfs.impl.disable.cache", true);

    if (skipEmbed)
        cfg.setBoolean(String.format(IgfsHadoopUtils.PARAM_IGFS_ENDPOINT_NO_EMBED, authority), true);

    if (skipLocShmem)
        cfg.setBoolean(String.format(IgfsHadoopUtils.PARAM_IGFS_ENDPOINT_NO_LOCAL_SHMEM, authority), true);

    return cfg;
}