Example usage for org.apache.hadoop.conf Configuration setBoolean

List of usage examples for org.apache.hadoop.conf Configuration setBoolean

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.conf.*
  5. Configuration
  6. setBoolean

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration setBoolean.

Prototype

public void setBoolean(String name, boolean value)

Source Link

Document

Set the value of the name property to a boolean.

Usage

From source file:org.apache.eagle.jpm.util.HDFSUtil.java

License:Apache License

public static void login(Configuration kConfig) throws IOException {
    if (kConfig.get("hdfs.kerberos.principal") == null || kConfig.get("hdfs.kerberos.principal").isEmpty()) {
        if (kConfig.get("hadoop.job.ugi") != null) {
            System.setProperty("HADOOP_USER_NAME", kConfig.get("hadoop.job.ugi"));
        }//from   w w  w . j a va  2  s  . c om
        return;
    }
    kConfig.setBoolean("hadoop.security.authorization", true);
    kConfig.set("hadoop.security.authentication", "kerberos");
    UserGroupInformation.setConfiguration(kConfig);
    UserGroupInformation.loginUserFromKeytab(kConfig.get("hdfs.kerberos.principal"),
            kConfig.get("hdfs.keytab.file"));
}

From source file:org.apache.eagle.security.util.HadoopSecurityUtil.java

License:Apache License

public static void login(Configuration kConfig) throws IOException {
    if (kConfig.get(EAGLE_KEYTAB_FILE_KEY) == null || kConfig.get(EAGLE_USER_NAME_KEY) == null)
        return;/*from  w w  w. ja  v  a 2  s .  co m*/

    kConfig.setBoolean("hadoop.security.authorization", true);
    kConfig.set("hadoop.security.authentication", "kerberos");
    UserGroupInformation.setConfiguration(kConfig);
    UserGroupInformation.loginUserFromKeytab(kConfig.get(EAGLE_USER_NAME_KEY),
            kConfig.get(EAGLE_KEYTAB_FILE_KEY));
}

From source file:org.apache.falcon.regression.core.helpers.entity.AbstractEntityHelper.java

License:Apache License

public FileSystem getHadoopFS() throws IOException {
    if (null == this.hadoopFS) {
        Configuration conf = new Configuration();
        conf.setBoolean("fs.hdfs.impl.disable.cache", true);
        conf.set("fs.default.name", "hdfs://" + this.hadoopURL);
        this.hadoopFS = FileSystem.get(conf);
    }//from  w w  w.j  a v a  2s. c  o  m
    return this.hadoopFS;
}

From source file:org.apache.falcon.regression.ExternalFSTest.java

License:Apache License

@BeforeClass
public void setUpClass() throws IOException {
    HadoopUtil.recreateDir(clusterFS, baseTestDir);
    Configuration conf = new Configuration();
    conf.set("fs.defaultFS", WASB_END_POINT);
    conf.set("fs.azure.account.key." + MerlinConstants.WASB_ACCOUNT, MerlinConstants.WASB_SECRET);
    conf.setBoolean("fs.hdfs.impl.disable.cache", false);
    wasbFS = FileSystem.get(conf);
    LOGGER.info("creating base wasb dir" + baseWasbDir);
}

From source file:org.apache.falcon.workflow.util.OozieActionConfigurationHelper.java

License:Apache License

public static Configuration createActionConf() throws IOException {
    Configuration conf = new Configuration();
    Path confPath = new Path("file:///" + System.getProperty("oozie.action.conf.xml"));

    final boolean actionConfExists = confPath.getFileSystem(conf).exists(confPath);
    LOG.info("Oozie Action conf {} found ? {}", confPath, actionConfExists);
    if (actionConfExists) {
        LOG.info("Oozie Action conf found, adding path={}, conf={}", confPath, conf.toString());
        conf.addResource(confPath);// w ww. j  a va2s  .com
        dumpConf(conf, "oozie action conf ");
    }

    String tokenFile = System.getenv("HADOOP_TOKEN_FILE_LOCATION");
    if (tokenFile != null) {
        if (Shell.WINDOWS) {
            if (tokenFile.charAt(0) == '"') {
                tokenFile = tokenFile.substring(1);
            }
            if (tokenFile.charAt(tokenFile.length() - 1) == '"') {
                tokenFile = tokenFile.substring(0, tokenFile.length() - 1);
            }
        }

        conf.set("mapreduce.job.credentials.binary", tokenFile);
        System.setProperty("mapreduce.job.credentials.binary", tokenFile);
        conf.set("tez.credentials.path", tokenFile);
        System.setProperty("tez.credentials.path", tokenFile);
    }

    conf.set("datanucleus.plugin.pluginRegistryBundleCheck", "LOG");
    conf.setBoolean("hive.exec.mode.local.auto", false);

    return conf;
}

From source file:org.apache.flume.sink.customhdfs.BucketWriter.java

License:Apache License

/**
 * open() is called by append()// ww w  . j  a v  a 2 s . c  o  m
 * @throws IOException
 * @throws InterruptedException
 */
private void open() throws IOException, InterruptedException {
    if ((filePath == null) || (writer == null)) {
        throw new IOException("Invalid file settings");
    }

    final Configuration config = new Configuration();
    // disable FileSystem JVM shutdown hook
    config.setBoolean("fs.automatic.close", false);

    // Hadoop is not thread safe when doing certain RPC operations,
    // including getFileSystem(), when running under Kerberos.
    // open() must be called by one thread at a time in the JVM.
    // NOTE: tried synchronizing on the underlying Kerberos principal previously
    // which caused deadlocks. See FLUME-1231.
    synchronized (staticLock) {
        checkAndThrowInterruptedException();

        try {
            long counter = fileExtensionCounter.incrementAndGet();

            String fullFileName = fileName + "." + counter;

            if (fileSuffix != null && fileSuffix.length() > 0) {
                fullFileName += fileSuffix;
            } else if (codeC != null) {
                fullFileName += codeC.getDefaultExtension();
            }

            bucketPath = filePath + "/" + inUsePrefix + fullFileName + inUseSuffix;
            targetPath = filePath + "/" + fullFileName;

            LOG.info("Creating " + bucketPath);
            callWithTimeout(new CallRunner<Void>() {
                @Override
                public Void call() throws Exception {
                    if (codeC == null) {
                        // Need to get reference to FS using above config before underlying
                        // writer does in order to avoid shutdown hook &
                        // IllegalStateExceptions
                        if (!mockFsInjected) {
                            fileSystem = new Path(bucketPath).getFileSystem(config);
                        }
                        writer.open(bucketPath);
                    } else {
                        // need to get reference to FS before writer does to
                        // avoid shutdown hook
                        if (!mockFsInjected) {
                            fileSystem = new Path(bucketPath).getFileSystem(config);
                        }
                        writer.open(bucketPath, codeC, compType);
                    }
                    return null;
                }
            });
        } catch (Exception ex) {
            sinkCounter.incrementConnectionFailedCount();
            if (ex instanceof IOException) {
                throw (IOException) ex;
            } else {
                throw Throwables.propagate(ex);
            }
        }
    }
    isClosedMethod = getRefIsClosed();
    sinkCounter.incrementConnectionCreatedCount();
    resetCounters();

    // if time-based rolling is enabled, schedule the roll
    if (rollInterval > 0) {
        Callable<Void> action = new Callable<Void>() {
            public Void call() throws Exception {
                LOG.debug("Rolling file ({}): Roll scheduled after {} sec elapsed.", bucketPath, rollInterval);
                try {
                    // Roll the file and remove reference from sfWriters map.
                    close(true);
                } catch (Throwable t) {
                    LOG.error("Unexpected error", t);
                }
                return null;
            }
        };
        timedRollFuture = timedRollerPool.schedule(action, rollInterval, TimeUnit.SECONDS);
    }

    isOpen = true;
}

From source file:org.apache.flume.sink.hdfs.BucketWriter.java

License:Apache License

/**
 * open() is called by append()/*from  w w w.  j  a  v  a  2 s  .c o  m*/
 * @throws IOException
 * @throws InterruptedException
 */
private void open() throws IOException, InterruptedException {
    if ((filePath == null) || (writer == null)) {
        throw new IOException("Invalid file settings");
    }

    final Configuration config = new Configuration();
    // disable FileSystem JVM shutdown hook
    config.setBoolean("fs.automatic.close", false);

    // Hadoop is not thread safe when doing certain RPC operations,
    // including getFileSystem(), when running under Kerberos.
    // open() must be called by one thread at a time in the JVM.
    // NOTE: tried synchronizing on the underlying Kerberos principal previously
    // which caused deadlocks. See FLUME-1231.
    synchronized (staticLock) {
        checkAndThrowInterruptedException();

        try {
            long counter = fileExtensionCounter.incrementAndGet();

            String fullFileName = null;
            LOG.info("\n---inFileNameUseTimeStamp " + inFileNameUseTimeStamp + "\n");
            if (inFileNameUseTimeStamp) {
                fullFileName = fileName + "." + counter;
                LOG.info("\n---fullFileName:if " + fullFileName + "\n");
            } else {
                fullFileName = fileName;
                LOG.info("\n---fullFileName:else " + fullFileName + "\n");
            }
            LOG.info("\n---fullFileName: " + fullFileName + "\n");
            if (fileSuffix != null && fileSuffix.length() > 0) {
                fullFileName += fileSuffix;
            } else if (codeC != null) {
                fullFileName += codeC.getDefaultExtension();
            }

            bucketPath = filePath + "/" + inUsePrefix + fullFileName + inUseSuffix;
            targetPath = filePath + "/" + fullFileName;

            LOG.info("Creating " + bucketPath);
            callWithTimeout(new CallRunner<Void>() {
                @Override
                public Void call() throws Exception {
                    if (codeC == null) {
                        // Need to get reference to FS using above config before underlying
                        // writer does in order to avoid shutdown hook &
                        // IllegalStateExceptions
                        if (!mockFsInjected) {
                            fileSystem = new Path(bucketPath).getFileSystem(config);
                        }
                        writer.open(bucketPath);
                    } else {
                        // need to get reference to FS before writer does to
                        // avoid shutdown hook
                        if (!mockFsInjected) {
                            fileSystem = new Path(bucketPath).getFileSystem(config);
                        }
                        writer.open(bucketPath, codeC, compType);
                    }
                    return null;
                }
            });
        } catch (Exception ex) {
            sinkCounter.incrementConnectionFailedCount();
            if (ex instanceof IOException) {
                throw (IOException) ex;
            } else {
                throw Throwables.propagate(ex);
            }
        }
    }
    isClosedMethod = getRefIsClosed();
    sinkCounter.incrementConnectionCreatedCount();
    resetCounters();

    // if time-based rolling is enabled, schedule the roll
    if (rollInterval > 0) {
        Callable<Void> action = new Callable<Void>() {
            public Void call() throws Exception {
                LOG.debug("Rolling file ({}): Roll scheduled after {} sec elapsed.", bucketPath, rollInterval);
                try {
                    // Roll the file and remove reference from sfWriters map.
                    close(true);
                } catch (Throwable t) {
                    LOG.error("Unexpected error", t);
                }
                return null;
            }
        };
        timedRollFuture = timedRollerPool.schedule(action, rollInterval, TimeUnit.SECONDS);
    }

    isOpen = true;
}

From source file:org.apache.giraffa.TestLeaseManagement.java

License:Apache License

@BeforeClass
public static void beforeClass() throws Exception {
    System.setProperty(HBaseCommonTestingUtility.BASE_TEST_DIRECTORY_KEY, GiraffaTestUtils.BASE_TEST_DIRECTORY);
    Configuration hbaseConf = UTIL.getConfiguration();
    hbaseConf.setInt("hbase.assignment.maximum.attempts", 3);
    // Put meta on master to avoid meta server shutdown handling
    hbaseConf.set("hbase.balancer.tablesOnMaster", "hbase:meta");
    hbaseConf.setInt("hbase.master.maximum.ping.server.attempts", 3);
    hbaseConf.setInt("hbase.master.ping.server.retry.sleep.interval", 1);
    hbaseConf.setBoolean("hbase.assignment.usezk", false);
    hbaseConf.setBoolean(DFS_PERMISSIONS_ENABLED_KEY, false);
    GiraffaTestUtils.enableMultipleUsers();
    UTIL.startMiniCluster(1);//from  w  ww  .  j av a  2s  .  c  o m
}

From source file:org.apache.giraph.lib.TestIdWithValueTextOutputFormat.java

License:Apache License

@Test
public void testReverseIdAndValue() throws IOException, InterruptedException {
    Configuration conf = new Configuration();
    conf.setBoolean(REVERSE_ID_AND_VALUE, true);
    Text expected = new Text("4.0\tFour Tops");

    IdWithValueTestWorker(conf, expected);
}

From source file:org.apache.giraph.utils.MasterLoggingAggregator.java

License:Apache License

/**
 * Set whether or not master logging aggregator should be used
 *
 * @param useMasterLoggingAggregator Whether or not we want
 *                                   master logging aggregator to be used
 * @param conf                       Configuration
 *///from  w  ww  .j a  v  a  2s  . co m
public static void setUseMasterLoggingAggregator(boolean useMasterLoggingAggregator, Configuration conf) {
    conf.setBoolean(USE_MASTER_LOGGING_AGGREGATOR, useMasterLoggingAggregator);
}