Example usage for org.apache.hadoop.conf Configuration setInt

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration setInt.

Prototype

public void setInt(String name, int value) 

Source Link

Document

Set the value of the name property to an int.

Usage

From source file:com.flipkart.aesop.runtime.producer.hbase.HBaseEventProducer.java

License:Apache License

/**
 * Interface method implementation. Starts up the SEP consumer
 * @see com.linkedin.databus2.producers.EventProducer#start(long)
 *//*from   w w  w.  j  a v  a  2 s  . c o m*/
public void start(long sinceSCN) {
    shutdownRequested.set(false);
    this.sinceSCN.set(sinceSCN);
    LOGGER.info("Starting SEP subscription : " + this.getName());
    LOGGER.info("ZK quorum hosts : " + this.zkQuorum);
    LOGGER.info("ZK client port : " + this.zkClientPort);
    LOGGER.info("ZK session timeout : " + this.zkSessionTimeout);
    LOGGER.info("RPC timeout : " + this.rpcTimeout);
    LOGGER.info("Using hostname to bind to : " + this.localHost);
    LOGGER.info("Using worker threads : " + this.workerThreads);
    LOGGER.info("Listening to WAL edits from : " + this.sinceSCN);
    try {
        Configuration hbaseConf = HBaseConfiguration.create();
        // enable replication to get WAL edits
        hbaseConf.setBoolean(HBASE_REPLICATION_CONFIG, true);
        // need to explicitly set the ZK host and port details - hosts separated from port - see SepModelImpl constructor source code
        hbaseConf.set(ZK_QUORUM_CONFIG, this.zkQuorum);
        hbaseConf.setInt(ZK_CLIENT_PORT_CONFIG, this.zkClientPort);
        hbaseConf.setInt(RPC_TIMEOUT_CONFIG, this.rpcTimeout);

        StringBuilder zkQuorumWithPort = new StringBuilder();
        String[] zkHostsList = this.zkQuorum.split(",");
        for (String zkHost : zkHostsList) {
            zkQuorumWithPort.append(zkHost);
            zkQuorumWithPort.append(":");
            zkQuorumWithPort.append(this.zkClientPort);
            zkQuorumWithPort.append(",");
        }

        LOGGER.info("ZK util connect string (host:port) : " + zkQuorumWithPort.toString());
        ZooKeeperItf zk = ZkUtil.connect(zkQuorumWithPort.toString(), this.zkSessionTimeout);

        StringBuilder hbaseConfBuilder = new StringBuilder();
        Iterator<Entry<String, String>> it = hbaseConf.iterator();
        while (it.hasNext()) {
            Entry<String, String> entry = it.next();
            if (entry.getKey().equalsIgnoreCase(HBASE_REPLICATION_CONFIG)
                    || entry.getKey().equalsIgnoreCase(ZK_QUORUM_CONFIG)
                    || entry.getKey().equalsIgnoreCase(ZK_CLIENT_PORT_CONFIG)) {
                hbaseConfBuilder.append(entry.getKey());
                hbaseConfBuilder.append(":");
                hbaseConfBuilder.append(entry.getValue());
                hbaseConfBuilder.append(",");
            }
        }
        LOGGER.info("SEP Model Hbase configuration = " + hbaseConfBuilder.toString());
        SepModel sepModel = new SepModelImpl(zk, hbaseConf);

        final String subscriptionName = this.getName();

        if (!sepModel.hasSubscription(subscriptionName)) {
            sepModel.addSubscriptionSilent(subscriptionName);
        }
        this.sepConsumer = new SepConsumer(subscriptionName, generateSEPSCN(this.sinceSCN.get()),
                new RelayAppender(), this.workerThreads, this.localHost, zk, hbaseConf);
        this.sepConsumer.start();
    } catch (Exception e) {
        LOGGER.error(
                "Error starting WAL edits consumer. Producer not started!. Error message : " + e.getMessage(),
                e);
    }
}

---

From source file:com.flipkart.foxtrot.core.datastore.impl.hbase.HBaseUtil.java

License:Apache License

public static Configuration create(final HbaseConfig hbaseConfig) throws IOException {
    Configuration configuration = HBaseConfiguration.create();

    if (isValidFile(hbaseConfig.getCoreSite())) {
        configuration.addResource(new File(hbaseConfig.getCoreSite()).toURI().toURL());
    }/*from  ww w  .j a  v a  2  s.  c  o m*/

    if (isValidFile(hbaseConfig.getHdfsSite())) {
        configuration.addResource(new File(hbaseConfig.getHdfsSite()).toURI().toURL());
    }

    if (isValidFile(hbaseConfig.getHbasePolicy())) {
        configuration.addResource(new File(hbaseConfig.getHbasePolicy()).toURI().toURL());
    }

    if (isValidFile(hbaseConfig.getHbaseSite())) {
        configuration.addResource(new File(hbaseConfig.getHbaseSite()).toURI().toURL());
    }

    if (hbaseConfig.isSecure() && isValidFile(hbaseConfig.getKeytabFileName())) {
        configuration.set("hbase.master.kerberos.principal", hbaseConfig.getAuthString());
        configuration.set("hadoop.kerberos.kinit.command", hbaseConfig.getKinitPath());
        UserGroupInformation.setConfiguration(configuration);
        System.setProperty("java.security.krb5.conf", hbaseConfig.getKerberosConfigFile());
        UserGroupInformation.loginUserFromKeytab(hbaseConfig.getAuthString(), hbaseConfig.getKeytabFileName());
        logger.info("Logged into Hbase with User: " + UserGroupInformation.getLoginUser());
    }

    if (null != hbaseConfig.getHbaseZookeeperQuorum()) {
        configuration.set("hbase.zookeeper.quorum", hbaseConfig.getHbaseZookeeperQuorum());
    }

    if (null != hbaseConfig.getHbaseZookeeperClientPort()) {
        configuration.setInt("hbase.zookeeper.property.clientPort", hbaseConfig.getHbaseZookeeperClientPort());
    }
    return configuration;
}

---

From source file:com.github.charithe.hbase.HBaseMiniClusterBooter.java

License:Apache License

private Configuration updateConfiguration(String zookeeperQuorum, int zkPort) throws IOException {
    LOGGER.debug("Updating configuration to use random ports and disable UIs");

    Configuration myConf = new Configuration(conf);
    myConf.setInt(HConstants.MASTER_PORT, getFreePort());
    myConf.setInt(HConstants.REGIONSERVER_PORT, getFreePort());
    myConf.setInt("hbase.master.info.port", -1);
    myConf.setInt("hbase.regionserver.info.port", -1);
    myConf.setBoolean("hbase.replication", false);

    myConf.setInt(HConstants.ZOOKEEPER_MAX_CLIENT_CNXNS, 80);
    myConf.set(HConstants.ZOOKEEPER_QUORUM, zookeeperQuorum);
    myConf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, zkPort);

    return myConf;
}

---

From source file:com.github.ygf.pagerank.InLinks.java

License:Apache License

public int run(String[] args) throws Exception {
    if (args.length != 3) {
        System.out.println("Usage: InLinks <links-simple-sorted.txt> <titles-dir> <output-dir>");
        ToolRunner.printGenericCommandUsage(System.out);
        return 2;
    }/*from   ww w  .java 2 s.co  m*/

    Path linksFile = new Path(args[0]);
    Path titlesDir = new Path(args[1]);
    Path outputDir = new Path(args[2]);

    Configuration conf = getConf();

    // Do not create _SUCCESS files. MapFileOutputFormat.getReaders calls
    // try to read the _SUCCESS as another MapFile dir.
    conf.set("mapreduce.fileoutputcommitter.marksuccessfuljobs", "false");

    // Default values of the parameters of the algorithm.
    conf.setInt("inlinks.top_results", conf.getInt("inlinks.top_results", 100));

    conf.set("inlinks.titles_dir", titlesDir.toString());

    computeInLinks(conf, linksFile, outputDir);
    summarizeResults(conf, outputDir);

    return 0;
}

---

From source file:com.github.ygf.pagerank.PageRank.java

License:Apache License

public int run(String[] args) throws Exception {
    if (args.length != 3) {
        System.out.println("Usage: PageRank <links-simple-sorted.txt> <titles-dir> <output-dir>");
        ToolRunner.printGenericCommandUsage(System.out);
        return 2;
    }/*from   w  ww . j  a  v  a2s.co m*/

    Path linksFile = new Path(args[0]);
    Path titlesDir = new Path(args[1]);
    Path outputDir = new Path(args[2]);

    Configuration conf = getConf();

    // Do not create _SUCCESS files. MapFileOutputFormat.getReaders calls
    // try to read the _SUCCESS as another MapFile dir.
    conf.set("mapreduce.fileoutputcommitter.marksuccessfuljobs", "false");

    // Default values of the parameters of the algorithm.
    conf.setInt("pagerank.block_size", conf.getInt("pagerank.block_size", 10000));
    conf.setInt("pagerank.max_iterations", conf.getInt("pagerank.max_iterations", 2));
    conf.setFloat("pagerank.damping_factor", conf.getFloat("pagerank.damping_factor", 0.85f));
    conf.setInt("pagerank.top_results", conf.getInt("pagerank.top_results", 100));

    conf.set("pagerank.titles_dir", titlesDir.toString());
    int numPages = getNumPages(conf, titlesDir);
    conf.setLong("pagerank.num_pages", numPages);

    createTransitionMatrix(conf, linksFile, outputDir);

    int maxIters = Integer.parseInt(conf.get("pagerank.max_iterations"));
    for (int iter = 1; iter <= maxIters; iter++) {
        conf.setInt("pagerank.iteration", iter);
        pageRankIteration(iter, conf, outputDir);
        cleanPreviousIteration(iter, conf, outputDir);
    }

    summarizeResults(maxIters, conf, outputDir);

    return 0;
}

---

From source file:com.google.appengine.tools.mapreduce.BlobstoreRecordReaderTest.java

License:Apache License

private BlobstoreRecordReader createRecordReader(BlobstoreInputSplit split,
        BlobstoreRecordReader.InputStreamFactory inputStreamFactory, byte[] persistedState)
        throws IOException, InterruptedException {
    BlobstoreRecordReader recordReader = new BlobstoreRecordReader();
    recordReader.setInputStreamFactory(inputStreamFactory);
    Configuration configuration = new Configuration();
    configuration.setInt(BlobstoreInputFormat.TERMINATOR, -1);
    recordReader.initialize(split, new TaskAttemptContext(configuration, new TaskAttemptID()));
    if (persistedState != null) {
        recordReader.readFields(new DataInputStream(new ByteArrayInputStream(persistedState)));
    }//w ww  .ja v a2s .c  o  m
    return recordReader;
}

---

From source file:com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystemIntegrationTest.java

License:Open Source License

/**
 * Validates success path in initialize().
 *///from ww  w  .j av  a 2 s . c o  m
@Test
@Override
public void testInitializeSuccess() throws IOException, URISyntaxException {
    GoogleHadoopFileSystem fs = null;

    // Reuse loadConfig() to initialize auth related settings.
    Configuration config = loadConfig();

    // Set up remaining settings to known test values.
    int bufferSize = 512;
    config.setInt(GoogleHadoopFileSystemBase.BUFFERSIZE_KEY, bufferSize);
    long blockSize = 1024;
    config.setLong(GoogleHadoopFileSystemBase.BLOCK_SIZE_KEY, blockSize);
    String systemBucketName = ghfsHelper.getUniqueBucketName("-system-bucket");
    String rootBucketName = ghfsHelper.getUniqueBucketName("-root-bucket");
    config.set(GoogleHadoopFileSystemBase.GCS_SYSTEM_BUCKET_KEY, systemBucketName);

    URI initUri = (new Path("gs://" + rootBucketName)).toUri();
    try {
        fs = new GoogleHadoopFileSystem();
        fs.initialize(initUri, config);
    } catch (IOException e) {
        Assert.fail("Unexpected exception");
    }

    // Verify that config settings were set correctly.
    Assert.assertEquals(bufferSize, fs.getBufferSizeOverride());
    Assert.assertEquals(blockSize, fs.getDefaultBlockSize());
    Assert.assertEquals(systemBucketName, fs.getSystemBucketName());
    Assert.assertEquals(initUri, fs.initUri);
    Assert.assertEquals(rootBucketName, fs.getRootBucketName());

    initUri = (new Path("gs:/foo")).toUri();
    try {
        fs = new GoogleHadoopFileSystem();
        fs.initialize(initUri, config);
    } catch (IOException e) {
        Assert.fail("Unexpected exception");
    }

    // Verify that config settings were set correctly.
    Assert.assertEquals(bufferSize, fs.getBufferSizeOverride());
    Assert.assertEquals(blockSize, fs.getDefaultBlockSize());
    Assert.assertEquals(systemBucketName, fs.getSystemBucketName());
    Assert.assertEquals(initUri, fs.initUri);
    Assert.assertEquals(systemBucketName, fs.getRootBucketName());
}

---

From source file:com.google.cloud.hadoop.fs.gcs.GoogleHadoopGlobalRootedFileSystemIntegrationTest.java

License:Open Source License

/**
 * Validates success path in initialize().
 *//*  w ww.j a v a  2  s .c o m*/
@Test
@Override
public void testInitializeSuccess() throws IOException, URISyntaxException {
    GoogleHadoopFileSystemBase fs = null;

    // Reuse loadConfig() to initialize auth related settings.
    Configuration config = loadConfig();

    // Set up remaining settings to known test values.
    int bufferSize = 512;
    config.setInt(GoogleHadoopFileSystemBase.BUFFERSIZE_KEY, bufferSize);
    long blockSize = 1024;
    config.setLong(GoogleHadoopFileSystemBase.BLOCK_SIZE_KEY, blockSize);
    String systemBucketName = ghfsHelper.getUniqueBucketName("-system-bucket");
    config.set(GoogleHadoopFileSystemBase.GCS_SYSTEM_BUCKET_KEY, systemBucketName);

    URI initUri = (new Path("gsg://bucket-should-be-ignored")).toUri();
    try {
        fs = new GoogleHadoopGlobalRootedFileSystem();
        fs.initialize(initUri, config);
    } catch (IOException e) {
        Assert.fail("Unexpected exception");
    }

    // Verify that config settings were set correctly.
    Assert.assertEquals(bufferSize, fs.getBufferSizeOverride());
    Assert.assertEquals(blockSize, fs.getDefaultBlockSize());
    Assert.assertEquals(systemBucketName, fs.getSystemBucketName());
    Assert.assertEquals(initUri, fs.initUri);
}

---

From source file:com.hortonworks.hbase.replication.bridge.HBaseClient.java

License:Apache License

/**
 * set the ping interval value in configuration
 *
 * @param conf Configuration/*ww w  . ja v a2  s  .  c om*/
 * @param pingInterval the ping interval
 */
public static void setPingInterval(Configuration conf, int pingInterval) {
    conf.setInt(PING_INTERVAL_NAME, pingInterval);
}

---

From source file:com.hortonworks.hbase.replication.bridge.HBaseClient.java

License:Apache License

/**
 * Set the socket timeout/*from w  w  w. j ava2 s. c  om*/
 * @param conf Configuration
 * @param socketTimeout the socket timeout
 */
public static void setSocketTimeout(Configuration conf, int socketTimeout) {
    conf.setInt(SOCKET_TIMEOUT, socketTimeout);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »