Example usage for org.apache.hadoop.conf Configuration getInt

List of usage examples for org.apache.hadoop.conf Configuration getInt

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.conf.*
  5. Configuration
  6. getInt

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration getInt.

Prototype

public int getInt(String name, int defaultValue)

Source Link

Document

Get the value of the name property as an int.

Usage

From source file:com.mycompany.keywordsearch.LineRecordReaderV2.java

public void initialize(InputSplit genericSplit, TaskAttemptContext context) throws IOException {
    FileSplit split = (FileSplit) genericSplit;
    Configuration job = context.getConfiguration();
    this.maxLineLength = job.getInt(MAX_LINE_LENGTH, Integer.MAX_VALUE);
    start = split.getStart();//from ww w.j a v a2s. com
    end = start + split.getLength();
    final Path file = split.getPath();

    // open the file and seek to the start of the split
    final FileSystem fs = file.getFileSystem(job);
    fileIn = fs.open(file);
    locationKey.set(file.toString());
    CompressionCodec codec = new CompressionCodecFactory(job).getCodec(file);
    if (null != codec) {
        isCompressedInput = true;
        decompressor = CodecPool.getDecompressor(codec);
        if (codec instanceof SplittableCompressionCodec) {
            final SplitCompressionInputStream cIn = ((SplittableCompressionCodec) codec).createInputStream(
                    fileIn, decompressor, start, end, SplittableCompressionCodec.READ_MODE.BYBLOCK);
            in = new CompressedSplitLineReader(cIn, job, this.recordDelimiterBytes);
            start = cIn.getAdjustedStart();
            end = cIn.getAdjustedEnd();
            filePosition = cIn;
        } else {
            in = new SplitLineReader(codec.createInputStream(fileIn, decompressor), job,
                    this.recordDelimiterBytes);
            filePosition = fileIn;
        }
    } else {
        fileIn.seek(start);
        in = new SplitLineReader(fileIn, job, this.recordDelimiterBytes);
        filePosition = fileIn;
    }
    // If this is not the first split, we always throw away first record
    // because we always (except the last split) read one extra line in
    // next() method.
    if (start != 0) {
        start += in.readLine(new Text(), 0, maxBytesToConsume(start));
    }
    this.pos = start;
}

From source file:com.navercorp.pinpoint.common.hbase.HBaseAsyncOperationFactory.java

License:Apache License

public static HBaseAsyncOperation create(Configuration configuration) throws IOException {
    boolean enableAsyncMethod = configuration.getBoolean(ENABLE_ASYNC_METHOD, DEFAULT_ENABLE_ASYNC_METHOD);
    if (!enableAsyncMethod) {
        return DisabledHBaseAsyncOperation.INSTANCE;
    }//from  www  .  j  a  v  a 2s  .  com

    int queueSize = configuration.getInt(ASYNC_IN_QUEUE_SIZE, DEFAULT_ASYNC_IN_QUEUE_SIZE);

    if (configuration.get(ASYNC_PERIODIC_FLUSH_TIME, null) == null) {
        configuration.setInt(ASYNC_PERIODIC_FLUSH_TIME, DEFAULT_ASYNC_PERIODIC_FLUSH_TIME);
    }

    if (configuration.get(ASYNC_RETRY_COUNT, null) == null) {
        configuration.setInt(ASYNC_RETRY_COUNT, DEFAULT_ASYNC_RETRY_COUNT);
    }

    return new HBaseAsyncTemplate(configuration, queueSize);
}

From source file:com.nearinfinity.honeycomb.hbase.HTableProvider.java

License:Apache License

public HTableProvider(final Configuration configuration) {
    String hTableName = configuration.get(ConfigConstants.TABLE_NAME);
    long writeBufferSize = configuration.getLong(ConfigConstants.WRITE_BUFFER,
            ConfigConstants.DEFAULT_WRITE_BUFFER);
    int poolSize = configuration.getInt(ConfigConstants.TABLE_POOL_SIZE,
            ConfigConstants.DEFAULT_TABLE_POOL_SIZE);
    boolean autoFlush = configuration.getBoolean(ConfigConstants.AUTO_FLUSH,
            ConfigConstants.DEFAULT_AUTO_FLUSH);

    tableName = hTableName;//from w  w w  . j  ava2s .  c om
    tablePool = new HTablePool(configuration, poolSize, new HTableFactory(writeBufferSize, autoFlush));
}

From source file:com.netflix.bdp.s3mper.metastore.impl.DynamoDBMetastore.java

License:Apache License

/**
 * Creates the metastore table in DynamoDB if it doesn't exist with the configured
 * read and write units.//ww  w . java2 s.com
 * 
 * @param uri
 * @param conf
 * @throws Exception 
 */
@Override
public void initalize(URI uri, Configuration conf) throws Exception {
    scheme = uri.getScheme();

    String keyId = conf.get("fs." + uri.getScheme() + ".awsAccessKeyId");
    String keySecret = conf.get("fs." + uri.getScheme() + ".awsSecretAccessKey");

    //An override option for accessing across accounts
    keyId = conf.get("s3mper.override.awsAccessKeyId", keyId);
    keySecret = conf.get("s3mper.override.awsSecretAccessKey", keySecret);

    db = new AmazonDynamoDBClient(new BasicAWSCredentials(keyId, keySecret));

    readUnits = conf.getLong("s3mper.metastore.read.units", readUnits);
    writeUnits = conf.getLong("s3mper.metastore.write.units", writeUnits);

    retryCount = conf.getInt("s3mper.metastore.retry", retryCount);
    timeout = conf.getInt("s3mper.metastore.timeout", timeout);

    tableName = conf.get("s3mper.metastore.name", tableName);

    deleteMarkerEnabled = conf.getBoolean("s3mper.metastore.deleteMarker.enabled", false);

    boolean checkTableExists = conf.getBoolean("s3mper.metastore.create", false);

    if (checkTableExists) {
        ListTablesResult tables = db.listTables();

        if (!tables.getTableNames().contains(tableName)) {
            createTable();
        }
    }
}

From source file:com.ngdata.hbaseindexer.Main.java

License:Apache License

/**
 * @param conf the configuration object containing the hbase-indexer configuration, as well
 *             as the hbase/hadoop settings. Typically created using {@link HBaseIndexerConfiguration}.
 *//*from w  w w .j  a  v  a2s .c  om*/
public void startServices(Configuration conf) throws Exception {
    String hostname = Strings.domainNamePointerToHostName(
            DNS.getDefaultHost(conf.get("hbase.regionserver.dns.interface", "default"),
                    conf.get("hbase.regionserver.dns.nameserver", "default")));

    log.debug("Using hostname " + hostname);

    String zkConnectString = conf.get(ConfKeys.ZK_CONNECT_STRING);
    int zkSessionTimeout = conf.getInt(ConfKeys.ZK_SESSION_TIMEOUT, 30000);
    zk = new StateWatchingZooKeeper(zkConnectString, zkSessionTimeout);

    tablePool = new HTablePool(conf, 10 /* TODO configurable */);

    String zkRoot = conf.get(ConfKeys.ZK_ROOT_NODE);

    indexerModel = new IndexerModelImpl(zk, zkRoot);

    sepModel = new SepModelImpl(zk, conf);

    indexerMaster = new IndexerMaster(zk, indexerModel, conf, conf, zkConnectString, sepModel);
    indexerMaster.start();

    IndexerRegistry indexerRegistry = new IndexerRegistry();
    IndexerProcessRegistry indexerProcessRegistry = new IndexerProcessRegistryImpl(zk, conf);
    indexerSupervisor = new IndexerSupervisor(indexerModel, zk, hostname, indexerRegistry,
            indexerProcessRegistry, tablePool, conf);

    indexerSupervisor.init();
    startHttpServer();

}

From source file:com.ngdata.hbaseindexer.Main.java

License:Apache License

private void setupMetrics(Configuration conf) {
    String gangliaHost = conf.get(ConfKeys.GANGLIA_SERVER);
    if (gangliaHost != null) {
        int gangliaPort = conf.getInt(ConfKeys.GANGLIA_PORT, 8649);
        int interval = conf.getInt(ConfKeys.GANGLIA_INTERVAL, 60);
        log.info("Enabling Ganglia reporting to " + gangliaHost + ":" + gangliaPort);
        GangliaReporter.enable(interval, TimeUnit.SECONDS, gangliaHost, gangliaPort);
    }//from  w ww.  ja  va 2s. c  om
}

From source file:com.ngdata.hbaseindexer.master.IndexerMaster.java

License:Apache License

public IndexerMaster(ZooKeeperItf zk, WriteableIndexerModel indexerModel, Configuration mapReduceConf,
        Configuration hbaseConf, String zkConnectString, SepModel sepModel) {

    this.zk = zk;
    this.indexerModel = indexerModel;
    this.mapReduceConf = mapReduceConf;
    this.hbaseConf = hbaseConf;
    this.zkConnectString = zkConnectString;
    this.sepModel = sepModel;

    this.batchStatePollInterval = hbaseConf.getInt("hbaseindexer.batch.poll.interval", 60000);

    registerLifecycleListeners();//from  w  w w  . ja  va 2  s . c o m
}

From source file:com.ngdata.sep.impl.fork.ForkedReplicationSource.java

License:Apache License

/**
 * Instantiation method used by region servers
 *
 * @param conf configuration to use/*from  w w  w .ja v  a  2 s  .c  o  m*/
 * @param fs file system to use
 * @param manager replication manager to ping to
 * @param stopper     the atomic boolean to use to stop the regionserver
 * @param replicating the atomic boolean that starts/stops replication
 * @param peerClusterZnode the name of our znode
 * @throws IOException
 */
public void init(final Configuration conf, final FileSystem fs, final ReplicationSourceManager manager,
        final Stoppable stopper, final AtomicBoolean replicating, final String peerClusterZnode)
        throws IOException {
    this.stopper = stopper;
    this.conf = conf;
    this.replicationQueueSizeCapacity = this.conf.getLong("replication.source.size.capacity", 1024 * 1024 * 64);
    this.replicationQueueNbCapacity = this.conf.getInt("replication.source.nb.capacity", 25000);
    this.entriesArray = new HLog.Entry[this.replicationQueueNbCapacity];
    for (int i = 0; i < this.replicationQueueNbCapacity; i++) {
        this.entriesArray[i] = new HLog.Entry();
    }
    this.maxRetriesMultiplier = this.conf.getInt("replication.source.maxretriesmultiplier", 10);
    this.socketTimeoutMultiplier = this.conf.getInt("replication.source.socketTimeoutMultiplier",
            maxRetriesMultiplier * maxRetriesMultiplier);
    this.queue = new PriorityBlockingQueue<Path>(conf.getInt("hbase.regionserver.maxlogs", 32),
            new LogsComparator());
    this.conn = HConnectionManager.getConnection(conf);
    this.zkHelper = manager.getRepZkWrapper();
    this.ratio = this.conf.getFloat("replication.source.ratio", 0.1f);
    this.currentPeers = new ArrayList<ServerName>();
    this.random = new Random();
    this.replicating = replicating;
    this.manager = manager;
    this.sleepForRetries = this.conf.getLong("replication.source.sleepforretries", 1000);
    this.fs = fs;
    this.metrics = new ReplicationSourceMetrics(peerClusterZnode);
    this.repLogReader = new ReplicationHLogReaderManager(this.fs, this.conf);
    try {
        this.clusterId = zkHelper.getUUIDForCluster(zkHelper.getZookeeperWatcher());
    } catch (KeeperException ke) {
        throw new IOException("Could not read cluster id", ke);
    }

    // Finally look if this is a recovered queue
    this.checkIfQueueRecovered(peerClusterZnode);

    registerPeerClusterListener();

    // SEP change: add an mbean to expose some replication info
    infoMBean = new ReplicationSourceInfo();
    registerMBean();
}

From source file:com.ngdata.sep.impl.SepConsumer.java

License:Apache License

/**
 * @param subscriptionTimestamp timestamp of when the index subscription became active (or more accurately, not
 *        inactive)/*ww  w.ja v  a  2 s. c o  m*/
 * @param listener listeners that will process the events
 * @param threadCnt number of worker threads that will handle incoming SEP events
 * @param hostName hostname to bind to
 * @param payloadExtractor extracts payloads to include in SepEvents
 */
public SepConsumer(String subscriptionId, long subscriptionTimestamp, EventListener listener, int threadCnt,
        String hostName, ZooKeeperItf zk, Configuration hbaseConf, PayloadExtractor payloadExtractor)
        throws IOException, InterruptedException {
    Preconditions.checkArgument(threadCnt > 0, "Thread count must be > 0");
    this.subscriptionId = SepModelImpl.toInternalSubscriptionName(subscriptionId);
    this.subscriptionTimestamp = subscriptionTimestamp;
    this.listener = listener;
    this.zk = zk;
    this.hbaseConf = hbaseConf;
    this.sepMetrics = new SepMetrics(subscriptionId);
    this.payloadExtractor = payloadExtractor;
    this.executors = Lists.newArrayListWithCapacity(threadCnt);

    InetSocketAddress initialIsa = new InetSocketAddress(hostName, 0);
    if (initialIsa.getAddress() == null) {
        throw new IllegalArgumentException("Failed resolve of " + initialIsa);
    }
    String name = "regionserver/" + initialIsa.toString();
    this.rpcServer = new RpcServer(this, name, getServices(), initialIsa, // BindAddress is IP we got for this server.
            hbaseConf,
            new FifoRpcScheduler(hbaseConf, hbaseConf.getInt("hbase.regionserver.handler.count", 10)));

    this.serverName = new ServerName(hostName, rpcServer.getListenerAddress().getPort(),
            System.currentTimeMillis());
    this.zkWatcher = new ZooKeeperWatcher(hbaseConf, this.serverName.toString(), null);

    // login the zookeeper client principal (if using security)
    ZKUtil.loginClient(hbaseConf, "hbase.zookeeper.client.keytab.file",
            "hbase.zookeeper.client.kerberos.principal", hostName);

    // login the server principal (if using secure Hadoop)
    User.login(hbaseConf, "hbase.regionserver.keytab.file", "hbase.regionserver.kerberos.principal", hostName);

    for (int i = 0; i < threadCnt; i++) {
        ThreadPoolExecutor executor = new ThreadPoolExecutor(1, 1, 10, TimeUnit.SECONDS,
                new ArrayBlockingQueue<Runnable>(100));
        executor.setRejectedExecutionHandler(new WaitPolicy());
        executors.add(executor);
    }
}

From source file:com.odiago.flumebase.exec.BucketedAggregationElement.java

License:Apache License

public BucketedAggregationElement(FlowElementContext ctxt, AggregateNode aggregateNode) {
    super(ctxt, (Schema) aggregateNode.getAttr(PlanNode.OUTPUT_SCHEMA_ATTR));

    Configuration conf = aggregateNode.getConf();
    assert null != conf;
    mNumBuckets = conf.getInt(NUM_BUCKETS_KEY, DEFAULT_NUM_BUCKETS);
    mContinuousOutput = conf.getBoolean(CONTINUOUS_OUTPUT_KEY, DEFAULT_CONTINUOUS_OUTPUT);
    mMaxPriorEmitInterval = conf.getLong(MAX_PRIOR_EMIT_INTERVAL_KEY, DEFAULT_MAX_PRIOR_EMIT_INTERVAL);
    int slackTime = conf.getInt(SLACK_INTERVAL_KEY, DEFAULT_SLACK_INTERVAL);
    if (slackTime < 0) {
        mSlackTime = DEFAULT_SLACK_INTERVAL;
    } else {/* w ww.  ja  v a 2s.  co  m*/
        mSlackTime = slackTime;
    }

    assert mMaxPriorEmitInterval > 0;
    assert mMaxPriorEmitInterval > mSlackTime;

    List<TypedField> groupByFields = aggregateNode.getGroupByFields();
    if (null == groupByFields) {
        mGroupByFields = Collections.emptyList();
    } else {
        mGroupByFields = groupByFields;
    }

    mAggregateExprs = aggregateNode.getAggregateExprs();
    assert mAggregateExprs != null;
    mPropagateFields = aggregateNode.getPropagateFields();

    Expr windowExpr = aggregateNode.getWindowExpr();
    assert windowExpr.isConstant();
    try {
        mWindowSpec = (WindowSpec) windowExpr.eval(new EmptyEventWrapper());
        assert mWindowSpec.getRangeSpec().isConstant();
        mTimeSpan = (TimeSpan) mWindowSpec.getRangeSpec().eval(new EmptyEventWrapper());
    } catch (IOException ioe) {
        // The only way this can be thrown is if the window expr isn't actually constant.
        // This should not happen due to the assert above..
        LOG.error("Got IOException when calculating window width: " + ioe);
        throw new RuntimeException(ioe);
    }

    mBucketMap = new HashMap<Pair<Long, HashedEvent>, List<Bucket>>(mNumBuckets);
    mBucketsByGroup = new HashMap<HashedEvent, List<Pair<Long, List<Bucket>>>>();

    // Calculate the width of each bucket.
    mTimeModulus = mTimeSpan.getWidth() / mNumBuckets;
    if (mTimeModulus * mNumBuckets != mTimeSpan.getWidth()) {
        LOG.warn("Aggregation time step does not cleanly divide the time interval; "
                + "results may be inaccurate. Set " + NUM_BUCKETS_KEY + " to a better divisor.");
    }
}