Example usage for org.apache.hadoop.conf Configuration getLong

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration getLong.

Prototype

public long getLong(String name, long defaultValue) 

Source Link

Document

Get the value of the name property as a long.

Usage

From source file:co.cask.cdap.data.runtime.main.TokenSecureStoreUpdater.java

License:Apache License

private long calculateUpdateInterval() {
    List<Long> renewalTimes = Lists.newArrayList();

    renewalTimes.add(hConf.getLong(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY,
            DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT));

    // The value contains in hbase-default.xml, so it should always there. If it is really missing, default it to 1 day.
    renewalTimes.add(hConf.getLong(Constants.HBase.AUTH_KEY_UPDATE_INTERVAL,
            TimeUnit.MILLISECONDS.convert(1, TimeUnit.DAYS)));

    if (secureExplore) {
        // Renewal interval for YARN
        renewalTimes.add(hConf.getLong(YarnConfiguration.DELEGATION_TOKEN_RENEW_INTERVAL_KEY,
                YarnConfiguration.DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT));

        // Renewal interval for Hive. Also see: https://issues.apache.org/jira/browse/HIVE-9214
        Configuration hiveConf = getHiveConf();
        if (hiveConf != null) {
            renewalTimes.add(hiveConf.getLong(HadoopThriftAuthBridge.Server.DELEGATION_TOKEN_RENEW_INTERVAL_KEY,
                    HadoopThriftAuthBridge.Server.DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT));
        } else {/*from w w  w.j  a  v  a2  s  . c  om*/
            renewalTimes.add(HadoopThriftAuthBridge.Server.DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT);
        }

        // Renewal interval for JHS
        renewalTimes.add(hConf.getLong(MRConfig.DELEGATION_TOKEN_RENEW_INTERVAL_KEY,
                MRConfig.DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT));
    }

    // Set the update interval to the shortest update interval of all required renewals.
    Long minimumInterval = Collections.min(renewalTimes);
    // Schedule it 5 min before it expires
    long delay = minimumInterval - TimeUnit.MINUTES.toMillis(5);
    // Safeguard: In practice, the value can't be that small, otherwise nothing would work.
    if (delay <= 0) {
        delay = (minimumInterval <= 2) ? 1 : minimumInterval / 2;
    }
    LOG.info("Setting token renewal time to: {} ms", delay);
    return delay;
}

---

From source file:co.cask.cdap.data.stream.AbstractStreamInputFormat.java

License:Apache License

@Override
public List<InputSplit> getSplits(JobContext context) throws IOException, InterruptedException {
    Configuration conf = context.getConfiguration();
    long ttl = conf.getLong(STREAM_TTL, Long.MAX_VALUE);
    long endTime = conf.getLong(EVENT_END_TIME, Long.MAX_VALUE);
    long startTime = Math.max(conf.getLong(EVENT_START_TIME, 0L), getCurrentTime() - ttl);
    long maxSplitSize = conf.getLong(MAX_SPLIT_SIZE, Long.MAX_VALUE);
    long minSplitSize = Math.min(conf.getLong(MIN_SPLIT_SIZE, 1L), maxSplitSize);
    StreamInputSplitFinder<InputSplit> splitFinder = StreamInputSplitFinder
            .builder(URI.create(conf.get(STREAM_PATH))).setStartTime(startTime).setEndTime(endTime)
            .setMinSplitSize(minSplitSize).setMaxSplitSize(maxSplitSize).build(splitFactory);
    return splitFinder.getSplits(conf);
}

---

From source file:co.cask.tephra.coprocessor.TransactionStateCache.java

License:Apache License

/**
 * Try to initialize the Configuration and TransactionStateStorage instances.  Obtaining the Configuration may
 * fail until ReactorServiceMain has been started.
 *//*from w w  w.j a  v a 2  s.  c o  m*/
private void tryInit() {
    try {
        Configuration conf = getSnapshotConfiguration();
        if (conf != null) {
            // Since this is only used for background loading of transaction snapshots, we use the no-op metrics collector,
            // as there are no relevant metrics to report
            this.storage = new HDFSTransactionStateStorage(conf, new SnapshotCodecProvider(conf),
                    new TxMetricsCollector());
            this.storage.startAndWait();
            this.snapshotRefreshFrequency = conf.getLong(TxConstants.Manager.CFG_TX_SNAPSHOT_INTERVAL,
                    TxConstants.Manager.DEFAULT_TX_SNAPSHOT_INTERVAL) * 1000;
            this.initialized = true;
        } else {
            LOG.info("Could not load configuration");
        }
    } catch (Exception e) {
        LOG.info("Failed to initialize TransactionStateCache due to: " + e.getMessage());
    }
}

---

From source file:co.cask.tephra.persist.LocalTransactionStateStorageTest.java

License:Apache License

@SuppressWarnings("deprecation")
@Test/*from w  w  w  .  j av  a 2  s  .c  o m*/
public void testLongTxnBackwardsCompatibility() throws Exception {
    Configuration conf = getConfiguration("testLongTxnBackwardsCompatibility");

    // Use SnapshotCodec version 1
    String latestSnapshotCodec = conf.get(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES);
    conf.set(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, DefaultSnapshotCodec.class.getName());

    TransactionStateStorage storage = null;
    try {
        storage = getStorage(conf);
        storage.startAndWait();

        // Create transaction snapshot and transaction edits with version when long running txns had -1 expiration.
        Collection<Long> invalid = Lists.newArrayList();
        NavigableMap<Long, TransactionManager.InProgressTx> inProgress = Maps.newTreeMap();
        long time1 = System.currentTimeMillis();
        long wp1 = time1 * TxConstants.MAX_TX_PER_MS;
        inProgress.put(wp1, new TransactionManager.InProgressTx(wp1 - 5, -1L));
        long time2 = time1 + 100;
        long wp2 = time2 * TxConstants.MAX_TX_PER_MS;
        inProgress.put(wp2, new TransactionManager.InProgressTx(wp2 - 50, time2 + 1000));
        Map<Long, Set<ChangeId>> committing = Maps.newHashMap();
        Map<Long, Set<ChangeId>> committed = Maps.newHashMap();
        TransactionSnapshot snapshot = new TransactionSnapshot(time2, 0, wp2, invalid, inProgress, committing,
                committed);
        long time3 = time1 + 200;
        long wp3 = time3 * TxConstants.MAX_TX_PER_MS;
        TransactionEdit edit1 = new TransactionEditV2(wp3, wp3 - 10, TransactionEdit.State.INPROGRESS, -1L,
                null, 0L, false, null);
        long time4 = time1 + 300;
        long wp4 = time4 * TxConstants.MAX_TX_PER_MS;
        TransactionEdit edit2 = new TransactionEditV2(wp4, wp4 - 10, TransactionEdit.State.INPROGRESS,
                time4 + 1000, null, 0L, false, null);

        // write snapshot and transaction edit
        storage.writeSnapshot(snapshot);
        TransactionLog log = storage.createLog(time2);
        log.append(edit1);
        log.append(edit2);
        log.close();

        // Start transaction manager
        conf.set(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, latestSnapshotCodec);
        long longTimeout = TimeUnit.SECONDS.toMillis(conf.getLong(TxConstants.Manager.CFG_TX_LONG_TIMEOUT,
                TxConstants.Manager.DEFAULT_TX_LONG_TIMEOUT));
        TransactionManager txm = new TransactionManager(conf, storage, new TxMetricsCollector());
        txm.startAndWait();
        try {
            // Verify that the txns in old format were read correctly.
            // There should be four in-progress transactions, and no invalid transactions
            TransactionSnapshot snapshot1 = txm.getCurrentState();
            Assert.assertEquals(ImmutableSortedSet.of(wp1, wp2, wp3, wp4), snapshot1.getInProgress().keySet());
            verifyInProgress(snapshot1.getInProgress().get(wp1), TransactionType.LONG, time1 + longTimeout);
            verifyInProgress(snapshot1.getInProgress().get(wp2), TransactionType.SHORT, time2 + 1000);
            verifyInProgress(snapshot1.getInProgress().get(wp3), TransactionType.LONG, time3 + longTimeout);
            verifyInProgress(snapshot1.getInProgress().get(wp4), TransactionType.SHORT, time4 + 1000);
            Assert.assertEquals(0, snapshot1.getInvalid().size());
        } finally {
            txm.stopAndWait();
        }
    } finally {
        if (storage != null) {
            storage.stopAndWait();
        }
    }
}

---

From source file:co.cask.tephra.TransactionManager.java

License:Apache License

@Inject
public TransactionManager(Configuration conf, @Nonnull TransactionStateStorage persistor,
        MetricsCollector txMetricsCollector) {
    this.persistor = persistor;
    cleanupInterval = conf.getInt(TxConstants.Manager.CFG_TX_CLEANUP_INTERVAL,
            TxConstants.Manager.DEFAULT_TX_CLEANUP_INTERVAL);
    defaultTimeout = conf.getInt(TxConstants.Manager.CFG_TX_TIMEOUT, TxConstants.Manager.DEFAULT_TX_TIMEOUT);
    defaultLongTimeout = conf.getInt(TxConstants.Manager.CFG_TX_LONG_TIMEOUT,
            TxConstants.Manager.DEFAULT_TX_LONG_TIMEOUT);
    snapshotFrequencyInSeconds = conf.getLong(TxConstants.Manager.CFG_TX_SNAPSHOT_INTERVAL,
            TxConstants.Manager.DEFAULT_TX_SNAPSHOT_INTERVAL);
    // must always keep at least 1 snapshot
    snapshotRetainCount = Math.max(conf.getInt(TxConstants.Manager.CFG_TX_SNAPSHOT_RETAIN,
            TxConstants.Manager.DEFAULT_TX_SNAPSHOT_RETAIN), 1);

    // intentionally not using a constant, as this config should not be exposed
    // TODO: REMOVE WITH txnBackwardsCompatCheck()
    longTimeoutTolerance = conf.getLong("data.tx.long.timeout.tolerance", 10000);

    ///*w  ww  .  j  a v a  2  s .  c o  m*/
    this.txMetricsCollector = txMetricsCollector;
    this.txMetricsCollector.configure(conf);
    clear();
}

---

From source file:com.aerospike.hadoop.mapreduce.AerospikeConfigUtil.java

License:Apache License

public static long getInputNumRangeBegin(Configuration conf) {
    long begin = conf.getLong(INPUT_NUMRANGE_BEGIN, INVALID_LONG);
    if (begin == INVALID_LONG && getInputOperation(conf).equals("numrange"))
        throw new UnsupportedOperationException("missing input numrange begin");
    log.info("using " + INPUT_NUMRANGE_BEGIN + " = " + begin);
    return begin;
}

---

From source file:com.aerospike.hadoop.mapreduce.AerospikeConfigUtil.java

License:Apache License

public static long getInputNumRangeEnd(Configuration conf) {
    long end = conf.getLong(INPUT_NUMRANGE_END, INVALID_LONG);
    if (end == INVALID_LONG && getInputOperation(conf).equals("numrange"))
        throw new UnsupportedOperationException("missing input numrange end");
    log.info("using " + INPUT_NUMRANGE_END + " = " + end);
    return end;//from   w w w  . j  a  v  a 2 s  .c o m
}

---

From source file:com.alibaba.wasp.ipc.ClientCache.java

License:Apache License

/**
 * Construct & cache an IPC client with the user-provided SocketFactory if no
 * cached client exists.//from w  w w. j  a va  2s .  c om
 * 
 * @param conf
 *          Configuration
 * @param factory
 *          socket factory
 * @return an IPC client
 */
protected synchronized NettyTransceiver getClient(InetSocketAddress addr, SocketFactory factory,
        Configuration conf) {

    NettyTransceiver client = clients.get(addr.toString());
    if (client == null) {
        Class<? extends NettyTransceiver> waspClientClass = NettyTransceiver.class;

        // Make an wasp rpc client.
        try {
            Constructor<? extends NettyTransceiver> cst = waspClientClass
                    .getConstructor(InetSocketAddress.class, Long.class);
            client = cst.newInstance(addr, conf.getLong(FConstants.WASP_CONNECTION_TIMEOUT_MILLIS,
                    FConstants.DEFAULT_CONNECTION_TIMEOUT_MILLIS));
            client.setConf(conf);
        } catch (InvocationTargetException e) {
            throw new RuntimeException(e);
        } catch (InstantiationException e) {
            throw new RuntimeException(e);
        } catch (IllegalAccessException e) {
            throw new RuntimeException(e);
        } catch (NoSuchMethodException e) {
            throw new RuntimeException("No matching constructor in " + waspClientClass.getName(), e);
        }

        clients.put(addr.toString(), client);
    } else {
        client.incCount();
    }
    return client;
}

---

From source file:com.alibaba.wasp.master.FServerManager.java

License:Apache License

FServerManager(final Server master, final FMasterServices services, final boolean connect)
        throws ZooKeeperConnectionException {
    this.master = master;
    this.services = services;
    Configuration c = master.getConfiguration();
    maxSkew = c.getLong("wasp.master.maxclockskew", 30000);
    warningSkew = c.getLong("wasp.master.warningclockskew", 10000);
    this.deadservers = new DeadServer();
    this.connection = connect ? FConnectionManager.getConnection(c) : null;
}

---

From source file:com.alibaba.wasp.master.GeneralBulkAssigner.java

License:Apache License

/**
 * // w  ww.j a v a  2  s .  c om
 * @param timeout How long to wait.
 * @return true if done.
 */
@Override
protected boolean waitUntilDone(final long timeout) throws InterruptedException {
    Set<EntityGroupInfo> entityGroupSet = new HashSet<EntityGroupInfo>();
    for (List<EntityGroupInfo> entityGroupList : bulkPlan.values()) {
        entityGroupSet.addAll(entityGroupList);
    }

    pool.shutdown(); // no more task allowed
    int serverCount = bulkPlan.size();
    int entityGroupCount = entityGroupSet.size();
    long startTime = System.currentTimeMillis();
    long rpcWaitTime = startTime + timeout;
    while (!server.isStopped() && !pool.isTerminated() && rpcWaitTime > System.currentTimeMillis()) {
        if (failedPlans.isEmpty()) {
            pool.awaitTermination(100, TimeUnit.MILLISECONDS);
        } else {
            reassignFailedPlans();
        }
    }
    if (!pool.isTerminated()) {
        LOG.warn("bulk assigner is still running after " + (System.currentTimeMillis() - startTime)
                + "ms, shut it down now");
        // some assigner hangs, can't wait any more, shutdown the pool now
        List<Runnable> notStarted = pool.shutdownNow();
        if (notStarted != null && !notStarted.isEmpty()) {
            server.abort("some single server assigner hasn't started yet" + " when the bulk assigner timed out",
                    null);
            return false;
        }
    }

    int reassigningEntityGroups = 0;
    if (!failedPlans.isEmpty() && !server.isStopped()) {
        reassigningEntityGroups = reassignFailedPlans();
    }

    Configuration conf = server.getConfiguration();
    long perEntityGroupOpenTimeGuesstimate = conf.getLong("wasp.bulk.assignment.perentityGroup.open.time",
            1000);
    long endTime = Math.max(System.currentTimeMillis(), rpcWaitTime)
            + perEntityGroupOpenTimeGuesstimate * (reassigningEntityGroups + 1);
    EntityGroupStates entityGroupStates = assignmentManager.getEntityGroupStates();
    // We're not synchronizing on entityGroupsInTransition now because we don't use
    // any iterator.
    while (!entityGroupSet.isEmpty() && !server.isStopped() && endTime > System.currentTimeMillis()) {
        Iterator<EntityGroupInfo> entityGroupInfoIterator = entityGroupSet.iterator();
        while (entityGroupInfoIterator.hasNext()) {
            EntityGroupInfo egi = entityGroupInfoIterator.next();
            EntityGroupState state = entityGroupStates.getEntityGroupState(egi);
            if ((!entityGroupStates.isEntityGroupInTransition(egi)
                    && entityGroupStates.isEntityGroupAssigned(egi)) || state.isSplit()
                    || state.isSplitting()) {
                entityGroupInfoIterator.remove();
            }
        }
        if (!entityGroupSet.isEmpty()) {
            entityGroupStates.waitForUpdate(100);
        }
    }

    if (LOG.isDebugEnabled()) {
        long elapsedTime = System.currentTimeMillis() - startTime;
        String status = "successfully";
        if (!entityGroupSet.isEmpty()) {
            status = "with " + entityGroupSet.size() + " entityGroups still not assigned yet";
        }
        LOG.debug("bulk assigning total " + entityGroupCount + " entityGroups to " + serverCount
                + " servers, took " + elapsedTime + "ms, " + status);
    }
    return entityGroupSet.isEmpty();
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• »