Example usage for org.apache.hadoop.conf Configuration getLong

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration getLong.

Prototype

public long getLong(String name, long defaultValue) 

Source Link

Document

Get the value of the name property as a long.

Usage

From source file:org.apache.sqoop.mapreduce.CombineFileInputFormat.java

License:Apache License

@Override
public List<InputSplit> getSplits(JobContext job) throws IOException {

    long minSizeNode = 0;
    long minSizeRack = 0;
    long maxSize = 0;
    Configuration conf = job.getConfiguration();

    // the values specified by setxxxSplitSize() takes precedence over the
    // values that might have been specified in the config
    if (minSplitSizeNode != 0) {
        minSizeNode = minSplitSizeNode;//from   ww  w.  j  a  va2s . co m
    } else {
        minSizeNode = conf.getLong(SPLIT_MINSIZE_PERNODE, 0);
    }
    if (minSplitSizeRack != 0) {
        minSizeRack = minSplitSizeRack;
    } else {
        minSizeRack = conf.getLong(SPLIT_MINSIZE_PERRACK, 0);
    }
    if (maxSplitSize != 0) {
        maxSize = maxSplitSize;
    } else {
        maxSize = conf.getLong("mapred.max.split.size", 0);
    }
    if (minSizeNode != 0 && maxSize != 0 && minSizeNode > maxSize) {
        throw new IOException("Minimum split size pernode " + minSizeNode
                + " cannot be larger than maximum split size " + maxSize);
    }
    if (minSizeRack != 0 && maxSize != 0 && minSizeRack > maxSize) {
        throw new IOException("Minimum split size per rack" + minSizeRack
                + " cannot be larger than maximum split size " + maxSize);
    }
    if (minSizeRack != 0 && minSizeNode > minSizeRack) {
        throw new IOException("Minimum split size per node" + minSizeNode
                + " cannot be smaller than minimum split " + "size per rack " + minSizeRack);
    }

    // all the files in input set
    Path[] paths = FileUtil.stat2Paths(listStatus(job).toArray(new FileStatus[0]));
    List<InputSplit> splits = new ArrayList<InputSplit>();
    if (paths.length == 0) {
        return splits;
    }

    // Convert them to Paths first. This is a costly operation and
    // we should do it first, otherwise we will incur doing it multiple
    // times, one time each for each pool in the next loop.
    List<Path> newpaths = new LinkedList<Path>();
    for (int i = 0; i < paths.length; i++) {
        FileSystem fs = paths[i].getFileSystem(conf);

        //the scheme and authority will be kept if the path is
        //a valid path for a non-default file system
        Path p = fs.makeQualified(paths[i]);
        newpaths.add(p);
    }
    paths = null;

    // In one single iteration, process all the paths in a single pool.
    // Processing one pool at a time ensures that a split contains paths
    // from a single pool only.
    for (MultiPathFilter onepool : pools) {
        ArrayList<Path> myPaths = new ArrayList<Path>();

        // pick one input path. If it matches all the filters in a pool,
        // add it to the output set
        for (Iterator<Path> iter = newpaths.iterator(); iter.hasNext();) {
            Path p = iter.next();
            if (onepool.accept(p)) {
                myPaths.add(p); // add it to my output set
                iter.remove();
            }
        }
        // create splits for all files in this pool.
        getMoreSplits(job, myPaths.toArray(new Path[myPaths.size()]), maxSize, minSizeNode, minSizeRack,
                splits);
    }

    // create splits for all files that are not in any pool.
    getMoreSplits(job, newpaths.toArray(new Path[newpaths.size()]), maxSize, minSizeNode, minSizeRack, splits);

    // free up rackToNodes map
    rackToNodes.clear();
    return splits;
}

---

From source file:org.apache.tajo.conf.TajoConf.java

License:Apache License

public static long getLongVar(Configuration conf, ConfVars var) {
    assert (var.valClass == Long.class || var.valClass == Integer.class);
    if (var.valClass == Integer.class) {
        return conf.getInt(var.varname, var.defaultIntVal);
    } else {//  www  .ja  va2s  .c o m
        return conf.getLong(var.varname, var.defaultLongVal);
    }
}

---

From source file:org.apache.tajo.conf.TajoConf.java

License:Apache License

public static long getLongVar(Configuration conf, ConfVars var, long defaultVal) {
    return conf.getLong(var.varname, defaultVal);
}

---

From source file:org.apache.tajo.util.JvmPauseMonitor.java

License:Apache License

public JvmPauseMonitor(Configuration conf) {
    this.warnThresholdMs = conf.getLong(WARN_THRESHOLD_KEY, WARN_THRESHOLD_DEFAULT);
    this.infoThresholdMs = conf.getLong(INFO_THRESHOLD_KEY, INFO_THRESHOLD_DEFAULT);
}

---

From source file:org.apache.tephra.coprocessor.TransactionStateCache.java

License:Apache License

/**
 * Try to initialize the Configuration and TransactionStateStorage instances.  Obtaining the Configuration may
 * fail until ReactorServiceMain has been started.
 *//*from   w w  w .j a v a2s . c  om*/
private void tryInit() {
    try {
        Configuration conf = getSnapshotConfiguration();
        if (conf != null) {
            // Since this is only used for background loading of transaction snapshots, we use the no-op metrics collector,
            // as there are no relevant metrics to report
            this.storage = new HDFSTransactionStateStorage(conf, new SnapshotCodecProvider(conf),
                    new TxMetricsCollector());
            this.storage.startAndWait();
            this.snapshotRefreshFrequency = conf.getLong(TxConstants.Manager.CFG_TX_SNAPSHOT_INTERVAL,
                    TxConstants.Manager.DEFAULT_TX_SNAPSHOT_INTERVAL) * 1000;
            this.initialized = true;
        } else {
            LOG.info("Could not load configuration");
        }
    } catch (Exception e) {
        LOG.info("Failed to initialize TransactionStateCache due to: ", e);
    }
}

---

From source file:org.apache.tephra.distributed.TransactionServiceClient.java

License:Apache License

/**
 * Create from a configuration. This will first attempt to find a zookeeper
 * for service discovery. Otherwise it will look for the port in the
 * config and use localhost.//from  w  w  w. j  av  a  2  s  .c  o m
 * @param config a configuration containing the zookeeper properties
 * @param clientId id of the client that identifies it when it starts a transaction
 */
@Inject
public TransactionServiceClient(Configuration config, ThriftClientProvider clientProvider,
        @Named(TxConstants.CLIENT_ID) String clientId) {

    // initialize the retry logic
    String retryStrat = config.get(TxConstants.Service.CFG_DATA_TX_CLIENT_RETRY_STRATEGY,
            TxConstants.Service.DEFAULT_DATA_TX_CLIENT_RETRY_STRATEGY);
    if ("backoff".equals(retryStrat)) {
        this.retryStrategyProvider = new RetryWithBackoff.Provider();
    } else if ("n-times".equals(retryStrat)) {
        this.retryStrategyProvider = new RetryNTimes.Provider();
    } else {
        try {
            this.retryStrategyProvider = (RetryStrategyProvider) Class.forName(retryStrat).newInstance();
        } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) {
            throw new IllegalArgumentException(
                    String.format("Unable to instantiate RetryStrategyProvider '%s'", retryStrat), e);
        }
    }
    this.retryStrategyProvider.configure(config);
    LOG.debug("Retry strategy is " + this.retryStrategyProvider);

    this.clientProvider = clientProvider;
    this.clientId = clientId;

    changeSetCountLimit = config.getInt(TxConstants.Manager.CFG_TX_CHANGESET_COUNT_LIMIT,
            TxConstants.Manager.DEFAULT_TX_CHANGESET_COUNT_LIMIT);
    changeSetCountThreshold = config.getInt(TxConstants.Manager.CFG_TX_CHANGESET_COUNT_WARN_THRESHOLD,
            TxConstants.Manager.DEFAULT_TX_CHANGESET_COUNT_WARN_THRESHOLD);
    changeSetSizeLimit = config.getLong(TxConstants.Manager.CFG_TX_CHANGESET_SIZE_LIMIT,
            TxConstants.Manager.DEFAULT_TX_CHANGESET_SIZE_LIMIT);
    changeSetSizeThreshold = config.getLong(TxConstants.Manager.CFG_TX_CHANGESET_SIZE_WARN_THRESHOLD,
            TxConstants.Manager.DEFAULT_TX_CHANGESET_SIZE_WARN_THRESHOLD);
}

---

From source file:org.apache.tephra.hbase.coprocessor.TransactionProcessor.java

License:Apache License

/**
 * Refresh the properties related to transaction pruning. This method needs to be invoked if there is change in the
 * prune related properties after clearing the state by calling {@link #resetPruneState}.
 *
 * @param env {@link RegionCoprocessorEnvironment} of this region
 *//*from w  ww  . j  av a  2s . c  o  m*/
protected void initializePruneState(RegionCoprocessorEnvironment env) {
    Configuration conf = getConfiguration(env);
    if (conf != null) {
        pruneEnable = conf.getBoolean(TxConstants.TransactionPruning.PRUNE_ENABLE,
                TxConstants.TransactionPruning.DEFAULT_PRUNE_ENABLE);

        if (Boolean.TRUE.equals(pruneEnable)) {
            TableName pruneTable = TableName.valueOf(conf.get(TxConstants.TransactionPruning.PRUNE_STATE_TABLE,
                    TxConstants.TransactionPruning.DEFAULT_PRUNE_STATE_TABLE));
            long pruneFlushInterval = TimeUnit.SECONDS
                    .toMillis(conf.getLong(TxConstants.TransactionPruning.PRUNE_FLUSH_INTERVAL,
                            TxConstants.TransactionPruning.DEFAULT_PRUNE_FLUSH_INTERVAL));

            compactionState = new CompactionState(env, pruneTable, pruneFlushInterval);
            if (LOG.isDebugEnabled()) {
                LOG.debug(String.format(
                        "Automatic invalid list pruning is enabled for table %s. Compaction state "
                                + "will be recorded in table %s",
                        env.getRegionInfo().getTable().getNameWithNamespaceInclAsString(),
                        pruneTable.getNameWithNamespaceInclAsString()));
            }
        }
    }
}

---

From source file:org.apache.tephra.persist.AbstractTransactionLog.java

License:Apache License

AbstractTransactionLog(long timestamp, MetricsCollector metricsCollector, Configuration conf) {
    this.timestamp = timestamp;
    this.metricsCollector = metricsCollector;
    this.slowAppendThreshold = conf.getLong(TxConstants.TransactionLog.CFG_SLOW_APPEND_THRESHOLD,
            TxConstants.TransactionLog.DEFAULT_SLOW_APPEND_THRESHOLD);
}

---

From source file:org.apache.tephra.persist.LocalTransactionStateStorageTest.java

License:Apache License

@SuppressWarnings("deprecation")
@Test//from ww w.j  av a  2  s  . c om
public void testLongTxnBackwardsCompatibility() throws Exception {
    Configuration conf = getConfiguration("testLongTxnBackwardsCompatibility");

    // Use SnapshotCodec version 1
    String latestSnapshotCodec = conf.get(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES);
    conf.set(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, DefaultSnapshotCodec.class.getName());

    TransactionStateStorage storage = null;
    try {
        storage = getStorage(conf);
        storage.startAndWait();

        // Create transaction snapshot and transaction edits with version when long running txns had -1 expiration.
        Collection<Long> invalid = Lists.newArrayList();
        NavigableMap<Long, TransactionManager.InProgressTx> inProgress = Maps.newTreeMap();
        long time1 = System.currentTimeMillis();
        long wp1 = time1 * TxConstants.MAX_TX_PER_MS;
        inProgress.put(wp1, new TransactionManager.InProgressTx(wp1 - 5, -1L));
        long time2 = time1 + 100;
        long wp2 = time2 * TxConstants.MAX_TX_PER_MS;
        inProgress.put(wp2, new TransactionManager.InProgressTx(wp2 - 50, time2 + 1000));
        Map<Long, Set<ChangeId>> committing = Maps.newHashMap();
        Map<Long, Set<ChangeId>> committed = Maps.newHashMap();
        TransactionSnapshot snapshot = new TransactionSnapshot(time2, 0, wp2, invalid, inProgress, committing,
                committed);
        long time3 = time1 + 200;
        long wp3 = time3 * TxConstants.MAX_TX_PER_MS;
        TransactionEdit edit1 = new TransactionEditV2(wp3, wp3 - 10, TransactionEdit.State.INPROGRESS, -1L,
                null, 0L, false, null);
        long time4 = time1 + 300;
        long wp4 = time4 * TxConstants.MAX_TX_PER_MS;
        TransactionEdit edit2 = new TransactionEditV2(wp4, wp4 - 10, TransactionEdit.State.INPROGRESS,
                time4 + 1000, null, 0L, false, null);

        // write snapshot and transaction edit
        storage.writeSnapshot(snapshot);
        TransactionLog log = storage.createLog(time2);
        log.append(edit1);
        log.append(edit2);
        log.close();

        // Start transaction manager
        conf.set(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, latestSnapshotCodec);
        long longTimeout = TimeUnit.SECONDS.toMillis(conf.getLong(TxConstants.Manager.CFG_TX_LONG_TIMEOUT,
                TxConstants.Manager.DEFAULT_TX_LONG_TIMEOUT));
        TransactionManager txm = new TransactionManager(conf, storage, new TxMetricsCollector());
        txm.startAndWait();
        try {
            // Verify that the txns in old format were read correctly.
            // There should be four in-progress transactions, and no invalid transactions
            TransactionSnapshot snapshot1 = txm.getCurrentState();
            Assert.assertEquals(ImmutableSortedSet.of(wp1, wp2, wp3, wp4), snapshot1.getInProgress().keySet());
            verifyInProgress(snapshot1.getInProgress().get(wp1), InProgressType.LONG, time1 + longTimeout);
            verifyInProgress(snapshot1.getInProgress().get(wp2), InProgressType.SHORT, time2 + 1000);
            verifyInProgress(snapshot1.getInProgress().get(wp3), InProgressType.LONG, time3 + longTimeout);
            verifyInProgress(snapshot1.getInProgress().get(wp4), InProgressType.SHORT, time4 + 1000);
            Assert.assertEquals(0, snapshot1.getInvalid().size());
        } finally {
            txm.stopAndWait();
        }
    } finally {
        if (storage != null) {
            storage.stopAndWait();
        }
    }
}

---

From source file:org.apache.tephra.TransactionManager.java

License:Apache License

@Inject
public TransactionManager(Configuration conf, @Nonnull TransactionStateStorage persistor,
        MetricsCollector txMetricsCollector) {
    this.persistor = persistor;
    cleanupInterval = conf.getInt(TxConstants.Manager.CFG_TX_CLEANUP_INTERVAL,
            TxConstants.Manager.DEFAULT_TX_CLEANUP_INTERVAL);
    maxTimeout = conf.getInt(TxConstants.Manager.CFG_TX_MAX_TIMEOUT,
            TxConstants.Manager.DEFAULT_TX_MAX_TIMEOUT);
    defaultTimeout = conf.getInt(TxConstants.Manager.CFG_TX_TIMEOUT, TxConstants.Manager.DEFAULT_TX_TIMEOUT);
    defaultLongTimeout = conf.getInt(TxConstants.Manager.CFG_TX_LONG_TIMEOUT,
            TxConstants.Manager.DEFAULT_TX_LONG_TIMEOUT);
    snapshotFrequencyInSeconds = conf.getLong(TxConstants.Manager.CFG_TX_SNAPSHOT_INTERVAL,
            TxConstants.Manager.DEFAULT_TX_SNAPSHOT_INTERVAL);
    // must always keep at least 1 snapshot
    snapshotRetainCount = Math.max(conf.getInt(TxConstants.Manager.CFG_TX_SNAPSHOT_RETAIN,
            TxConstants.Manager.DEFAULT_TX_SNAPSHOT_RETAIN), 1);

    changeSetCountLimit = conf.getInt(TxConstants.Manager.CFG_TX_CHANGESET_COUNT_LIMIT,
            TxConstants.Manager.DEFAULT_TX_CHANGESET_COUNT_LIMIT);
    changeSetCountThreshold = conf.getInt(TxConstants.Manager.CFG_TX_CHANGESET_COUNT_WARN_THRESHOLD,
            TxConstants.Manager.DEFAULT_TX_CHANGESET_COUNT_WARN_THRESHOLD);
    changeSetSizeLimit = conf.getLong(TxConstants.Manager.CFG_TX_CHANGESET_SIZE_LIMIT,
            TxConstants.Manager.DEFAULT_TX_CHANGESET_SIZE_LIMIT);
    changeSetSizeThreshold = conf.getLong(TxConstants.Manager.CFG_TX_CHANGESET_SIZE_WARN_THRESHOLD,
            TxConstants.Manager.DEFAULT_TX_CHANGESET_SIZE_WARN_THRESHOLD);

    // intentionally not using a constant, as this config should not be exposed
    // TODO: REMOVE WITH txnBackwardsCompatCheck()
    longTimeoutTolerance = conf.getLong("data.tx.long.timeout.tolerance", 10000);

    ClientIdRetention retention = ClientIdRetention.valueOf(conf
            .get(TxConstants.Manager.CFG_TX_RETAIN_CLIENT_ID, TxConstants.Manager.DEFAULT_TX_RETAIN_CLIENT_ID)
            .toUpperCase());//from   w w  w. j  av a  2 s.  co m
    this.retainClientId = retention != ClientIdRetention.OFF;
    this.retainClientIdPastCommit = retention == ClientIdRetention.COMMITTED;

    this.txMetricsCollector = txMetricsCollector;
    this.txMetricsCollector.configure(conf);
    clear();
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• »