Example usage for org.apache.hadoop.conf Configuration setLong

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration setLong.

Prototype

public void setLong(String name, long value)

Source Link

Document

Set the value of the name property to a long.

Usage

From source file:co.cask.cdap.data.stream.AbstractStreamInputFormat.java

License:Apache License

/**
 * Sets the time range for the stream events.
 *
 * @param conf The configuration to modify
 * @param startTime Timestamp in milliseconds of the event start time (inclusive).
 * @param endTime Timestamp in milliseconds of the event end time (exclusive).
 *//*from   www.  j  a va2 s.c o  m*/
public static void setTimeRange(Configuration conf, long startTime, long endTime) {
    Preconditions.checkArgument(startTime >= 0, "Start time must be >= 0");
    Preconditions.checkArgument(endTime >= 0, "End time must be >= 0");

    conf.setLong(EVENT_START_TIME, startTime);
    conf.setLong(EVENT_END_TIME, endTime);
}

From source file:co.cask.cdap.data.stream.AbstractStreamInputFormat.java

License:Apache License

/**
 * Sets the maximum split size./*  w  w  w  . ja  v a2  s.  c  om*/
 *
 * @param conf The conf to modify.
 * @param maxSplits Maximum split size in bytes.
 */
public static void setMaxSplitSize(Configuration conf, long maxSplits) {
    conf.setLong(MAX_SPLIT_SIZE, maxSplits);
}

From source file:co.cask.cdap.data.stream.AbstractStreamInputFormat.java

License:Apache License

/**
 * Sets the minimum split size./*from w w w.j  ava 2s.  c  o  m*/
 *
 * @param conf The conf to modify.
 * @param minSplits Minimum split size in bytes.
 */
public static void setMinSplitSize(Configuration conf, long minSplits) {
    conf.setLong(MIN_SPLIT_SIZE, minSplits);
}

From source file:co.cask.tephra.distributed.ThriftTransactionServerTest.java

License:Apache License

@BeforeClass
public static void start() throws Exception {
    zkServer = InMemoryZKServer.builder().setDataDir(tmpFolder.newFolder()).build();
    zkServer.startAndWait();/*from  www  .j  av  a 2  s. c  om*/

    Configuration conf = new Configuration();
    conf.setBoolean(TxConstants.Manager.CFG_DO_PERSIST, false);
    conf.set(TxConstants.Service.CFG_DATA_TX_ZOOKEEPER_QUORUM, zkServer.getConnectionStr());
    conf.set(TxConstants.Service.CFG_DATA_TX_CLIENT_RETRY_STRATEGY, "n-times");
    conf.setInt(TxConstants.Service.CFG_DATA_TX_CLIENT_ATTEMPTS, 1);
    conf.setInt(TxConstants.Service.CFG_DATA_TX_CLIENT_COUNT, NUM_CLIENTS);
    conf.setLong(TxConstants.Service.CFG_DATA_TX_CLIENT_TIMEOUT, TimeUnit.HOURS.toMillis(1));
    conf.setInt(TxConstants.Service.CFG_DATA_TX_SERVER_IO_THREADS, 2);
    conf.setInt(TxConstants.Service.CFG_DATA_TX_SERVER_THREADS, 4);

    injector = Guice.createInjector(new ConfigModule(conf), new ZKModule(),
            new DiscoveryModules().getDistributedModules(),
            Modules.override(new TransactionModules().getDistributedModules()).with(new AbstractModule() {
                @Override
                protected void configure() {
                    bind(TransactionStateStorage.class).to(SlowTransactionStorage.class).in(Scopes.SINGLETON);
                }
            }), new TransactionClientModule());

    zkClientService = injector.getInstance(ZKClientService.class);
    zkClientService.startAndWait();

    // start a tx server
    txService = injector.getInstance(TransactionService.class);
    storage = injector.getInstance(TransactionStateStorage.class);
    try {
        LOG.info("Starting transaction service");
        txService.startAndWait();
    } catch (Exception e) {
        LOG.error("Failed to start service: ", e);
    }
}

From source file:co.cask.tephra.TransactionManagerTest.java

License:Apache License

@Test
public void testTruncateInvalid() throws Exception {
    InMemoryTransactionStateStorage storage = new InMemoryTransactionStateStorage();
    Configuration testConf = new Configuration(conf);
    // No snapshots
    testConf.setLong(TxConstants.Manager.CFG_TX_SNAPSHOT_INTERVAL, -1);
    TransactionManager txm1 = new TransactionManager(testConf, storage, new TxMetricsCollector());
    txm1.startAndWait();//  ww w  . j a  v a 2 s  .c  o  m

    TransactionManager txm2 = null;
    Transaction tx1;
    Transaction tx2;
    Transaction tx3;
    Transaction tx4;
    Transaction tx5;
    Transaction tx6;
    try {
        Assert.assertEquals(0, txm1.getInvalidSize());

        // start a few transactions
        tx1 = txm1.startLong();
        tx2 = txm1.startShort();
        tx3 = txm1.startLong();
        tx4 = txm1.startShort();
        tx5 = txm1.startLong();
        tx6 = txm1.startShort();

        // invalidate tx1, tx2, tx5 and tx6
        txm1.invalidate(tx1.getTransactionId());
        txm1.invalidate(tx2.getTransactionId());
        txm1.invalidate(tx5.getTransactionId());
        txm1.invalidate(tx6.getTransactionId());

        // tx1, tx2, tx5 and tx6 should be in invalid list
        Assert.assertEquals(ImmutableList.of(tx1.getTransactionId(), tx2.getTransactionId(),
                tx5.getTransactionId(), tx6.getTransactionId()), txm1.getCurrentState().getInvalid());

        // remove tx1 and tx6 from invalid list
        Assert.assertTrue(
                txm1.truncateInvalidTx(ImmutableSet.of(tx1.getTransactionId(), tx6.getTransactionId())));

        // only tx2 and tx5 should be in invalid list now
        Assert.assertEquals(ImmutableList.of(tx2.getTransactionId(), tx5.getTransactionId()),
                txm1.getCurrentState().getInvalid());

        // removing in-progress transactions should not have any effect
        Assert.assertEquals(ImmutableSet.of(tx3.getTransactionId(), tx4.getTransactionId()),
                txm1.getCurrentState().getInProgress().keySet());
        Assert.assertFalse(
                txm1.truncateInvalidTx(ImmutableSet.of(tx3.getTransactionId(), tx4.getTransactionId())));
        // no change to in-progress
        Assert.assertEquals(ImmutableSet.of(tx3.getTransactionId(), tx4.getTransactionId()),
                txm1.getCurrentState().getInProgress().keySet());
        // no change to invalid list
        Assert.assertEquals(ImmutableList.of(tx2.getTransactionId(), tx5.getTransactionId()),
                txm1.getCurrentState().getInvalid());

        // Test transaction edit logs replay
        // Start another transaction manager without stopping txm1 so that snapshot does not get written,
        // and all logs can be replayed.
        txm2 = new TransactionManager(testConf, storage, new TxMetricsCollector());
        txm2.startAndWait();
        Assert.assertEquals(ImmutableList.of(tx2.getTransactionId(), tx5.getTransactionId()),
                txm2.getCurrentState().getInvalid());
        Assert.assertEquals(ImmutableSet.of(tx3.getTransactionId(), tx4.getTransactionId()),
                txm2.getCurrentState().getInProgress().keySet());
    } finally {
        txm1.stopAndWait();
        if (txm2 != null) {
            txm2.stopAndWait();
        }
    }
}

From source file:co.cask.tephra.TransactionManagerTest.java

License:Apache License

@Test
public void testTruncateInvalidBeforeTime() throws Exception {
    InMemoryTransactionStateStorage storage = new InMemoryTransactionStateStorage();
    Configuration testConf = new Configuration(conf);
    // No snapshots
    testConf.setLong(TxConstants.Manager.CFG_TX_SNAPSHOT_INTERVAL, -1);
    TransactionManager txm1 = new TransactionManager(testConf, storage, new TxMetricsCollector());
    txm1.startAndWait();/* www.ja  v a2 s.  co m*/

    TransactionManager txm2 = null;
    Transaction tx1;
    Transaction tx2;
    Transaction tx3;
    Transaction tx4;
    Transaction tx5;
    Transaction tx6;
    try {
        Assert.assertEquals(0, txm1.getInvalidSize());

        // start a few transactions
        tx1 = txm1.startLong();
        tx2 = txm1.startShort();
        // Sleep so that transaction ids get generated a millisecond apart for assertion
        // TEPHRA-63 should eliminate the need to sleep
        TimeUnit.MILLISECONDS.sleep(1);
        long timeBeforeTx3 = System.currentTimeMillis();
        tx3 = txm1.startLong();
        tx4 = txm1.startShort();
        TimeUnit.MILLISECONDS.sleep(1);
        long timeBeforeTx5 = System.currentTimeMillis();
        tx5 = txm1.startLong();
        tx6 = txm1.startShort();

        // invalidate tx1, tx2, tx5 and tx6
        txm1.invalidate(tx1.getTransactionId());
        txm1.invalidate(tx2.getTransactionId());
        txm1.invalidate(tx5.getTransactionId());
        txm1.invalidate(tx6.getTransactionId());

        // tx1, tx2, tx5 and tx6 should be in invalid list
        Assert.assertEquals(ImmutableList.of(tx1.getTransactionId(), tx2.getTransactionId(),
                tx5.getTransactionId(), tx6.getTransactionId()), txm1.getCurrentState().getInvalid());

        // remove transactions before tx3 from invalid list
        Assert.assertTrue(txm1.truncateInvalidTxBefore(timeBeforeTx3));

        // only tx5 and tx6 should be in invalid list now
        Assert.assertEquals(ImmutableList.of(tx5.getTransactionId(), tx6.getTransactionId()),
                txm1.getCurrentState().getInvalid());

        // removing invalid transactions before tx5 should throw exception since tx3 and tx4 are in-progress
        Assert.assertEquals(ImmutableSet.of(tx3.getTransactionId(), tx4.getTransactionId()),
                txm1.getCurrentState().getInProgress().keySet());
        try {
            txm1.truncateInvalidTxBefore(timeBeforeTx5);
            Assert.fail("Expected InvalidTruncateTimeException exception");
        } catch (InvalidTruncateTimeException e) {
            // Expected exception
        }
        // no change to in-progress
        Assert.assertEquals(ImmutableSet.of(tx3.getTransactionId(), tx4.getTransactionId()),
                txm1.getCurrentState().getInProgress().keySet());
        // no change to invalid list
        Assert.assertEquals(ImmutableList.of(tx5.getTransactionId(), tx6.getTransactionId()),
                txm1.getCurrentState().getInvalid());

        // Test transaction edit logs replay
        // Start another transaction manager without stopping txm1 so that snapshot does not get written, 
        // and all logs can be replayed.
        txm2 = new TransactionManager(testConf, storage, new TxMetricsCollector());
        txm2.startAndWait();
        Assert.assertEquals(ImmutableList.of(tx5.getTransactionId(), tx6.getTransactionId()),
                txm2.getCurrentState().getInvalid());
        Assert.assertEquals(ImmutableSet.of(tx3.getTransactionId(), tx4.getTransactionId()),
                txm2.getCurrentState().getInProgress().keySet());
    } finally {
        txm1.stopAndWait();
        if (txm2 != null) {
            txm2.stopAndWait();
        }
    }
}

From source file:com.aerospike.hadoop.mapreduce.AerospikeConfigUtil.java

License:Apache License

public static void setInputNumRangeBegin(Configuration conf, long begin) {
    log.info("setting " + INPUT_NUMRANGE_BEGIN + " to " + begin);
    conf.setLong(INPUT_NUMRANGE_BEGIN, begin);
}

From source file:com.aerospike.hadoop.mapreduce.AerospikeConfigUtil.java

License:Apache License

public static void setInputNumRangeEnd(Configuration conf, long end) {
    log.info("setting " + INPUT_NUMRANGE_END + " to " + end);
    conf.setLong(INPUT_NUMRANGE_END, end);
}

From source file:com.aliyun.fs.oss.contract.TestAliyunOSSContractDistCp.java

License:Apache License

@Override
protected Configuration createConfiguration() {
    Configuration newConf = super.createConfiguration();
    newConf.setLong("fs.oss.local.block.size", MULTIPART_SETTING);
    return newConf;
}

From source file:com.aliyun.fs.oss.TestAliyunOSSBlockOutputStream.java

License:Apache License

@Before
public void setUp() throws Exception {
    Configuration conf = new Configuration();
    conf.set("mapreduce.job.run-local", "true");
    conf.set("fs.oss.buffer.dirs", "/tmp");
    conf.setLong("fs.oss.local.block.size", 5 * 1024 * 1024);
    fs = AliyunOSSTestUtils.createTestFileSystem(conf);
}