Example usage for org.apache.hadoop.conf Configuration setInt

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration setInt.

Prototype

public void setInt(String name, int value) 

Source Link

Document

Set the value of the name property to an int.

Usage

From source file:co.cask.cdap.internal.app.runtime.batch.MapReduceContextConfigTest.java

License:Apache License

@Test
public void testManyMacrosInAppSpec() {
    Configuration hConf = new Configuration();
    MapReduceContextConfig cfg = new MapReduceContextConfig(hConf);
    StringBuilder appCfg = new StringBuilder();
    for (int i = 0; i < 100; i++) {
        appCfg.append("${").append(i).append("}");
        hConf.setInt(String.valueOf(i), i);
    }/*from   w  w  w.  j  ava  2 s. c  o m*/
    ApplicationSpecification appSpec = new DefaultApplicationSpecification("name", "desc", appCfg.toString(),
            new ArtifactId("artifact", new ArtifactVersion("1.0.0"), ArtifactScope.USER),
            Collections.<String, StreamSpecification>emptyMap(), Collections.<String, String>emptyMap(),
            Collections.<String, DatasetCreationSpec>emptyMap(),
            Collections.<String, FlowSpecification>emptyMap(),
            Collections.<String, MapReduceSpecification>emptyMap(),
            Collections.<String, SparkSpecification>emptyMap(),
            Collections.<String, WorkflowSpecification>emptyMap(),
            Collections.<String, ServiceSpecification>emptyMap(),
            Collections.<String, ScheduleSpecification>emptyMap(),
            Collections.<String, WorkerSpecification>emptyMap(), Collections.<String, Plugin>emptyMap());
    cfg.setApplicationSpecification(appSpec);
    Assert.assertEquals(appSpec.getConfiguration(), cfg.getApplicationSpecification().getConfiguration());
}

---

From source file:co.cask.hydrator.plugin.batch.source.ExcelInputFormat.java

License:Apache License

public static void setConfigurations(Job job, String filePattern, String sheetName, boolean reprocess,
        int sheetNo, String columnList, boolean skipFirstRow, String terminateIfEmptyRow, String rowLimit,
        String ifErrorRecord, String processedFiles) {

    Configuration configuration = job.getConfiguration();
    configuration.set(FILE_PATTERN, filePattern);
    configuration.set(SHEET_NAME, sheetName);
    configuration.setBoolean(RE_PROCESS, reprocess);
    configuration.setInt(SHEET_NO, sheetNo);
    configuration.set(COLUMN_LIST, columnList);
    configuration.setBoolean(SKIP_FIRST_ROW, skipFirstRow);
    configuration.set(TERMINATE_IF_EMPTY_ROW, terminateIfEmptyRow);

    if (!Strings.isNullOrEmpty(rowLimit)) {
        configuration.set(ROWS_LIMIT, rowLimit);
    }//from w w  w. j a va2  s .c  o m

    configuration.set(IF_ERROR_RECORD, ifErrorRecord);
    configuration.set(PROCESSED_FILES, processedFiles);
}

---

From source file:co.cask.hydrator.plugin.db.batch.source.DBSource.java

License:Apache License

@Override
public void prepareRun(BatchSourceContext context) throws Exception {
    sourceConfig.substituteMacros(context);
    LOG.debug(//from  w  ww.j a  v  a  2  s . c o m
            "pluginType = {}; pluginName = {}; connectionString = {}; importQuery = {}; "
                    + "boundingQuery = {}",
            sourceConfig.jdbcPluginType, sourceConfig.jdbcPluginName, sourceConfig.connectionString,
            sourceConfig.getImportQuery(), sourceConfig.getBoundingQuery());
    Configuration hConf = new Configuration();
    hConf.clear();

    // Load the plugin class to make sure it is available.
    Class<? extends Driver> driverClass = context.loadPluginClass(getJDBCPluginId());
    if (sourceConfig.user == null && sourceConfig.password == null) {
        DBConfiguration.configureDB(hConf, driverClass.getName(), sourceConfig.connectionString);
    } else {
        DBConfiguration.configureDB(hConf, driverClass.getName(), sourceConfig.connectionString,
                sourceConfig.user, sourceConfig.password);
    }
    DataDrivenETLDBInputFormat.setInput(hConf, DBRecord.class, sourceConfig.getImportQuery(),
            sourceConfig.getBoundingQuery(), sourceConfig.getEnableAutoCommit());
    if (sourceConfig.numSplits == null || sourceConfig.numSplits != 1) {
        hConf.set(DBConfiguration.INPUT_ORDER_BY_PROPERTY, sourceConfig.splitBy);
    }
    if (sourceConfig.numSplits != null) {
        hConf.setInt(MRJobConfig.NUM_MAPS, sourceConfig.numSplits);
    }
    context.setInput(Input.of(sourceConfig.referenceName,
            new SourceInputFormatProvider(DataDrivenETLDBInputFormat.class, hConf)));
}

---

From source file:co.cask.tephra.distributed.ThriftTransactionServerTest.java

License:Apache License

@BeforeClass
public static void start() throws Exception {
    zkServer = InMemoryZKServer.builder().setDataDir(tmpFolder.newFolder()).build();
    zkServer.startAndWait();// www  .j a v  a 2s . c  o m

    Configuration conf = new Configuration();
    conf.setBoolean(TxConstants.Manager.CFG_DO_PERSIST, false);
    conf.set(TxConstants.Service.CFG_DATA_TX_ZOOKEEPER_QUORUM, zkServer.getConnectionStr());
    conf.set(TxConstants.Service.CFG_DATA_TX_CLIENT_RETRY_STRATEGY, "n-times");
    conf.setInt(TxConstants.Service.CFG_DATA_TX_CLIENT_ATTEMPTS, 1);
    conf.setInt(TxConstants.Service.CFG_DATA_TX_CLIENT_COUNT, NUM_CLIENTS);
    conf.setLong(TxConstants.Service.CFG_DATA_TX_CLIENT_TIMEOUT, TimeUnit.HOURS.toMillis(1));
    conf.setInt(TxConstants.Service.CFG_DATA_TX_SERVER_IO_THREADS, 2);
    conf.setInt(TxConstants.Service.CFG_DATA_TX_SERVER_THREADS, 4);

    injector = Guice.createInjector(new ConfigModule(conf), new ZKModule(),
            new DiscoveryModules().getDistributedModules(),
            Modules.override(new TransactionModules().getDistributedModules()).with(new AbstractModule() {
                @Override
                protected void configure() {
                    bind(TransactionStateStorage.class).to(SlowTransactionStorage.class).in(Scopes.SINGLETON);
                }
            }), new TransactionClientModule());

    zkClientService = injector.getInstance(ZKClientService.class);
    zkClientService.startAndWait();

    // start a tx server
    txService = injector.getInstance(TransactionService.class);
    storage = injector.getInstance(TransactionStateStorage.class);
    try {
        LOG.info("Starting transaction service");
        txService.startAndWait();
    } catch (Exception e) {
        LOG.error("Failed to start service: ", e);
    }
}

---

From source file:co.cask.tephra.examples.BalanceBooksTest.java

License:Apache License

@BeforeClass
public static void setup() throws Exception {
    testUtil = new HBaseTestingUtility();
    Configuration conf = testUtil.getConfiguration();
    conf.setBoolean(TxConstants.Manager.CFG_DO_PERSIST, false);
    conf.set(TxConstants.Manager.CFG_TX_SNAPSHOT_DIR, "/tx.snapshot");

    // Tune down the connection thread pool size
    conf.setInt("hbase.hconnection.threads.core", 5);
    conf.setInt("hbase.hconnection.threads.max", 10);
    // Tunn down handler threads in regionserver
    conf.setInt("hbase.regionserver.handler.count", 10);

    // Set to random port
    conf.setInt("hbase.master.port", 0);
    conf.setInt("hbase.master.info.port", 0);
    conf.setInt("hbase.regionserver.port", 0);
    conf.setInt("hbase.regionserver.info.port", 0);

    testUtil.startMiniCluster();//  w  ww .  j  ava  2  s .  c om

    String zkClusterKey = testUtil.getClusterKey(); // hostname:clientPort:parentZnode
    String zkQuorum = zkClusterKey.substring(0, zkClusterKey.lastIndexOf(':'));
    LOG.info("Zookeeper Quorum is running at {}", zkQuorum);
    conf.set(TxConstants.Service.CFG_DATA_TX_ZOOKEEPER_QUORUM, zkQuorum);

    Injector injector = Guice.createInjector(new ConfigModule(conf), new ZKModule(),
            new DiscoveryModules().getDistributedModules(),
            Modules.override(new TransactionModules().getDistributedModules()).with(new AbstractModule() {
                @Override
                protected void configure() {
                    bind(TransactionStateStorage.class).to(InMemoryTransactionStateStorage.class)
                            .in(Scopes.SINGLETON);
                }
            }), new TransactionClientModule());

    zkClientService = injector.getInstance(ZKClientService.class);
    zkClientService.startAndWait();

    // start a tx server
    txService = injector.getInstance(TransactionService.class);
    try {
        LOG.info("Starting transaction service");
        txService.startAndWait();
    } catch (Exception e) {
        LOG.error("Failed to start service: ", e);
    }

}

---

From source file:co.cask.tephra.persist.AbstractTransactionStateStorageTest.java

License:Apache License

@Test
public void testTransactionManagerPersistence() throws Exception {
    Configuration conf = getConfiguration("testTransactionManagerPersistence");
    conf.setInt(TxConstants.Manager.CFG_TX_CLEANUP_INTERVAL, 0); // no cleanup thread
    // start snapshot thread, but with long enough interval so we only get snapshots on shutdown
    conf.setInt(TxConstants.Manager.CFG_TX_SNAPSHOT_INTERVAL, 600);

    TransactionStateStorage storage = null;
    TransactionStateStorage storage2 = null;
    TransactionStateStorage storage3 = null;
    try {/*from  w w  w. j  a v a  2s  . com*/
        storage = getStorage(conf);
        TransactionManager txManager = new TransactionManager(conf, storage, new TxMetricsCollector());
        txManager.startAndWait();

        // TODO: replace with new persistence tests
        final byte[] a = { 'a' };
        final byte[] b = { 'b' };
        // start a tx1, add a change A and commit
        Transaction tx1 = txManager.startShort();
        Assert.assertTrue(txManager.canCommit(tx1, Collections.singleton(a)));
        Assert.assertTrue(txManager.commit(tx1));
        // start a tx2 and add a change B
        Transaction tx2 = txManager.startShort();
        Assert.assertTrue(txManager.canCommit(tx2, Collections.singleton(b)));
        // start a tx3
        Transaction tx3 = txManager.startShort();
        // restart
        txManager.stopAndWait();
        TransactionSnapshot origState = txManager.getCurrentState();
        LOG.info("Orig state: " + origState);

        Thread.sleep(100);
        // starts a new tx manager
        storage2 = getStorage(conf);
        txManager = new TransactionManager(conf, storage2, new TxMetricsCollector());
        txManager.startAndWait();

        // check that the reloaded state matches the old
        TransactionSnapshot newState = txManager.getCurrentState();
        LOG.info("New state: " + newState);
        assertEquals(origState, newState);

        // commit tx2
        Assert.assertTrue(txManager.commit(tx2));
        // start another transaction, must be greater than tx3
        Transaction tx4 = txManager.startShort();
        Assert.assertTrue(tx4.getTransactionId() > tx3.getTransactionId());
        // tx1 must be visble from tx2, but tx3 and tx4 must not
        Assert.assertTrue(tx2.isVisible(tx1.getTransactionId()));
        Assert.assertFalse(tx2.isVisible(tx3.getTransactionId()));
        Assert.assertFalse(tx2.isVisible(tx4.getTransactionId()));
        // add same change for tx3
        Assert.assertFalse(txManager.canCommit(tx3, Collections.singleton(b)));
        // check visibility with new xaction
        Transaction tx5 = txManager.startShort();
        Assert.assertTrue(tx5.isVisible(tx1.getTransactionId()));
        Assert.assertTrue(tx5.isVisible(tx2.getTransactionId()));
        Assert.assertFalse(tx5.isVisible(tx3.getTransactionId()));
        Assert.assertFalse(tx5.isVisible(tx4.getTransactionId()));
        // can commit tx3?
        txManager.abort(tx3);
        txManager.abort(tx4);
        txManager.abort(tx5);
        // start new tx and verify its exclude list is empty
        Transaction tx6 = txManager.startShort();
        Assert.assertFalse(tx6.hasExcludes());
        txManager.abort(tx6);

        // now start 5 x claim size transactions
        Transaction tx = txManager.startShort();
        for (int i = 1; i < 50; i++) {
            tx = txManager.startShort();
        }
        origState = txManager.getCurrentState();

        Thread.sleep(100);
        // simulate crash by starting a new tx manager without a stopAndWait
        storage3 = getStorage(conf);
        txManager = new TransactionManager(conf, storage3, new TxMetricsCollector());
        txManager.startAndWait();

        // verify state again matches (this time should include WAL replay)
        newState = txManager.getCurrentState();
        assertEquals(origState, newState);

        // get a new transaction and verify it is greater
        Transaction txAfter = txManager.startShort();
        Assert.assertTrue(txAfter.getTransactionId() > tx.getTransactionId());
    } finally {
        if (storage != null) {
            storage.stopAndWait();
        }
        if (storage2 != null) {
            storage2.stopAndWait();
        }
        if (storage3 != null) {
            storage3.stopAndWait();
        }
    }
}

---

From source file:co.cask.tephra.persist.AbstractTransactionStateStorageTest.java

License:Apache License

/**
 * Tests whether the committed set is advanced properly on WAL replay.
 */// w  w  w.  ja va2  s.co  m
@Test
public void testCommittedSetClearing() throws Exception {
    Configuration conf = getConfiguration("testCommittedSetClearing");
    conf.setInt(TxConstants.Manager.CFG_TX_CLEANUP_INTERVAL, 0); // no cleanup thread
    conf.setInt(TxConstants.Manager.CFG_TX_SNAPSHOT_INTERVAL, 0); // no periodic snapshots

    TransactionStateStorage storage1 = null;
    TransactionStateStorage storage2 = null;
    try {
        storage1 = getStorage(conf);
        TransactionManager txManager = new TransactionManager(conf, storage1, new TxMetricsCollector());
        txManager.startAndWait();

        // TODO: replace with new persistence tests
        final byte[] a = { 'a' };
        final byte[] b = { 'b' };
        // start a tx1, add a change A and commit
        Transaction tx1 = txManager.startShort();
        Assert.assertTrue(txManager.canCommit(tx1, Collections.singleton(a)));
        Assert.assertTrue(txManager.commit(tx1));
        // start a tx2 and add a change B
        Transaction tx2 = txManager.startShort();
        Assert.assertTrue(txManager.canCommit(tx2, Collections.singleton(b)));
        // start a tx3
        Transaction tx3 = txManager.startShort();
        TransactionSnapshot origState = txManager.getCurrentState();
        LOG.info("Orig state: " + origState);

        // simulate a failure by starting a new tx manager without stopping first
        storage2 = getStorage(conf);
        txManager = new TransactionManager(conf, storage2, new TxMetricsCollector());
        txManager.startAndWait();

        // check that the reloaded state matches the old
        TransactionSnapshot newState = txManager.getCurrentState();
        LOG.info("New state: " + newState);
        assertEquals(origState, newState);

    } finally {
        if (storage1 != null) {
            storage1.stopAndWait();
        }
        if (storage2 != null) {
            storage2.stopAndWait();
        }
    }
}

---

From source file:co.cask.tephra.ThriftTransactionSystemTest.java

License:Apache License

@BeforeClass
public static void start() throws Exception {
    zkServer = InMemoryZKServer.builder().setDataDir(tmpFolder.newFolder()).build();
    zkServer.startAndWait();/*from   w  w w. j a v  a2s  .  com*/

    Configuration conf = new Configuration();
    conf.setBoolean(TxConstants.Manager.CFG_DO_PERSIST, false);
    conf.set(TxConstants.Service.CFG_DATA_TX_ZOOKEEPER_QUORUM, zkServer.getConnectionStr());
    conf.set(TxConstants.Service.CFG_DATA_TX_CLIENT_RETRY_STRATEGY, "n-times");
    conf.setInt(TxConstants.Service.CFG_DATA_TX_CLIENT_ATTEMPTS, 1);

    Injector injector = Guice.createInjector(new ConfigModule(conf), new ZKModule(),
            new DiscoveryModules().getDistributedModules(),
            Modules.override(new TransactionModules().getDistributedModules()).with(new AbstractModule() {
                @Override
                protected void configure() {
                    bind(TransactionStateStorage.class).to(InMemoryTransactionStateStorage.class)
                            .in(Scopes.SINGLETON);
                }
            }), new TransactionClientModule());

    zkClientService = injector.getInstance(ZKClientService.class);
    zkClientService.startAndWait();

    // start a tx server
    txService = injector.getInstance(TransactionService.class);
    storage = injector.getInstance(TransactionStateStorage.class);
    txClient = injector.getInstance(TransactionSystemClient.class);
    try {
        LOG.info("Starting transaction service");
        txService.startAndWait();
    } catch (Exception e) {
        LOG.error("Failed to start service: ", e);
    }
}

---

From source file:com.aerospike.hadoop.mapreduce.AerospikeConfigUtil.java

License:Apache License

public static void setInputPort(Configuration conf, int port) {
    log.info("setting " + INPUT_PORT + " to " + port);
    conf.setInt(INPUT_PORT, port);
}

---

From source file:com.aerospike.hadoop.mapreduce.AerospikeConfigUtil.java

License:Apache License

public static void setOutputPort(Configuration conf, int port) {
    log.info("setting " + OUTPUT_PORT + " to " + port);
    conf.setInt(OUTPUT_PORT, port);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »