Example usage for org.apache.hadoop.conf Configuration getInt

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration getInt.

Prototype

public int getInt(String name, int defaultValue) 

Source Link

Document

Get the value of the name property as an int.

Usage

From source file:com.inmobi.conduit.distcp.tools.util.DistCpUtils.java

License:Apache License

/**
 * Utility to retrieve a specified key from a Configuration. Throw exception
 * if not found./* w w w  .  ja  v  a2  s.com*/
 * @param configuration The Configuration in which the key is sought.
 * @param label The key being sought.
 * @return Integer value of the key.
 */
public static int getInt(Configuration configuration, String label) {
    int value = configuration.getInt(label, -1);
    assert value >= 0 : "Couldn't find " + label;
    return value;
}

---

From source file:com.inmobi.grill.driver.cube.MockDriver.java

License:Apache License

@Override
public void configure(Configuration conf) throws GrillException {
    this.conf = conf;
    ioTestVal = conf.getInt("mock.driver.test.val", -1);
}

---

From source file:com.inmobi.grill.driver.impala.ImpalaDriver.java

License:Apache License

@Override
public void configure(Configuration conf) throws GrillException {
    final String HOST = "HOST";
    final String PORT = "PORT";
    TSocket sock = new TSocket(conf.get(HOST), conf.getInt(PORT, 9999));
    try {/*from   www . j  a  va2 s.  co  m*/
        sock.open();
    } catch (TTransportException e) {
        logger.error(e.getMessage());
        throw new GrillException(e.getMessage(), e);
    }
    TBinaryProtocol protocol = new TBinaryProtocol(sock);
    this.client = new ImpalaService.Client(protocol);
    logger.info("Successfully connected to host" + conf.get(HOST) + ":" + conf.getInt(PORT, 9999));

}

---

From source file:com.inmobi.grill.driver.impala.TestImpalaDriver.java

License:Apache License

@Test
public void testConfigure() {

    try {//from   w  ww  .  ja va  2  s  . c om
        Configuration config = new Configuration();
        config.set("PORT", "123");
        config.set("HOST", "test.com");

        TSocket mockSocket = PowerMockito.mock(TSocket.class);
        TBinaryProtocol mockTProtocol = mock(TBinaryProtocol.class);
        ImpalaService.Client mockClient = mock(ImpalaService.Client.class);

        whenNew(TSocket.class).withArguments(config.get("HOST"), config.getInt("PORT", 9999))
                .thenReturn(mockSocket);
        whenNew(TBinaryProtocol.class).withArguments(mockSocket).thenReturn(mockTProtocol);
        whenNew(ImpalaService.Client.class).withArguments(mockTProtocol).thenReturn(mockClient);

        this.testInst.configure(config);
        verifyNew(TSocket.class).withArguments("test.com", 123);
        verifyNew(TBinaryProtocol.class).withArguments(mockSocket);
        verifyNew(ImpalaService.Client.class).withArguments(mockTProtocol);

        Mockito.verify(mockSocket, Mockito.times(1)).open();
    } catch (Exception e) {
        Assert.fail();
    }

}

---

From source file:com.inmobi.grill.driver.impala.TestImpalaDriver.java

License:Apache License

@Test
public void testExecute() {
    try {/*from  ww w  .j a v  a 2 s.  c o m*/

        // configure before executing
        Configuration config = new Configuration();
        config.set("PORT", "123");
        config.set("HOST", "test.com");

        TSocket mockSocket = PowerMockito.mock(TSocket.class);

        TBinaryProtocol mockTProtocol = PowerMockito.mock(TBinaryProtocol.class);
        ImpalaService.Client mockClient = Mockito.mock(ImpalaService.Client.class);

        Query q = mock(Query.class);
        QueryHandle qh = mock(QueryHandle.class);
        ImpalaResultSet mockResultSet = mock(ImpalaResultSet.class);

        when(mockResultSet.hasNext()).thenReturn(true);
        whenNew(Query.class).withNoArguments().thenReturn(q);
        when(mockClient.query(q)).thenReturn(qh);
        when(mockClient.get_state(qh)).thenReturn(QueryState.FINISHED);

        whenNew(TSocket.class).withArguments(config.get("HOST"), config.getInt("PORT", 9999))
                .thenReturn(mockSocket);
        whenNew(TBinaryProtocol.class).withArguments(mockSocket).thenReturn(mockTProtocol);
        whenNew(ImpalaService.Client.class).withArguments(mockTProtocol).thenReturn(mockClient);
        whenNew(ImpalaResultSet.class).withArguments(mockClient, qh).thenReturn(mockResultSet);

        // actual run
        this.testInst.configure(config);
        GrillResultSet br = this.testInst.execute("query", null);

        // test and verify
        Assert.assertEquals(true, ((ImpalaResultSet) br).hasNext());
        Mockito.verify(mockClient).query(q);
        Mockito.verify(mockClient).get_state(qh);

    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail();
    }

}

---

From source file:com.inmobi.grill.driver.jdbc.DataSourceConnectionProvider.java

License:Apache License

@Override
public synchronized Connection getConnection(Configuration conf) throws SQLException {
    DriverConfig config = getDriverConfigfromConf(conf);
    if (!dataSourceMap.containsKey(config)) {
        ComboPooledDataSource cpds = new ComboPooledDataSource();
        try {// w w  w . ja  va  2  s  .  c  o  m
            cpds.setDriverClass(config.driverClass);
        } catch (PropertyVetoException e) {
            throw new IllegalArgumentException("Unable to set driver class:" + config.driverClass, e);
        }
        cpds.setJdbcUrl(config.jdbcURI);
        cpds.setUser(config.user);
        cpds.setPassword(config.password);

        // Maximum number of connections allowed in the pool
        cpds.setMaxPoolSize(conf.getInt(JDBCDriverConfConstants.JDBC_POOL_MAX_SIZE,
                JDBCDriverConfConstants.JDBC_POOL_MAX_SIZE_DEFAULT));
        // Max idle time before a connection is closed
        cpds.setMaxIdleTime(conf.getInt(JDBCDriverConfConstants.JDBC_POOL_IDLE_TIME,
                JDBCDriverConfConstants.JDBC_POOL_IDLE_TIME_DEFAULT));
        // Max idel time before connection is closed if 
        // number of connections is > min pool size (default = 3)
        cpds.setMaxIdleTimeExcessConnections(conf.getInt(JDBCDriverConfConstants.JDBC_POOL_IDLE_TIME,
                JDBCDriverConfConstants.JDBC_POOL_IDLE_TIME_DEFAULT));
        // Maximum number of prepared statements to cache per connection
        cpds.setMaxStatementsPerConnection(
                conf.getInt(JDBCDriverConfConstants.JDBC_MAX_STATEMENTS_PER_CONNECTION,
                        JDBCDriverConfConstants.JDBC_MAX_STATEMENTS_PER_CONNECTION_DEFAULT));
        dataSourceMap.put(config, cpds);
        LOG.info("Created new datasource for config: " + config);
    }
    return dataSourceMap.get(config).getConnection();
}

---

From source file:com.jumptap.h2redis.RedisHMRecordWriter.java

License:Open Source License

public RedisHMRecordWriter(Configuration conf) {
    String host = conf.get(RedisDriver.REDIS_HOST);
    int port = conf.getInt(RedisDriver.REDIS_PORT, 6379);
    String pw = conf.get(RedisDriver.REDIS_PW, null);
    int db = conf.getInt(RedisDriver.REDIS_DB, 0);
    init(host, port, pw, db, 5);//from   w w  w .  j a  v  a 2 s.c om

    String pfx = conf.get(RedisDriver.REDIS_KEY_PREFIX);
    String hkpfx = conf.get(RedisDriver.REDIS_HASHKEY_PREFIX);
    String delim = conf.get(RedisDriver.REDIS_KEY_PREFIX_DELIM, ".");

    this.keyMaker = new KeyMaker(pfx, hkpfx, delim);
    this.lastUpdateKey = conf.get(RedisDriver.REDIS_KEY_TS);
    this.ttl = conf.getInt(RedisDriver.REDIS_KEY_TTL, 0);
}

---

From source file:com.jumptap.h2redis.RedisOutputFormat.java

License:Open Source License

@Override
public void checkOutputSpecs(JobContext context) throws IOException, InterruptedException {
    Configuration conf = context.getConfiguration();
    String host = conf.get(RedisDriver.REDIS_HOST);
    int key = conf.getInt(RedisDriver.REDIS_KEY_FIELD, -1);
    int hash = conf.getInt(RedisDriver.REDIS_HASHKEY_FIELD, -1);
    int val = conf.getInt(RedisDriver.REDIS_HASHVAL_FIELD, -1);
    if (host == null || host.isEmpty() || key == -1 || hash == -1 || val == -1)
        throw new IOException("Missing configuration param, check usage.");
}

---

From source file:com.jumptap.h2redis.RedisOutputMapper.java

License:Open Source License

@Override
public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
    Configuration conf = context.getConfiguration();
    int keyIdx = conf.getInt(RedisDriver.REDIS_KEY_FIELD, -1);
    int hashIdx = conf.getInt(RedisDriver.REDIS_HASHKEY_FIELD, -1);
    int valIdx = conf.getInt(RedisDriver.REDIS_HASHVAL_FIELD, -1);

    if (keyIdx == -1 || hashIdx == -1 || valIdx == -1)
        return;/*from w ww  .  j  a va  2s  .co  m*/

    String[] payload = StringUtils.getStrings(value.toString());
    String keyStr = payload[keyIdx];
    String hashStr = payload[hashIdx];
    String valStr = payload[valIdx];

    // check filters
    Pattern p = conf.getPattern(RedisDriver.REDIS_KEY_FILTER, null);
    if (p != null && p.matcher(keyStr).find()) {
        return;
    }

    p = conf.getPattern(RedisDriver.REDIS_HASH_FILTER, null);
    if (p != null && p.matcher(hashStr).find()) {
        return;
    }

    p = conf.getPattern(RedisDriver.REDIS_VAL_FILTER, null);
    if (p != null && p.matcher(valStr).find()) {
        return;
    }

    outkey.set(keyStr);
    outvalue.set(hashStr + "," + valStr);
    context.write(outkey, outvalue);
}

---

From source file:com.junz.hadoop.custom.SytsLogInputFormat.java

License:Apache License

public List<InputSplit> getSplits(JobContext context) throws IOException, InterruptedException {
    Configuration job = context.getConfiguration();
    List<InputSplit> splits = new ArrayList<InputSplit>();

    try {//  w  w w . j a  v  a  2  s.  c o  m
        long startId = job.getLong(START_ID_PROPERTY, 1);
        long numberOfIds = job.getLong(NUMBER_LOG_PROPERTY, 1);
        int groups = job.getInt(NUMBER_MAP_PROPERTY, 1);
        long groupSize = (numberOfIds / groups);

        // Split the rows into n-number of chunks and adjust the last chunk
        // accordingly
        for (int i = 0; i < groups; i++) {
            DBInputSplit split;

            if ((i + 1) == groups)
                split = new DBInputSplit(i * groupSize + startId, numberOfIds + startId);
            else
                split = new DBInputSplit(i * groupSize + startId, (i * groupSize) + groupSize + startId);

            splits.add(split);
        }

        return splits;
    } catch (Exception e) {
        throw new IOException(e.getMessage());
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »