Example usage for org.apache.hadoop.conf Configuration getInt

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration getInt.

Prototype

public int getInt(String name, int defaultValue) 

Source Link

Document

Get the value of the name property as an int.

Usage

From source file:com.cloudera.sqoop.shims.CDH3Shim.java

License:Apache License

@Override
public int getConfNumMaps(Configuration conf) {
    return conf.getInt("mapred.map.tasks", 1);
}

---

From source file:com.cmcc.hy.bigdata.weijifen.jobs.hubei.score.ScoreInfoDayJob.java

License:Open Source License

@Override
public int run(String[] args) throws Exception {
    // TODO Auto-generated method stub

    Configuration conf = ConfigurationUtil.loginAuthentication(args, SEPCIFIC_CONFIG_NAME, getConf());

    // ?()//from  w  ww  .j  a v a  2s .com
    String statDate = DateUtil.getFilterDate(args);
    if (statDate == null) {
        System.exit(1);
    }

    conf.set(STAT_DAY, statDate);

    // ?job
    Job job = Job.getInstance(conf, JOB_NAME + ":" + statDate);
    job.setJarByClass(ScoreInfoDayJob.class);
    String scoreInfoInput = conf.get(SCORE_INFO_INPUT_PATH);
    Path scoreInfoPath = new Path(scoreInfoInput);

    String acctPhoneMapInfoInput = conf.get(ACCT_PHONE_MAP_INPUT_PATH);
    Path accPhoneMapInfoPath = new Path(acctPhoneMapInfoInput);

    // ?
    if (FileSystemUtil.exists(scoreInfoPath)) {
        MultipleInputs.addInputPath(job, scoreInfoPath, SequenceFileInputFormat.class,
                ScoreInfoDayMapper.class);
        logger.info("SocreInfoPath is " + scoreInfoInput);
    } else {
        logger.error("Path [{}] not exist!", scoreInfoInput);
    }

    // ??
    //        if (FileSystemUtil.exists(accPhoneMapInfoPath)) {
    //            MultipleInputs.addInputPath(job, accPhoneMapInfoPath, TextInputFormat.class,
    //                    AcctPhoneMapper.class);
    //            logger.info("AccPhoneMapInfoPath is " + acctPhoneMapInfoInput);
    //        } else {
    //            logger.error("Path [{}] not exist!", acctPhoneMapInfoInput);
    //        }
    // job
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(ScoreInfo.class);
    job.setNumReduceTasks(conf.getInt(REDUCE_NUMBER, 40));
    job.setOutputFormatClass(NullOutputFormat.class);

    //        TableMapReduceUtil.initTableReducerJob(HBaseTableSchema.USER_INFO_TABLE2,
    //                ScoreInfoDayReducer.class, job);

    return (job.waitForCompletion(true) ? 0 : 1);
}

---

From source file:com.codefollower.lealone.hbase.server.HBaseTcpServer.java

License:Apache License

public static int getMasterTcpPort(Configuration conf) {
    return conf.getInt(Constants.PROJECT_NAME_PREFIX + "master.tcp.port", Constants.DEFAULT_TCP_PORT - 1);
}

---

From source file:com.codefollower.lealone.hbase.server.HBaseTcpServer.java

License:Apache License

public static int getRegionServerTcpPort(Configuration conf) {
    return conf.getInt(Constants.PROJECT_NAME_PREFIX + "regionserver.tcp.port", Constants.DEFAULT_TCP_PORT);
}

---

From source file:com.codefollower.lealone.omid.client.TSOClient.java

License:Open Source License

public TSOClient(Configuration conf) throws IOException {
    state = State.DISCONNECTED;//from   w w  w.j a v  a 2 s . c o m
    queuedOps = new ArrayBlockingQueue<Op>(200);
    retryTimer = new Timer(true);

    commitCallbacks = Collections.synchronizedMap(new HashMap<Long, CommitCallback>());
    isCommittedCallbacks = Collections.synchronizedMap(new HashMap<Long, List<CommitQueryCallback>>());
    createCallbacks = new ConcurrentLinkedQueue<CreateCallback>();
    channel = null;

    LOG.info("Starting TSOClient");

    // Start client with Nb of active threads = 3 as maximum.
    factory = new NioClientSocketChannelFactory(Executors.newCachedThreadPool(),
            Executors.newCachedThreadPool(), 3);
    // Create the bootstrap
    bootstrap = new ClientBootstrap(factory);

    int executorThreads = conf.getInt("tso.executor.threads", 3);

    bootstrap.getPipeline().addLast("executor", new ExecutionHandler(
            new OrderedMemoryAwareThreadPoolExecutor(executorThreads, 1024 * 1024, 4 * 1024 * 1024)));
    bootstrap.getPipeline().addLast("handler", this);
    bootstrap.setOption("tcpNoDelay", false);
    bootstrap.setOption("keepAlive", true);
    bootstrap.setOption("reuseAddress", true);
    bootstrap.setOption("connectTimeoutMillis", 100);

    String host = conf.get("tso.host");
    int port = conf.getInt("tso.port", 1234);
    maxRetries = conf.getInt("tso.max_retries", 100);
    retryDelayMs = conf.getInt("tso.retry_delay_ms", 1000);

    if (host == null) {
        throw new IOException("tso.host missing from configuration");
    }

    addr = new InetSocketAddress(host, port);
    connectIfNeeded();
}

---

From source file:com.codefollower.lealone.omid.regionserver.Compacter.java

License:Open Source License

@Override
public void start(CoprocessorEnvironment e) throws IOException {
    System.out.println("Starting compacter");
    Configuration conf = e.getConfiguration();
    factory = new NioClientSocketChannelFactory(bossExecutor, workerExecutor, 3);
    bootstrap = new ClientBootstrap(factory);

    bootstrap.getPipeline().addLast("decoder", new ObjectDecoder());
    bootstrap.getPipeline().addLast("handler", new Handler());
    bootstrap.setOption("tcpNoDelay", false);
    bootstrap.setOption("keepAlive", true);
    bootstrap.setOption("reuseAddress", true);
    bootstrap.setOption("connectTimeoutMillis", 100);

    String host = conf.get("tso.host");
    int port = conf.getInt("tso.port", 1234) + 1;

    if (host == null) {
        throw new IOException("tso.host missing from configuration");
    }/*from w  w  w  .  j a v a  2 s. c o m*/

    bootstrap.connect(new InetSocketAddress(host, port)).addListener(new ChannelFutureListener() {

        @Override
        public void operationComplete(ChannelFuture future) throws Exception {
            if (future.isSuccess()) {
                System.out.println("Compacter connected!");
                channel = future.getChannel();
            } else {
                System.out.println("Connection failed");
            }
        }
    });
}

---

From source file:com.conductor.hadoop.WritableValueInputFormat.java

License:Apache License

@Override
public List<InputSplit> getSplits(final JobContext context) throws IOException, InterruptedException {
    final Configuration conf = context.getConfiguration();

    // init the reader
    final String filePath = conf.get(INPUT_FILE_LOCATION_CONF);
    checkArgument(!Strings.isNullOrEmpty(filePath), "Missing property: " + INPUT_FILE_LOCATION_CONF);

    final FileSystem fs = getFileSystem(conf);
    final Path path = fs.makeQualified(new Path(filePath));
    final SequenceFile.Reader reader = getReader(conf, path);

    // create the splits by looping through the values of the input file
    int totalInputs = 0;
    int maxInputsPerSplit = conf.getInt(INPUTS_PER_SPLIT_CONF, DEFAULT_INPUTS_PER_SPLIT);
    long pos = 0L;
    long last = 0L;
    long lengthRemaining = fs.getFileStatus(path).getLen();
    final List<InputSplit> splits = Lists.newArrayList();
    final V value = getV(conf);
    for (final NullWritable key = NullWritable.get(); reader.next(key, value); last = reader.getPosition()) {
        if (++totalInputs % maxInputsPerSplit == 0) {
            long splitSize = last - pos;
            splits.add(new FileSplit(path, pos, splitSize, null));
            lengthRemaining -= splitSize;
            pos = last;/*from ww  w  .j ava 2s. c  om*/
        }
    }
    // create the last split if there is data remaining
    if (lengthRemaining != 0) {
        splits.add(new FileSplit(path, pos, lengthRemaining, null));
    }
    return splits;
}

---

From source file:com.dalabs.droop.util.password.CryptoFileLoader.java

License:Apache License

@Override
public String loadPassword(String p, Configuration configuration) throws IOException {
    LOG.debug("Fetching password from specified path: " + p);
    Path path = new Path(p);
    FileSystem fs = path.getFileSystem(configuration);

    byte[] encrypted;
    try {/*from www  .j a  va  2  s  .  com*/
        verifyPath(fs, path);
        encrypted = readBytes(fs, path);
    } finally {
        fs.close();
    }

    String passPhrase = configuration.get(PROPERTY_CRYPTO_PASSPHRASE);
    if (passPhrase == null) {
        throw new IOException("Passphrase is missing in property " + PROPERTY_CRYPTO_PASSPHRASE);
    }

    String alg = configuration.get(PROPERTY_CRYPTO_ALG, DEFAULT_ALG);
    String algOnly = alg.split("/")[0];
    String salt = configuration.get(PROPERTY_CRYPTO_SALT, DEFAULT_SALT);
    int iterations = configuration.getInt(PROPERTY_CRYPTO_ITERATIONS, DEFAULT_ITERATIONS);
    int keyLen = configuration.getInt(PROPERTY_CRYPTO_KEY_LEN, DEFAULT_KEY_LEN);

    SecretKeyFactory factory = null;
    try {
        factory = SecretKeyFactory.getInstance("PBKDF2WithHmacSHA1");
    } catch (NoSuchAlgorithmException e) {
        throw new IOException("Can't load SecretKeyFactory", e);
    }

    SecretKeySpec key = null;
    try {
        key = new SecretKeySpec(factory
                .generateSecret(new PBEKeySpec(passPhrase.toCharArray(), salt.getBytes(), iterations, keyLen))
                .getEncoded(), algOnly);
    } catch (Exception e) {
        throw new IOException("Can't generate secret key", e);
    }

    Cipher crypto = null;

    try {
        crypto = Cipher.getInstance(alg);
    } catch (Exception e) {
        throw new IOException("Can't initialize the decryptor", e);
    }

    byte[] decryptedBytes;

    try {
        crypto.init(Cipher.DECRYPT_MODE, key);
        decryptedBytes = crypto.doFinal(encrypted);
    } catch (Exception e) {
        throw new IOException("Can't decrypt the password", e);
    }

    return new String(decryptedBytes);
}

---

From source file:com.datasalt.pangool.tuplemr.avro.AvroOutputFormat.java

License:Apache License

static <T> void configureDataFileWriter(DataFileWriter<T> writer, TaskAttemptContext job, String codecName,
        int deflateLevel) throws UnsupportedEncodingException {
    Configuration conf = job.getConfiguration();
    if (FileOutputFormat.getCompressOutput(job)) {
        CodecFactory factory = codecName.equals(DEFLATE_CODEC) ? CodecFactory.deflateCodec(deflateLevel)
                : CodecFactory.fromString(codecName);
        writer.setCodec(factory);/*from   ww w. j a v  a 2 s.  com*/
    }

    writer.setSyncInterval(conf.getInt(SYNC_INTERVAL_KEY, DEFAULT_SYNC_INTERVAL));

    // copy metadata from job
    for (Map.Entry<String, String> e : conf) {
        if (e.getKey().startsWith(AvroJob.TEXT_PREFIX))
            writer.setMeta(e.getKey().substring(AvroJob.TEXT_PREFIX.length()), e.getValue());
        if (e.getKey().startsWith(AvroJob.BINARY_PREFIX))
            writer.setMeta(e.getKey().substring(AvroJob.BINARY_PREFIX.length()),
                    URLDecoder.decode(e.getValue(), "ISO-8859-1").getBytes("ISO-8859-1"));
    }
}

---

From source file:com.datatorrent.contrib.frauddetect.Application.java

License:Open Source License

public BankIdNumberSamplerOperator getBankIdNumberSamplerOperator(String name, DAG dag, Configuration conf) {
    BankIdNumberSamplerOperator oper = dag.addOperator(name, BankIdNumberSamplerOperator.class);
    oper.setThreshold(conf.getInt(BIN_THRESHOLD_PROPERTY, 20));
    // dag.setAttribute(oper, OperatorContext.APPLICATION_WINDOW_COUNT, binSamplerWindowCount);
    dag.setAttribute(oper, OperatorContext.APPLICATION_WINDOW_COUNT, 10);
    return oper;//from w  ww. j  a  v  a 2  s . c  o m
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »