Example usage for org.apache.hadoop.conf Configuration getInt

List of usage examples for org.apache.hadoop.conf Configuration getInt

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.conf.*
  5. Configuration
  6. getInt

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration getInt.

Prototype

public int getInt(String name, int defaultValue)

Source Link

Document

Get the value of the name property as an int.

Usage

From source file:com.splout.db.engine.MySQLOutputFormat.java

License:Apache License

/**
 * Loads global variable configuration//w w w .  j a v  a  2 s  .c  o m
 */
protected void loadGlobalConf() {
    Configuration conf = getConf();
    if (conf.get(GLOBAL_AUTO_TRIM_STRING) != null) {
        globalAutoTrim = conf.getBoolean(GLOBAL_AUTO_TRIM_STRING, true);
    }
    if (conf.get(GLOBAL_STRING_FIELD_SIZE) != null) {
        globalStringFieldSize = conf.getInt(GLOBAL_STRING_FIELD_SIZE, 255);
    }
}

From source file:com.ssamples.hbase.stochasticbalancer.StochasticLoadBalancer.java

License:Apache License

@Override
public synchronized void setConf(Configuration conf) {
    super.setConf(conf);
    maxSteps = conf.getInt(MAX_STEPS_KEY, maxSteps);
    stepsPerRegion = conf.getInt(STEPS_PER_REGION_KEY, stepsPerRegion);
    maxRunningTime = conf.getLong(MAX_RUNNING_TIME_KEY, maxRunningTime);
    runMaxSteps = conf.getBoolean(RUN_MAX_STEPS_KEY, runMaxSteps);

    numRegionLoadsToRemember = conf.getInt(KEEP_REGION_LOADS, numRegionLoadsToRemember);
    isByTable = conf.getBoolean(HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE, isByTable);
    minCostNeedBalance = conf.getFloat(MIN_COST_NEED_BALANCE_KEY, minCostNeedBalance);
    if (localityCandidateGenerator == null) {
        localityCandidateGenerator = new LocalityBasedCandidateGenerator(services);
    }//from w w  w .  ja va2s. com
    localityCost = new ServerLocalityCostFunction(conf, services);
    rackLocalityCost = new RackLocalityCostFunction(conf, services);

    if (this.candidateGenerators == null) {
        candidateGenerators = Lists.newArrayList();
        candidateGenerators.add(new RandomCandidateGenerator());
        candidateGenerators.add(new LoadCandidateGenerator());
        candidateGenerators.add(localityCandidateGenerator);
        candidateGenerators.add(new RegionReplicaRackCandidateGenerator());
    }
    regionLoadFunctions = new CostFromRegionLoadFunction[] { new ReadRequestCostFunction(conf),
            new CPRequestCostFunction(conf), new WriteRequestCostFunction(conf),
            new MemStoreSizeCostFunction(conf), new StoreFileCostFunction(conf) };
    regionReplicaHostCostFunction = new RegionReplicaHostCostFunction(conf);
    regionReplicaRackCostFunction = new RegionReplicaRackCostFunction(conf);
    costFunctions = new CostFunction[] { new RegionCountSkewCostFunction(conf),
            new PrimaryRegionCountSkewCostFunction(conf), new MoveCostFunction(conf), localityCost,
            rackLocalityCost, new TableSkewCostFunction(conf), regionReplicaHostCostFunction,
            regionReplicaRackCostFunction, regionLoadFunctions[0], regionLoadFunctions[1],
            regionLoadFunctions[2], regionLoadFunctions[3], regionLoadFunctions[4] };
    curFunctionCosts = new Double[costFunctions.length];
    tempFunctionCosts = new Double[costFunctions.length];
    LOG.info("Loaded config; maxSteps=" + maxSteps + ", stepsPerRegion=" + stepsPerRegion + ", maxRunningTime="
            + maxRunningTime + ", isByTable=" + isByTable + ", etc.");
}

From source file:com.ssamples.hbase.stochasticbalancer.StochasticLoadBalancerNew.java

License:Apache License

@Override
public synchronized void setConf(Configuration conf) {
    super.setConf(conf);
    maxSteps = conf.getInt(MAX_STEPS_KEY, maxSteps);
    stepsPerRegion = conf.getInt(STEPS_PER_REGION_KEY, stepsPerRegion);
    maxRunningTime = conf.getLong(MAX_RUNNING_TIME_KEY, maxRunningTime);
    runMaxSteps = conf.getBoolean(RUN_MAX_STEPS_KEY, runMaxSteps);

    numRegionLoadsToRemember = conf.getInt(KEEP_REGION_LOADS, numRegionLoadsToRemember);
    isByTable = conf.getBoolean(HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE, isByTable);
    minCostNeedBalance = conf.getFloat(MIN_COST_NEED_BALANCE_KEY, minCostNeedBalance);
    if (localityCandidateGenerator == null) {
        localityCandidateGenerator = new LocalityBasedCandidateGenerator(services);
    }/*from   ww  w. j  av  a2s .  c om*/
    localityCost = new ServerLocalityCostFunction(conf, services);
    rackLocalityCost = new RackLocalityCostFunction(conf, services);

    if (this.candidateGenerators == null) {
        candidateGenerators = Lists.newArrayList();
        candidateGenerators.add(new RandomCandidateGenerator());
        candidateGenerators.add(new LoadCandidateGenerator());
        candidateGenerators.add(localityCandidateGenerator);
        candidateGenerators.add(new RegionReplicaRackCandidateGenerator());
    }
    regionLoadFunctions = new CostFromRegionLoadFunction[] { new ReadRequestCostFunction(conf),
            new CPRequestCostFunction(conf), new WriteRequestCostFunction(conf),
            new MemStoreSizeCostFunction(conf), new StoreFileCostFunction(conf) };
    regionReplicaHostCostFunction = new RegionReplicaHostCostFunction(conf);
    regionReplicaRackCostFunction = new RegionReplicaRackCostFunction(conf);
    costFunctions = new CostFunction[] {
            //new RegionCountSkewCostFunction(conf),
            new PrimaryRegionCountSkewCostFunction(conf), new MoveCostFunction(conf), localityCost,
            rackLocalityCost,
            //new TableSkewCostFunction(conf),
            new TableSkewCostFunctionNew(conf),
            //new TableRegionSkewCostFunction(conf),
            new ServerResourceCostFunction(conf), regionReplicaHostCostFunction, regionReplicaRackCostFunction,
            regionLoadFunctions[0], regionLoadFunctions[1], regionLoadFunctions[2], regionLoadFunctions[3],
            regionLoadFunctions[4] };
    curFunctionCosts = new Double[costFunctions.length];
    tempFunctionCosts = new Double[costFunctions.length];
    LOG.info("Loaded config; maxSteps=" + maxSteps + ", stepsPerRegion=" + stepsPerRegion + ", maxRunningTime="
            + maxRunningTime + ", isByTable=" + isByTable + ", etc.");
}

From source file:com.streamsets.pipeline.lib.util.AvroToParquetConverterUtil.java

License:Apache License

public static ParquetWriter.Builder initializeWriter(Path tempFile, Schema avroSchema, Configuration conf)
        throws IOException {

    //private final ByteArrayOutputStream baos = new ByteArrayOutputStream();

    // Detect Parquet version to see if it supports logical types
    LOG.info("Detected Parquet version: " + Version.FULL_VERSION);

    // Parquet Avro pre-1.9 doesn't work with logical types, so in that case we use custom Builder that injects our own
    // avro schema -> parquet schema generator class (which is a copy of the one that was provided in PARQUET-358).
    ParquetWriter.Builder builder = null;
    try {/*from w  ww .ja  v  a2 s.  c o m*/
        SemanticVersion parquetVersion = SemanticVersion.parse(Version.VERSION_NUMBER);
        if (parquetVersion.major > 1 || (parquetVersion.major == 1 && parquetVersion.minor >= 9)) {
            builder = AvroParquetWriter.builder(tempFile).withSchema(avroSchema);
        } else {
            builder = new AvroParquetWriterBuilder(tempFile).withSchema(avroSchema);
        }
    } catch (SemanticVersion.SemanticVersionParseException e) {
        LOG.warn("Can't parse parquet version string: " + Version.VERSION_NUMBER, e);
        builder = new AvroParquetWriterBuilder(tempFile).withSchema(avroSchema);
    }

    // Generic arguments from the Job
    if (propertyDefined(conf, AvroParquetConstants.COMPRESSION_CODEC_NAME)) {
        String codec = conf.get(AvroParquetConstants.COMPRESSION_CODEC_NAME);
        LOG.info("Using compression codec: {}", codec);
        builder.withCompressionCodec(CompressionCodecName.fromConf(codec));
    }
    if (propertyDefined(conf, AvroParquetConstants.ROW_GROUP_SIZE)) {
        int size = conf.getInt(AvroParquetConstants.ROW_GROUP_SIZE, -1);
        LOG.info("Using row group size: {}", size);
        builder.withRowGroupSize(size);
    }
    if (propertyDefined(conf, AvroParquetConstants.PAGE_SIZE)) {
        int size = conf.getInt(AvroParquetConstants.PAGE_SIZE, -1);
        LOG.info("Using page size: {}", size);
        builder.withPageSize(size);
    }
    if (propertyDefined(conf, AvroParquetConstants.DICTIONARY_PAGE_SIZE)) {
        int size = conf.getInt(AvroParquetConstants.DICTIONARY_PAGE_SIZE, -1);
        LOG.info("Using dictionary page size: {}", size);
        builder.withDictionaryPageSize(size);
    }
    if (propertyDefined(conf, AvroParquetConstants.MAX_PADDING_SIZE)) {
        int size = conf.getInt(AvroParquetConstants.MAX_PADDING_SIZE, -1);
        LOG.info("Using max padding size: {}", size);
        builder.withMaxPaddingSize(size);
    }

    return builder;
}

From source file:com.streamsets.pipeline.stage.destination.mapreduce.jobtype.avroorc.AvroOrcConvertMapper.java

License:Apache License

@Override
protected void initializeWriter(Path tempFile, Schema avroSchema, Configuration conf, Context context)
        throws IOException {

    int batchSize = AvroToOrcRecordConverter.DEFAULT_ORC_BATCH_SIZE;
    if (propertyDefined(conf, AvroOrcConstants.ORC_BATCH_SIZE)) {
        batchSize = conf.getInt(AvroOrcConstants.ORC_BATCH_SIZE,
                AvroToOrcRecordConverter.DEFAULT_ORC_BATCH_SIZE);
        LOG.info("Using ORC batch size: {}", batchSize);
    }/*  ww w . j av  a 2  s.  co  m*/

    LOG.info("Classpath: \n{}", System.getProperty("java.class.path"));

    avroOrcRecordConverter = new AvroToOrcRecordConverter(batchSize, new Properties(), conf);
    avroOrcRecordConverter.initializeWriter(avroSchema, tempFile);
}

From source file:com.taobao.adfs.database.DatabaseExecutor.java

License:Apache License

public static DatabaseExecutor get(Configuration conf) throws IOException {
    String clientClassName = null;
    try {/*from  w w w.j  ava2 s  .  c o  m*/
        clientClassName = conf.get("database.executor.class.name",
                DatabaseExecutorForMysqlClient.class.getName());
        int clientNumber = conf.getInt("database.executor.client.number", 1);
        if (clientNumber < 1)
            clientNumber = 1;
        Utilities.logInfo(logger, "create DatabaseExecutor with database.executor.class.name=",
                clientClassName);
        Utilities.logInfo(logger, "create DatabaseExecutor with database.executor.client.number=",
                clientNumber);
        @SuppressWarnings("unchecked")
        Class<DatabaseExecutor> clientClass = (Class<DatabaseExecutor>) Class.forName(clientClassName);
        Constructor<DatabaseExecutor> clientConstructor = clientClass.getConstructor(Configuration.class);
        return clientConstructor.newInstance(conf);
    } catch (Throwable t) {
        throw new IOException(
                "fail to create DatabaseExecutor with database.executor.class.name=" + clientClassName, t);
    }
}

From source file:com.taobao.adfs.database.MysqlServerController.java

License:Apache License

public void formatData(Configuration conf) throws Throwable {
    Utilities.logInfo(logger, "mysql server is formatting");
    setMysqlDefaultConf(conf);//from w w  w . j  a v a  2s  .  c  o  m
    // stop mysql server and initialize data
    String mysqlServerPid = stopServer(conf);
    backupData(conf);
    String mysqlDataPath = Utilities.getNormalPath(conf.get("mysql.server.data.path", "."));
    Utilities.mkdirs(mysqlDataPath, true);
    String commandForCreateMysqlData = "mysql_install_db";
    commandForCreateMysqlData += " --force";
    commandForCreateMysqlData += " --general_log";
    commandForCreateMysqlData += " --no-defaults";
    commandForCreateMysqlData += " --basedir=" + getMysqlConf(conf, "mysqld.basedir");
    commandForCreateMysqlData += " --datadir=" + mysqlDataPath;
    Utilities.logInfo(logger, "mysql server is installing new data, command=", commandForCreateMysqlData);
    Utilities.runCommand(commandForCreateMysqlData, 0, conf.get("mysql.server.bin.path"), null);
    Utilities.logInfo(logger, "mysql server has installed new data");

    // start mysql server and set access control
    startServer(conf);
    String commandForSetMysqlAccess = "mysql -uroot";
    commandForSetMysqlAccess += " --socket=" + getMysqlConf(conf, "mysqld.socket");
    // commandForSetMysqlServerPassword += " password '" + conf.get("mysql.password", "root") + "'";
    commandForSetMysqlAccess += " --execute=\"";
    commandForSetMysqlAccess += "use mysql;delete from user;grant all privileges on *.* to 'root'@'%' identified by 'root';flush privileges;";
    commandForSetMysqlAccess += "\"";
    Utilities.logInfo(logger, "mysql server is setting privileges, command=", commandForSetMysqlAccess);
    Utilities.runCommand(commandForSetMysqlAccess, 0, conf.get("mysql.server.bin.path"),
            getMysqlLibPath(conf.get("mysql.server.bin.path")));
    Utilities.logInfo(logger, "mysql server has set privileges");

    // create database
    try {
        createDatabase(conf);
    } catch (Throwable t) {
        int retryIndex = conf.getInt("mysql.server.format.retry.index", 0);
        if (retryIndex >= conf.getInt("mysql.server.format.retry.max", 3))
            throw new IOException(t);
        conf.setInt("mysql.server.format.retry.index", ++retryIndex);
        Utilities.logError(logger, "mysql server fails to create database, retryIndex=", retryIndex, t);
        formatData(conf);
        return;
    }

    // restore mysql server status before format
    if (mysqlServerPid.isEmpty() && conf.getBoolean("mysql.server.restore", false))
        stopServer(conf);
    Utilities.logInfo(logger, "mysql server is formatted");
}

From source file:com.taobao.adfs.distributed.rpc.Client.java

License:Apache License

/**
 * Get the ping interval from configuration; If not set in the configuration, return the default value.
 * //w ww .ja  va2 s.  c  o  m
 * @param conf
 *          Configuration
 * @return the ping interval
 */
final static int getPingInterval(Configuration conf) {
    return conf.getInt(PING_INTERVAL_NAME, DEFAULT_PING_INTERVAL);
}

From source file:com.taobao.adfs.distributed.rpc.Client.java

License:Apache License

/**
 * Construct an IPC client whose values are of the given {@link Writable} class.
 *///from  www . j av a  2 s. c  om
public Client(Class<? extends Writable> valueClass, Configuration conf, SocketFactory factory) {
    this.valueClass = valueClass;
    this.maxIdleTime = conf.getInt("ipc.client.connection.maxidletime", 3600000); // 3600s
    this.maxRetriesForIOException = conf.getInt("ipc.client.connect.max.retries.ioexception", 0);
    this.maxRetriesForSocketTimeoutException = conf
            .getInt("ipc.client.connect.max.retries.sockettimeoutexception", 0);
    this.tcpNoDelay = conf.getBoolean("ipc.client.tcpnodelay", true);
    this.pingInterval = getPingInterval(conf);
    if (LOG.isDebugEnabled()) {
        LOG.debug("The ping interval is" + this.pingInterval + "ms.");
    }
    this.conf = conf;
    this.socketFactory = factory;
    this.connectionNumberPerAddress = conf.getInt("ipc.client.connection.number.per.address", 1);
    if (LOG.isDebugEnabled()) {
        LOG.debug("ipc.client.connection.number.per.address=" + connectionNumberPerAddress);
    }
}

From source file:com.taobao.adfs.distributed.rpc.Server.java

License:Apache License

/**
 * Constructs a server listening on the named port and address. Parameters passed must be of the named class. The
 * <code>handlerCount</handlerCount> determines
 * the number of handler threads that will be used to process calls.
 * /*  w ww.j a v  a 2  s  .c  o m*/
 */
protected Server(String bindAddress, int port, Class<? extends Writable> paramClass, int handlerCount,
        Configuration conf, String serverName) throws IOException {
    this.bindAddress = bindAddress;
    this.conf = conf;
    this.port = port;
    this.paramClass = paramClass;
    this.handlerCount = handlerCount;
    this.socketSendBufferSize = 0;
    this.maxQueueSize = handlerCount * MAX_QUEUE_SIZE_PER_HANDLER;
    this.callQueue = new LinkedBlockingQueue<Call>(maxQueueSize);
    this.readThreads = conf.getInt("ipc.server.read.threadpool.size", 40);
    this.maxIdleTime = 2 * conf.getInt("ipc.client.connection.maxidletime", 1000);
    this.maxConnectionsToNuke = conf.getInt("ipc.client.kill.max", 10);
    this.thresholdIdleConnections = conf.getInt("ipc.client.idlethreshold", 4000);

    // Start the listener here and let it bind to the port
    listener = new Listener();
    this.port = listener.getAddress().getPort();
    this.tcpNoDelay = conf.getBoolean("ipc.server.tcpnodelay", false);

    // Create the responder here
    responder = new Responder();
}