Example usage for org.apache.hadoop.conf Configuration getInt

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration getInt.

Prototype

public int getInt(String name, int defaultValue) 

Source Link

Document

Get the value of the name property as an int.

Usage

From source file:com.hortonworks.hbase.replication.bridge.HBaseClient.java

License:Apache License

/**
 * Construct an IPC client whose values are of the given {@link Writable}
 * class.// w w w . j a v a  2s  .co  m
 * @param valueClass value class
 * @param conf configuration
 * @param factory socket factory
 */
public HBaseClient(Class<? extends Writable> valueClass, Configuration conf, SocketFactory factory) {
    this.valueClass = valueClass;
    this.maxIdleTime = conf.getInt("hbase.ipc.client.connection.maxidletime", 10000); //10s
    this.maxRetries = conf.getInt("hbase.ipc.client.connect.max.retries", 0);
    this.failureSleep = conf.getInt("hbase.client.pause", 1000);
    this.tcpNoDelay = conf.getBoolean("hbase.ipc.client.tcpnodelay", false);
    this.tcpKeepAlive = conf.getBoolean("hbase.ipc.client.tcpkeepalive", true);
    this.pingInterval = getPingInterval(conf);
    if (LOG.isDebugEnabled()) {
        LOG.debug("The ping interval is" + this.pingInterval + "ms.");
    }
    this.conf = conf;
    this.socketFactory = factory;
    this.clusterId = conf.get(HConstants.CLUSTER_ID, "default");
    this.connections = new PoolMap<ConnectionId, Connection>(getPoolType(conf), getPoolSize(conf));
    this.failedServers = new FailedServers(conf);
}

---

From source file:com.hortonworks.hbase.replication.bridge.HBaseClient.java

License:Apache License

/**
 * Return the pool size specified in the configuration, which is applicable only if
 * the pool type is {@link PoolType#RoundRobin}.
 *
 * @param config//w  w  w  .ja  v a2 s  .  co  m
 * @return the maximum pool size
 */
protected static int getPoolSize(Configuration config) {
    return config.getInt(HConstants.HBASE_CLIENT_IPC_POOL_SIZE, 1);
}

---

From source file:com.hortonworks.hbase.replication.bridge.HBaseServer.java

License:Apache License

protected HBaseServer(String bindAddress, int port, Class<? extends Writable> paramClass, int handlerCount,
        int priorityHandlerCount, Configuration conf, String serverName, int highPriorityLevel)
        throws IOException {
    this.bindAddress = bindAddress;
    this.conf = conf;
    this.port = port;
    this.paramClass = paramClass;
    this.handlerCount = handlerCount;
    this.priorityHandlerCount = priorityHandlerCount;
    this.socketSendBufferSize = 0;

    // temporary backward compatibility
    String oldMaxQueueSize = this.conf.get("ipc.server.max.queue.size");
    if (oldMaxQueueSize == null) {
        this.maxQueueLength = this.conf.getInt("ipc.server.max.callqueue.length",
                handlerCount * DEFAULT_MAX_CALLQUEUE_LENGTH_PER_HANDLER);
    } else {/*from ww w  . j a v a2 s.  com*/
        LOG.warn("ipc.server.max.queue.size was renamed " + "ipc.server.max.callqueue.length, "
                + "please update your configuration");
        this.maxQueueLength = Integer.getInteger(oldMaxQueueSize);
    }

    this.maxQueueSize = this.conf.getInt("ipc.server.max.callqueue.size", DEFAULT_MAX_CALLQUEUE_SIZE);
    this.readThreads = conf.getInt("ipc.server.read.threadpool.size", 10);
    this.callQueue = new LinkedBlockingQueue<Call>(maxQueueLength);
    if (priorityHandlerCount > 0) {
        this.priorityCallQueue = new LinkedBlockingQueue<Call>(maxQueueLength); // TODO hack on size
    } else {
        this.priorityCallQueue = null;
    }
    this.highPriorityLevel = highPriorityLevel;
    this.maxIdleTime = 2 * conf.getInt("ipc.client.connection.maxidletime", 1000);
    this.maxConnectionsToNuke = conf.getInt("ipc.client.kill.max", 10);
    this.thresholdIdleConnections = conf.getInt("ipc.client.idlethreshold", 4000);
    this.purgeTimeout = conf.getLong("ipc.client.call.purge.timeout", 2 * HConstants.DEFAULT_HBASE_RPC_TIMEOUT);
    this.numOfReplicationHandlers = conf.getInt("hbase.regionserver.replication.handler.count", 3);
    if (numOfReplicationHandlers > 0) {
        this.replicationQueue = new LinkedBlockingQueue<Call>(maxQueueSize);
    }
    // Start the listener here and let it bind to the port
    listener = new Listener();
    this.port = listener.getAddress().getPort();
    this.rpcMetrics = new HBaseRpcMetrics(serverName, Integer.toString(this.port));
    this.tcpNoDelay = conf.getBoolean("ipc.server.tcpnodelay", false);
    this.tcpKeepAlive = conf.getBoolean("ipc.server.tcpkeepalive", true);

    this.warnDelayedCalls = conf.getInt(WARN_DELAYED_CALLS, DEFAULT_WARN_DELAYED_CALLS);
    this.delayedCalls = new AtomicInteger(0);

    this.responseQueuesSizeThrottler = new SizeBasedThrottler(
            conf.getLong(RESPONSE_QUEUES_MAX_SIZE, DEFAULT_RESPONSE_QUEUES_MAX_SIZE));

    // Create the responder here
    responder = new Responder();
}

---

From source file:com.hortonworks.hbase.replication.bridge.ReplicationBridgeServer.java

License:Apache License

/**
 * Starts a HRegionServer at the default location
 *
 * @param conf/*www .j  av  a 2  s.  co m*/
 * @throws IOException
 * @throws InterruptedException
 * @throws KeeperException 
 * @throws ZkConnectException 
 */
public ReplicationBridgeServer(Configuration conf) throws IOException, InterruptedException, KeeperException {
    this.conf = conf;

    // Set how many times to retry talking to another server over HConnection.
    HConnectionManager.setServerSideHConnectionRetries(this.conf, LOG);

    // Server to handle client requests.
    String hostname = conf.get("hbase.regionserver.ipc.address",
            Strings.domainNamePointerToHostName(
                    DNS.getDefaultHost(conf.get("hbase.regionserver.dns.interface", "default"),
                            conf.get("hbase.regionserver.dns.nameserver", "default"))));
    port = conf.getInt("hbase.bridge.server.port", BRIDGE_SERVER_PORT);
    // Creation of a HSA will force a resolve.
    InetSocketAddress initialIsa = new InetSocketAddress(hostname, port);
    if (initialIsa.getAddress() == null) {
        throw new IllegalArgumentException("Failed resolve of " + initialIsa);
    }

    this.rpcServer = HBaseRPC.getServer(this, new Class<?>[] { HRegionInterface.class },
            initialIsa.getHostName(), // BindAddress is IP we got for this server.
            initialIsa.getPort(), conf.getInt("hbase.regionserver.handler.count", 10),
            conf.getInt("hbase.regionserver.metahandler.count", 10),
            conf.getBoolean("hbase.rpc.verbose", false), conf, HConstants.QOS_THRESHOLD);
}

---

From source file:com.iflytek.spider.crawl.AbstractFetchSchedule.java

License:Apache License

public void setConf(Configuration conf) {
    super.setConf(conf);
    if (conf == null)
        return;/*from  ww w.  j  av  a2s  .  com*/
    int oldDefaultInterval = conf.getInt("db.default.fetch.interval", 0);
    defaultInterval = conf.getInt("db.fetch.interval.default", 0);
    if (oldDefaultInterval > 0 && defaultInterval == 0)
        defaultInterval = oldDefaultInterval * SECONDS_PER_DAY;
    int oldMaxInterval = conf.getInt("db.max.fetch.interval", 0);
    maxInterval = conf.getInt("db.fetch.interval.max", 0);
    if (oldMaxInterval > 0 && maxInterval == 0)
        maxInterval = oldMaxInterval * FetchSchedule.SECONDS_PER_DAY;
    LOG.info("defaultInterval=" + defaultInterval);
    LOG.info("maxInterval=" + maxInterval);
}

---

From source file:com.iflytek.spider.crawl.AdaptiveFetchSchedule.java

License:Apache License

public void setConf(Configuration conf) {
    super.setConf(conf);
    if (conf == null)
        return;//from   w  w  w  .  ja v a  2s.  c  o  m
    INC_RATE = conf.getFloat("db.fetch.schedule.adaptive.inc_rate", 0.2f);
    DEC_RATE = conf.getFloat("db.fetch.schedule.adaptive.dec_rate", 0.2f);
    MIN_INTERVAL = conf.getInt("db.fetch.schedule.adaptive.min_interval", 60);
    MAX_INTERVAL = conf.getInt("db.fetch.schedule.adaptive.max_interval", SECONDS_PER_DAY * 365); // 1 year
    SYNC_DELTA = conf.getBoolean("db.fetch.schedule.adaptive.sync_delta", true);
    SYNC_DELTA_RATE = conf.getFloat("db.fetch.schedule.adaptive.sync_delta_rate", 0.2f);
}

---

From source file:com.iflytek.spider.protocol.http.HttpBase.java

License:Apache License

public void setConf(Configuration conf) {
    this.conf = conf;
    this.proxyHost = conf.get("http.proxy.host");
    this.proxyPort = conf.getInt("http.proxy.port", 8080);
    this.useProxy = (proxyHost != null && proxyHost.length() > 0);
    this.timeout = conf.getInt("http.timeout", 10000);
    this.maxContent = conf.getInt("http.content.limit", 64 * 1024);
    this.maxDelays = conf.getInt("http.max.delays", 3);
    this.maxThreadsPerHost = conf.getInt("fetcher.threads.per.host", 1);
    this.userAgent = getAgentString(conf.get("http.agent.name"), conf.get("http.agent.version"),
            conf.get("http.agent.description"), conf.get("http.agent.url"), conf.get("http.agent.email"));
    this.acceptLanguage = conf.get("http.accept.language", acceptLanguage);
    this.serverDelay = (long) (conf.getFloat("fetcher.server.delay", 1.0f) * 1000);
    this.maxCrawlDelay = (long) (conf.getInt("fetcher.max.crawl.delay", -1) * 1000);
    // backward-compatible default setting
    this.byIP = conf.getBoolean("fetcher.threads.per.host.by.ip", true);
    this.useHttp11 = conf.getBoolean("http.useHttp11", false);
    //logConf();/* www  .j a  v a 2 s  . co  m*/
}

---

From source file:com.iflytek.spider.protocol.http.HttpBaseSimply.java

License:Apache License

public void setConf(Configuration conf) {
    this.conf = conf;
    this.proxyHost = conf.get("http.proxy.host");
    this.proxyPort = conf.getInt("http.proxy.port", 8080);
    this.useProxy = (proxyHost != null && proxyHost.length() > 0);
    this.timeout = conf.getInt("http.timeout", 10000);
    this.maxContent = conf.getInt("http.content.limit", 64 * 1024);
    this.userAgent = getAgentString(conf.get("http.agent.name"), conf.get("http.agent.version"),
            conf.get("http.agent.description"), conf.get("http.agent.url"), conf.get("http.agent.email"));
    this.acceptLanguage = conf.get("http.accept.language", acceptLanguage);
    this.useHttp11 = conf.getBoolean("http.useHttp11", false);
    //logConf();/*from  w w w.j a v a2s .  c o  m*/
}

---

From source file:com.iflytek.spider.protocol.httpclient.Http.java

License:Apache License

/**
 * Reads the configuration from the Nutch configuration files and sets the
 * configuration.//from ww  w  .j  a  v a  2 s  .co m
 * 
 * @param conf
 *            Configuration
 */
public void setConf(Configuration conf) {
    super.setConf(conf);
    this.conf = conf;
    this.maxThreadsTotal = conf.getInt("fetcher.threads.fetch", 10);
    this.proxyUsername = conf.get("http.proxy.username", "");
    this.proxyPassword = conf.get("http.proxy.password", "");
    this.proxyRealm = conf.get("http.proxy.realm", "");
    agentHost = conf.get("http.agent.host", "");
    authFile = conf.get("http.auth.file", "");
    configureClient();
    try {
        setCredentials();
    } catch (Exception ex) {
        if (LOG.isFatalEnabled()) {
            LOG.fatal("Could not read " + authFile + " : " + ex.getMessage());
            ex.printStackTrace(LogUtil.getErrorStream(LOG));
        }
    }
}

---

From source file:com.iflytek.spider.util.EncodingDetector.java

License:Apache License

public EncodingDetector(Configuration conf) {
    minConfidence = conf.getInt(MIN_CONFIDENCE_KEY, -1);
    detector = new CharsetDetector();
    clues = new ArrayList<EncodingClue>();
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »