List of usage examples for org.apache.hadoop.conf Configuration getInt
public int getInt(String name, int defaultValue)
name property as an int. From source file:com.alibaba.wasp.fserver.EntityGroup.java
License:Apache License
/** * default//from w w w. j av a 2 s .c o m * * @throws java.io.IOException **/ @SuppressWarnings("unchecked") public EntityGroup(Configuration conf, EntityGroupInfo egi, FTable table, FServerServices service) throws IOException { this.conf = conf; this.ftableDescriptor = table; this.entityGroupInfo = egi; try { Class<? extends Redo> redoClass = (Class<? extends Redo>) conf.getClass(FConstants.REDO_IMPL, RedoLog.class); Constructor<? extends Redo> c = redoClass.getConstructor(EntityGroupInfo.class, Configuration.class); this.redo = c.newInstance(entityGroupInfo, conf); } catch (Throwable e) { throw new IllegalStateException("Could not instantiate a entityGroup instance.", e); } this.transactionRetryPause = conf.getInt(FConstants.WASP_TRANSACTION_RETRY_PAUSE, FConstants.DEFAULT_WASP_TRANSACTION_RETRY_PAUSE); this.rowLockWaitDuration = conf.getInt("wasp.rowlock.wait.duration", DEFAULT_ROWLOCK_WAIT_DURATION); this.services = service; this.leases = new Leases( conf.getInt(FConstants.THREAD_WAKE_FREQUENCY, FConstants.DEFAULT_THREAD_WAKE_FREQUENCY)); this.lockTimeoutPeriod = conf.getInt(FConstants.SELECT_FOR_UPDATE_LOCK_TIMEOUT, FConstants.DEFAULT_SELECT_FOR_UPDATE_LOCK_TIMEOUT); if (this.services != null) { this.metricsEntityGroup = new MetricsEntityGroup(new MetricsEntityGroupWrapperImpl(this)); } else { this.metricsEntityGroup = null; } if (LOG.isDebugEnabled()) { // Write out EntityGroup name as string and its encoded name. LOG.debug("Instantiated " + this); } }
From source file:com.alibaba.wasp.fserver.FServer.java
License:Apache License
/** * Starts a FServer at the default location * //from www . ja va2 s. c o m * @param conf * @throws java.io.IOException * @throws InterruptedException */ public FServer(Configuration conf) throws IOException, InterruptedException { this.conf = conf; this.isOnline = false; // Set how many times to retry talking to another server over FConnection. FConnectionManager.setServerSideFConnectionRetries(this.conf, LOG); // Config'ed params this.msgInterval = conf.getInt("wasp.fserver.msginterval", 3 * 1000); this.sleeper = new Sleeper(this.msgInterval, this); this.numEntityGroupsToReport = conf.getInt("wasp.fserver.numentitygroupstoreport", 10); this.rpcTimeout = conf.getInt(FConstants.WASP_RPC_TIMEOUT_KEY, FConstants.DEFAULT_WASP_RPC_TIMEOUT); this.abortRequested = false; this.stopped = false; this.actionManager = new StorageActionManager(conf); // Server to handle client requests. String hostname = Strings .domainNamePointerToHostName(DNS.getDefaultHost(conf.get("wasp.fserver.dns.interface", "default"), conf.get("wasp.fserver.dns.nameserver", "default"))); int port = conf.getInt(FConstants.FSERVER_PORT, FConstants.DEFAULT_FSERVER_PORT); // Creation of a HSA will force a resolve. InetSocketAddress initialIsa = new InetSocketAddress(hostname, port); if (initialIsa.getAddress() == null) { throw new IllegalArgumentException("Failed resolve of " + initialIsa); } this.rpcServer = WaspRPC.getServer(FServer.class, this, new Class<?>[] { ClientProtocol.class, AdminProtocol.class, WaspRPCErrorHandler.class, OnlineEntityGroups.class }, initialIsa.getHostName(), // BindAddress is // IP we got for // this server. initialIsa.getPort(), conf); // Set our address. this.isa = this.rpcServer.getListenerAddress(); this.leases = new Leases(conf.getInt(FConstants.THREAD_WAKE_FREQUENCY, 10 * 1000)); this.startcode = System.currentTimeMillis(); int maxThreads = conf.getInt("wasp.transaction.threads.max", 150); this.pool = new ThreadPoolExecutor(1, maxThreads, 60, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(), new DaemonThreadFactory("thread factory")); ((ThreadPoolExecutor) this.pool).allowCoreThreadTimeOut(true); this.scannerLeaseTimeoutPeriod = conf.getInt(FConstants.WASP_CLIENT_SCANNER_TIMEOUT_PERIOD, FConstants.DEFAULT_WASP_CLIENT_SCANNER_TIMEOUT_PERIOD); this.driver = new BaseDriver(this); this.splitThread = new SplitThread(this); this.globalEntityGroup = new GlobalEntityGroup(this); }
From source file:com.alibaba.wasp.jdbcx.JdbcConnectionPool.java
License:Apache License
protected JdbcConnectionPool(ConnectionPoolDataSource dataSource, Configuration conf) { this.dataSource = dataSource; this.conf = conf; this.maxConnections = conf.getInt(FConstants.JDBC_POOL_MAX_CONNECTIONS, FConstants.DEFAULT_JDBC_POOL_MAX_CONNECTIONS); this.timeout = conf.getInt(FConstants.JDBC_CONNECTION_TIMEOUT, FConstants.DEFAULT_JDBC_CONNECTION_TIMEOUT); if (dataSource != null) { try {// w ww. ja v a 2 s.co m logWriter = dataSource.getLogWriter(); } catch (SQLException e) { // ignore } } }
From source file:com.alibaba.wasp.master.AssignmentManager.java
License:Apache License
/** * Constructs a new assignment manager./*from ww w. j a v a 2s . c o m*/ * * @param server * @param serverManager * @param service * @param metricsMaster * @throws org.apache.zookeeper.KeeperException * @throws java.io.IOException */ public AssignmentManager(Server server, FServerManager serverManager, final LoadBalancer balancer, final ExecutorService service, MetricsMaster metricsMaster) throws KeeperException, IOException { super(server.getZooKeeper()); this.server = server; this.serverManager = serverManager; this.executorService = service; this.entityGroupsToReopen = Collections.synchronizedMap(new HashMap<String, EntityGroupInfo>()); Configuration conf = server.getConfiguration(); this.timeoutMonitor = new TimeoutMonitor(conf.getInt("wasp.master.assignment.timeoutmonitor.period", 30000), server, serverManager, conf.getInt("wasp.master.assignment.timeoutmonitor.timeout", 600000)); this.timerUpdater = new TimerUpdater(conf.getInt("wasp.master.assignment.timerupdater.period", 10000), server); Threads.setDaemonThreadRunning(timerUpdater.getThread(), server.getServerName() + ".timerUpdater"); this.zkTable = new ZKTable(this.watcher); this.maximumAttempts = this.server.getConfiguration().getInt("wasp.assignment.maximum.attempts", 10); this.balancer = balancer; int maxThreads = conf.getInt("wasp.assignment.threads.max", 30); this.threadPoolExecutorService = Threads.getBoundedCachedThreadPool(maxThreads, 60L, TimeUnit.SECONDS, newDaemonThreadFactory("hbase-am")); this.metricsMaster = metricsMaster;// can be null only with tests. this.entityGroupStates = new EntityGroupStates(server, serverManager); int workers = conf.getInt("wasp.assignment.zkevent.workers", 5); zkEventWorkers = new java.util.concurrent.ExecutorService[workers]; ThreadFactory threadFactory = newDaemonThreadFactory("am-zkevent-worker"); for (int i = 0; i < workers; i++) { zkEventWorkers[i] = Threads.getBoundedCachedThreadPool(1, 60L, TimeUnit.SECONDS, threadFactory); } }
From source file:com.alibaba.wasp.master.FMaster.java
License:Apache License
/** * Initializes the FMaster. The steps are as follows: * <p>//from ww w . jav a 2 s . c o m * <ol> * <li>Initialize FMaster RPC and address * <li>Connect to ZooKeeper. * </ol> * <p> * Remaining steps of initialization occur in {@link #run()} so that they run * in their own thread rather than within the context of the constructor. * * @throws InterruptedException */ public FMaster(final Configuration conf) throws IOException, KeeperException, InterruptedException { this.conf = new Configuration(conf); // Set how many times to retry talking to another server over HConnection. FConnectionManager.setServerSideFConnectionRetries(this.conf, LOG); // Server to handle client requests. String hostname = Strings .domainNamePointerToHostName(DNS.getDefaultHost(conf.get("wasp.master.dns.interface", "default"), conf.get("wasp.master.dns.nameserver", "default"))); int port = conf.getInt(FConstants.MASTER_PORT, FConstants.DEFAULT_MASTER_PORT); // Creation of a ISA will force a resolve. InetSocketAddress initialIsa = new InetSocketAddress(hostname, port); if (initialIsa.getAddress() == null) { throw new IllegalArgumentException("Failed resolve of " + initialIsa); } this.rpcServer = WaspRPC.getServer(FMaster.class, this, new Class<?>[] { FMasterMonitorProtocol.class, FMasterAdminProtocol.class, FServerStatusProtocol.class, FMetaServerProtocol.class }, initialIsa.getHostName(), // BindAddress is IP we got for this server. initialIsa.getPort(), conf); // Set our address. this.isa = this.rpcServer.getListenerAddress(); this.serverName = new ServerName(this.isa.getHostName(), this.isa.getPort(), System.currentTimeMillis()); // set the thread name now we have an address setName(MASTER + "-" + this.serverName.toString()); this.zooKeeper = new ZooKeeperWatcher(conf, MASTER + ":" + isa.getPort(), this, true); // metrics interval: using the same property as fserver. this.msgInterval = conf.getInt("wasp.fserver.msginterval", 3 * 1000); this.metricsMaster = new MetricsMaster(new MetricsMasterWrapperImpl(this)); }
From source file:com.alibaba.wasp.master.FMaster.java
License:Apache License
/** * Stall startup if we are designated a backup master; i.e. we want someone * else to become the master before proceeding. * // w w w.ja v a2s . c o m * @param c * @param amm * @throws InterruptedException */ private static void stallIfBackupMaster(final Configuration c, final ActiveMasterManager amm) throws InterruptedException { // If we're a backup master, stall until a primary to writes his address if (!c.getBoolean(FConstants.MASTER_TYPE_BACKUP, FConstants.DEFAULT_MASTER_TYPE_BACKUP)) { return; } LOG.debug("FMaster started in backup mode. " + "Stalling until master znode is written."); // This will only be a minute or so while the cluster starts up, // so don't worry about setting watches on the parent znode while (!amm.isActiveMaster()) { LOG.debug("Waiting for master address ZNode to be written " + "(Also watching cluster state node)"); Thread.sleep(c.getInt("zookeeper.session.timeout", 180 * 1000)); } }
From source file:com.alibaba.wasp.master.FMasterCommandLine.java
License:Apache License
private int startMaster() { Configuration conf = getConf(); try {// w ww . j a va2 s .c o m // If 'local', defer to LocalWaspCluster instance. Starts master // and fserver both in the one JVM. if (LocalWaspCluster.isLocal(conf)) { final MiniZooKeeperCluster zooKeeperCluster = new MiniZooKeeperCluster(); File zkDataPath = new File(conf.get(FConstants.ZOOKEEPER_DATA_DIR)); int zkClientPort = conf.getInt(FConstants.ZOOKEEPER_CLIENT_PORT, 0); if (zkClientPort == 0) { throw new IOException("No config value for " + FConstants.ZOOKEEPER_CLIENT_PORT); } zooKeeperCluster.setDefaultClientPort(zkClientPort); int clientPort = zooKeeperCluster.startup(zkDataPath); if (clientPort != zkClientPort) { String errorMsg = "Could not start ZK at requested port of " + zkClientPort + ". ZK was started at port: " + clientPort + ". Aborting as clients (e.g. shell) will not be able to find " + "this ZK quorum."; System.err.println(errorMsg); throw new IOException(errorMsg); } conf.set(FConstants.ZOOKEEPER_CLIENT_PORT, Integer.toString(clientPort)); // Need to have the zk cluster shutdown when master is shutdown. // Run a subclass that does the zk cluster shutdown on its way out. LocalWaspCluster cluster = new LocalWaspCluster(conf, 1, 1, LocalFMaster.class, FServer.class); ((LocalFMaster) cluster.getMaster(0)).setZKCluster(zooKeeperCluster); cluster.startup(); waitOnMasterThreads(cluster); } else { FMaster master = FMaster.constructMaster(masterClass, conf); if (master.isStopped()) { LOG.info("Won't bring the Master up as a shutdown is requested"); return -1; } master.start(); master.join(); if (master.isAborted()) throw new RuntimeException("FMaster Aborted"); } } catch (Throwable t) { LOG.error("Failed to start master", t); return -1; } return 0; }
From source file:com.alibaba.wasp.messagequeue.MessageBroker.java
License:Apache License
/** * @param service//from w ww . j a v a 2s.c om * entityGroup server instance * @param conf * configuration */ public MessageBroker(OnlineEntityGroups service, Configuration conf) { this.service = service; this.conf = conf; this.leases = new Leases(conf.getInt(FConstants.THREAD_WAKE_FREQUENCY, 10 * 1000)); this.subscriberLeaseTimeoutPeriod = conf.getInt(FConstants.WASP_FSEVER_SUBSCRIBER_TIMEOUT_PERIOD, FConstants.DEFAULT_WASP_FSEVER_SUBSCRIBER_TIMEOUT_PERIOD); this.renew = new RenewRunnable(); this.renew.start(); }
From source file:com.alibaba.wasp.meta.FMetaScanner.java
License:Apache License
public static void metaScan(Configuration configuration, MetaScannerVisitor visitor, byte[] tableName, byte[] row, int rowLimit) throws IOException { int rowUpperLimit = rowLimit > 0 ? rowLimit : Integer.MAX_VALUE; // if row is not null, we want to use the startKey of the row's entityGroup // as//from w ww .j a va 2s . c o m // the startRow for the meta scan. byte[] startRow = tableName; if (tableName == null || tableName.length == 0) { // Full META scan startRow = FConstants.EMPTY_START_ROW; } String metaTableString = configuration.get(FConstants.METASTORE_TABLE, FConstants.DEFAULT_METASTORE_TABLE); HTable fmetaTable = new HTable(configuration, metaTableString); // Scan over each meta entityGroup int rows = Math.min(rowLimit, configuration.getInt(HConstants.HBASE_META_SCANNER_CACHING, HConstants.DEFAULT_HBASE_META_SCANNER_CACHING)); final Scan scan = new Scan(startRow); if (LOG.isDebugEnabled()) { LOG.debug("Scanning " + metaTableString + " starting at row=" + Bytes.toStringBinary(startRow) + " for max=" + rowUpperLimit + " rows "); } scan.addFamily(FConstants.CATALOG_FAMILY); scan.setCaching(rows); if (tableName == null || tableName.length == 0) { FilterList allFilters = new FilterList(); allFilters.addFilter(new PrefixFilter(tableName)); scan.setFilter(allFilters); } int processedRows = 0; try { ResultScanner scanner = fmetaTable.getScanner(scan); for (Result r = scanner.next(); r != null; r = scanner.next()) { if (processedRows >= rowUpperLimit) { break; } if (Bytes.startsWith(r.getRow(), FConstants.TABLEROW_PREFIX)) { continue; } if (!visitor.processRow(r)) break; // exit completely processedRows++; } // here, we didn't break anywhere. Check if we have more rows } finally { // Close scanner fmetaTable.close(); } }
From source file:com.alibaba.wasp.meta.FMetaStore.java
License:Apache License
public FMetaStore(Configuration conf) throws MetaException { this.setConf(conf); fMetaServices = new FMetaServicesImplWithoutRetry(conf); maxRetries = conf.getInt(FConstants.WASP_FMETA_RETRIES_NUMBER, FConstants.DEFAULT_WASP_FMETA_RETRIES_NUMBER); pause = conf.getLong(HConstants.HBASE_CLIENT_PAUSE, HConstants.DEFAULT_HBASE_CLIENT_PAUSE); }