List of usage examples for java.util.concurrent Executors newSingleThreadScheduledExecutor
public static ScheduledExecutorService newSingleThreadScheduledExecutor(ThreadFactory threadFactory)
From source file:org.apache.lens.server.LensServices.java
public synchronized void start() { if (getServiceState() != STATE.STARTED) { super.start(); }/*from w w w. j a v a2s . c o m*/ if (!isServerStatePersistenceEnabled) { log.info("Server restart is not enabled. Not persisting lens server state"); } else { ThreadFactory factory = new BasicThreadFactory.Builder() .namingPattern("Lens-server-snapshotter-Thread-%d").daemon(true).priority(Thread.NORM_PRIORITY) .build(); serverSnapshotScheduler = Executors.newSingleThreadScheduledExecutor(factory); serverSnapshotScheduler.scheduleWithFixedDelay(new Runnable() { @Override public void run() { try { final String runId = UUID.randomUUID().toString(); logSegregationContext.setLogSegregationId(runId); persistLensServiceState(); log.info("SnapShot of Lens Services created"); } catch (Exception e) { incrCounter(SERVER_STATE_PERSISTENCE_ERRORS); log.error("Unable to persist lens server state", e); } } }, serverStatePersistenceInterval, serverStatePersistenceInterval, TimeUnit.MILLISECONDS); log.info("Enabled periodic persistence of lens server state at {} millis interval", serverStatePersistenceInterval); } }
From source file:org.apache.pulsar.broker.service.BrokerService.java
public BrokerService(PulsarService pulsar) throws Exception { this.pulsar = pulsar; this.managedLedgerFactory = pulsar.getManagedLedgerFactory(); this.port = new URI(pulsar.getBrokerServiceUrl()).getPort(); this.tlsPort = new URI(pulsar.getBrokerServiceUrlTls()).getPort(); this.topics = new ConcurrentOpenHashMap<>(); this.replicationClients = new ConcurrentOpenHashMap<>(); this.keepAliveIntervalSeconds = pulsar.getConfiguration().getKeepAliveIntervalSeconds(); this.configRegisteredListeners = new ConcurrentOpenHashMap<>(); this.pendingTopicLoadingQueue = Queues.newConcurrentLinkedQueue(); this.multiLayerTopicsMap = new ConcurrentOpenHashMap<>(); this.pulsarStats = new PulsarStats(pulsar); this.offlineTopicStatCache = new ConcurrentOpenHashMap<>(); final DefaultThreadFactory acceptorThreadFactory = new DefaultThreadFactory("pulsar-acceptor"); final DefaultThreadFactory workersThreadFactory = new DefaultThreadFactory("pulsar-io"); final int numThreads = Runtime.getRuntime().availableProcessors() * 2; log.info("Using {} threads for broker service IO", numThreads); EventLoopGroup acceptorEventLoop, workersEventLoop; if (SystemUtils.IS_OS_LINUX) { try {/*from w ww . j a v a 2s . c o m*/ acceptorEventLoop = new EpollEventLoopGroup(1, acceptorThreadFactory); workersEventLoop = new EpollEventLoopGroup(numThreads, workersThreadFactory); } catch (UnsatisfiedLinkError e) { acceptorEventLoop = new NioEventLoopGroup(1, acceptorThreadFactory); workersEventLoop = new NioEventLoopGroup(numThreads, workersThreadFactory); } } else { acceptorEventLoop = new NioEventLoopGroup(1, acceptorThreadFactory); workersEventLoop = new NioEventLoopGroup(numThreads, workersThreadFactory); } this.acceptorGroup = acceptorEventLoop; this.workerGroup = workersEventLoop; this.statsUpdater = Executors .newSingleThreadScheduledExecutor(new DefaultThreadFactory("pulsar-stats-updater")); if (pulsar.getConfiguration().isAuthorizationEnabled()) { this.authorizationManager = new AuthorizationManager(pulsar.getConfiguration(), pulsar.getConfigurationCache()); } if (pulsar.getConfigurationCache() != null) { pulsar.getConfigurationCache().policiesCache().registerListener(this); } this.inactivityMonitor = Executors .newSingleThreadScheduledExecutor(new DefaultThreadFactory("pulsar-inactivity-monitor")); this.messageExpiryMonitor = Executors .newSingleThreadScheduledExecutor(new DefaultThreadFactory("pulsar-msg-expiry-monitor")); this.backlogQuotaManager = new BacklogQuotaManager(pulsar); this.backlogQuotaChecker = Executors .newSingleThreadScheduledExecutor(new DefaultThreadFactory("pulsar-backlog-quota-checker")); this.authenticationService = new AuthenticationService(pulsar.getConfiguration()); this.dynamicConfigurationCache = new ZooKeeperDataCache<Map<String, String>>(pulsar().getLocalZkCache()) { @Override public Map<String, String> deserialize(String key, byte[] content) throws Exception { return ObjectMapperFactory.getThreadLocal().readValue(content, HashMap.class); } }; this.blockedDispatchers = new ConcurrentOpenHashSet<>(); // update dynamic configuration and register-listener updateConfigurationAndRegisterListeners(); this.lookupRequestSemaphore = new AtomicReference<Semaphore>( new Semaphore(pulsar.getConfiguration().getMaxConcurrentLookupRequest(), false)); this.topicLoadRequestSemaphore = new AtomicReference<Semaphore>( new Semaphore(pulsar.getConfiguration().getMaxConcurrentTopicLoadRequest(), false)); if (pulsar.getConfiguration().getMaxUnackedMessagesPerBroker() > 0 && pulsar.getConfiguration().getMaxUnackedMessagesPerSubscriptionOnBrokerBlocked() > 0.0) { this.maxUnackedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerBroker(); this.maxUnackedMsgsPerDispatcher = (int) ((maxUnackedMessages * pulsar.getConfiguration().getMaxUnackedMessagesPerSubscriptionOnBrokerBlocked()) / 100); log.info("Enabling per-broker unack-message limit {} and dispatcher-limit {} on blocked-broker", maxUnackedMessages, maxUnackedMsgsPerDispatcher); // block misbehaving dispatcher by checking periodically pulsar.getExecutor().scheduleAtFixedRate(() -> checkUnAckMessageDispatching(), 600, 30, TimeUnit.SECONDS); } else { this.maxUnackedMessages = 0; this.maxUnackedMsgsPerDispatcher = 0; log.info( "Disabling per broker unack-msg blocking due invalid unAckMsgSubscriptionPercentageLimitOnBrokerBlocked {} ", pulsar.getConfiguration().getMaxUnackedMessagesPerSubscriptionOnBrokerBlocked()); } PersistentReplicator.setReplicatorQueueSize(pulsar.getConfiguration().getReplicationProducerQueueSize()); }
From source file:password.pwm.util.java.JavaHelper.java
public static ScheduledExecutorService makeSingleThreadExecutorService(final PwmApplication pwmApplication, final Class clazz) { return Executors.newSingleThreadScheduledExecutor( makePwmThreadFactory(JavaHelper.makeThreadName(pwmApplication, clazz) + "-", true)); }
From source file:it.crs4.pydoop.mapreduce.pipes.TaskLog.java
public static ScheduledExecutorService createLogSyncer() { final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() { @Override//from ww w . j a v a 2 s . c o m public Thread newThread(Runnable r) { final Thread t = Executors.defaultThreadFactory().newThread(r); t.setDaemon(true); t.setName("Thread for syncLogs"); return t; } }); ShutdownHookManager.get().addShutdownHook(new Runnable() { @Override public void run() { TaskLog.syncLogsShutdown(scheduler); } }, 50); scheduler.scheduleWithFixedDelay(new Runnable() { @Override public void run() { TaskLog.syncLogs(); } }, 0L, 5L, TimeUnit.SECONDS); return scheduler; }
From source file:com.chiralBehaviors.autoconfigure.AutoConfigure.java
/** * Run the auto configuration process./*from ww w .j ava2 s . c o m*/ * * @param configuredService * - the service instance being configured * @param timeout * - the length of time to wait for auto configuration to * complete * @param unit * - the unit of the wait time * @param environment * - a map of variables that override any configured variables */ public void configure(Map<String, String> environment, AutoConfigureService configuredService, long timeout, TimeUnit unit) { File restartFile = new File(config.restartStateFile); if (restartFile.exists()) { restart(configuredService); return; } if (environment == null) { environment = Collections.emptyMap(); } this.environment.putAll(environment); logger.info(String.format("Using runtime property overrides %s", environment)); logger.info("Beginning auto configuration process"); Runnable successAction = successAction(configuredService); int cardinality = getCardinality(); if (!rendezvous.compareAndSet(null, new Rendezvous(cardinality, successAction, failureAction(configuredService)))) { throw new IllegalStateException("System is already configuring!"); } try { registerService(); } catch (Throwable e) { logger.error("Unable to register this service!", e); failed.set(true); try { configuredService.fail(generatedConfigurations); } catch (Exception e1) { logger.info("Exception encountered during the running failure action", e); } return; } if (cardinality == 0) { // no services required successAction.run(); return; } rendezvous.get().scheduleCancellation(timeout, unit, Executors.newSingleThreadScheduledExecutor( new LabeledThreadFactory("Auto Configuration Scheduling Thread"))); try { registerListeners(); } catch (Throwable e) { logger.error("Error registering service listeners", e); failed.set(true); rendezvous.get().cancel(); try { configuredService.fail(generatedConfigurations); } catch (Exception e1) { logger.error( "Exception encountered during the running failure action", e); } } }
From source file:org.apache.hadoop.hive.ql.exec.tez.WorkloadManager.java
@VisibleForTesting
WorkloadManager(LlapPluginEndpointClientImpl amComm, String yarnQueue, HiveConf conf,
QueryAllocationManager qam, WMFullResourcePlan plan) throws ExecutionException, InterruptedException {
this.yarnQueue = yarnQueue;
this.conf = conf;
this.totalQueryParallelism = determineQueryParallelism(plan);
this.allocationManager = qam;
this.allocationManager.setClusterChangedCallback(() -> notifyOfClusterStateChange());
this.amComm = amComm;
if (this.amComm != null) {
this.amComm.init(conf);
}/* w w w .j a v a 2s . c o m*/
LOG.info("Initializing with " + totalQueryParallelism + " total query parallelism");
this.amRegistryTimeoutMs = (int) HiveConf.getTimeVar(conf, ConfVars.HIVE_SERVER2_TEZ_WM_AM_REGISTRY_TIMEOUT,
TimeUnit.MILLISECONDS);
tezAmPool = new TezSessionPool<>(conf, totalQueryParallelism, true,
oldSession -> createSession(oldSession == null ? null : oldSession.getConf()));
restrictedConfig = new RestrictedConfigChecker(conf);
// Only creates the expiration tracker if expiration is configured.
expirationTracker = SessionExpirationTracker.create(conf, this);
workPool = Executors.newFixedThreadPool(HiveConf.getIntVar(conf, ConfVars.HIVE_SERVER2_WM_WORKER_THREADS),
new ThreadFactoryBuilder().setDaemon(true).setNameFormat("Workload management worker %d").build());
timeoutPool = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryBuilder().setDaemon(true)
.setNameFormat("Workload management timeout thread").build());
allowAnyPool = HiveConf.getBoolVar(conf, ConfVars.HIVE_SERVER2_WM_ALLOW_ANY_POOL_VIA_JDBC);
if (HiveConf.getBoolVar(conf, ConfVars.HIVE_SERVER2_WM_POOL_METRICS)) {
metricsSystem = DefaultMetricsSystem.instance();
} else {
metricsSystem = null;
}
wmThread = new Thread(() -> runWmThread(), "Workload management master");
wmThread.setDaemon(true);
wmThread.start();
updateResourcePlanAsync(plan).get(); // Wait for the initial resource plan to be applied.
objectMapper = new ObjectMapper();
objectMapper.configure(SerializationConfig.Feature.FAIL_ON_EMPTY_BEANS, false);
// serialize json based on field annotations only
objectMapper.setVisibilityChecker(objectMapper.getSerializationConfig().getDefaultVisibilityChecker()
.withGetterVisibility(JsonAutoDetect.Visibility.NONE)
.withSetterVisibility(JsonAutoDetect.Visibility.NONE));
}
From source file:org.apache.hadoop.hive.llap.tezplugins.LlapTaskSchedulerService.java
@VisibleForTesting public LlapTaskSchedulerService(TaskSchedulerContext taskSchedulerContext, Clock clock, boolean initMetrics) { super(taskSchedulerContext); this.clock = clock; this.delayedTaskSchedulerCallable = createDelayedTaskSchedulerCallable(); try {/* w w w.j a v a 2 s . c o m*/ this.conf = TezUtils.createConfFromUserPayload(taskSchedulerContext.getInitialUserPayload()); } catch (IOException e) { throw new TezUncheckedException( "Failed to parse user payload for " + LlapTaskSchedulerService.class.getSimpleName(), e); } this.containerFactory = new ContainerFactory(taskSchedulerContext.getApplicationAttemptId(), taskSchedulerContext.getCustomClusterIdentifier()); // TODO HIVE-13483 Get all of these properties from the registry. This will need to take care of different instances // publishing potentially different values when we support changing configurations dynamically. // For now, this can simply be fetched from a single registry instance. this.nodeBlacklistConf = new NodeBlacklistConf( HiveConf.getTimeVar(conf, ConfVars.LLAP_TASK_SCHEDULER_NODE_REENABLE_MIN_TIMEOUT_MS, TimeUnit.MILLISECONDS), HiveConf.getTimeVar(conf, ConfVars.LLAP_TASK_SCHEDULER_NODE_REENABLE_MAX_TIMEOUT_MS, TimeUnit.MILLISECONDS), HiveConf.getFloatVar(conf, ConfVars.LLAP_TASK_SCHEDULER_NODE_DISABLE_BACK_OFF_FACTOR)); this.numSchedulableTasksPerNode = HiveConf.getIntVar(conf, ConfVars.LLAP_TASK_SCHEDULER_NUM_SCHEDULABLE_TASKS_PER_NODE); long localityDelayMs = HiveConf.getTimeVar(conf, ConfVars.LLAP_TASK_SCHEDULER_LOCALITY_DELAY, TimeUnit.MILLISECONDS); this.localityDelayConf = new LocalityDelayConf(localityDelayMs); this.timeoutMonitor = new SchedulerTimeoutMonitor(); this.timeout = HiveConf.getTimeVar(conf, ConfVars.LLAP_DAEMON_TASK_SCHEDULER_TIMEOUT_SECONDS, TimeUnit.MILLISECONDS); this.timeoutExecutor = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryBuilder().setDaemon(true) .setNameFormat("LlapTaskSchedulerTimeoutMonitor").build()); this.timeoutFuture = null; this.scheduledLoggingExecutor = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryBuilder() .setDaemon(true).setNameFormat("LlapTaskSchedulerTimedLogThread").build()); String instanceId = HiveConf.getTrimmedVar(conf, ConfVars.LLAP_DAEMON_SERVICE_HOSTS); Preconditions.checkNotNull(instanceId, ConfVars.LLAP_DAEMON_SERVICE_HOSTS.varname + " must be defined"); ExecutorService executorServiceRaw = Executors.newSingleThreadExecutor( new ThreadFactoryBuilder().setDaemon(true).setNameFormat("LlapSchedulerNodeEnabler").build()); nodeEnabledExecutor = MoreExecutors.listeningDecorator(executorServiceRaw); ExecutorService delayedTaskSchedulerExecutorRaw = Executors.newFixedThreadPool(1, new ThreadFactoryBuilder() .setDaemon(true).setNameFormat("LlapSchedulerDelayedTaskHandler").build()); delayedTaskSchedulerExecutor = MoreExecutors.listeningDecorator(delayedTaskSchedulerExecutorRaw); ExecutorService schedulerExecutorServiceRaw = Executors.newSingleThreadExecutor( new ThreadFactoryBuilder().setDaemon(true).setNameFormat("LlapScheduler").build()); schedulerExecutor = MoreExecutors.listeningDecorator(schedulerExecutorServiceRaw); if (initMetrics && !conf.getBoolean(ConfVars.HIVE_IN_TEST.varname, false)) { // Initialize the metrics system LlapMetricsSystem.initialize("LlapTaskScheduler"); this.pauseMonitor = new JvmPauseMonitor(conf); pauseMonitor.start(); String displayName = "LlapTaskSchedulerMetrics-" + MetricsUtils.getHostName(); String sessionId = conf.get("llap.daemon.metrics.sessionid"); // TODO: Not sure about the use of this. Should we instead use workerIdentity as sessionId? this.metrics = LlapTaskSchedulerMetrics.create(displayName, sessionId); } else { this.metrics = null; this.pauseMonitor = null; } String hostsString = HiveConf.getVar(conf, ConfVars.LLAP_DAEMON_SERVICE_HOSTS); LOG.info( "Running with configuration: hosts={}, numSchedulableTasksPerNode={}, nodeBlacklistConf={}, localityConf={}", hostsString, numSchedulableTasksPerNode, nodeBlacklistConf, localityDelayConf); }
From source file:org.apache.bookkeeper.client.BookKeeper.java
/** * Contructor for use with the builder. Other constructors also use it. *//*from w ww .j a v a2s.c o m*/ private BookKeeper(ClientConfiguration conf, ZooKeeper zkc, ClientSocketChannelFactory channelFactory, StatsLogger statsLogger, DNSToSwitchMapping dnsResolver, HashedWheelTimer requestTimer, FeatureProvider featureProvider) throws IOException, InterruptedException, KeeperException { this.conf = conf; // initialize zookeeper client if (zkc == null) { this.zk = ZooKeeperClient.newBuilder().connectString(conf.getZkServers()) .sessionTimeoutMs(conf.getZkTimeout()) .operationRetryPolicy( new BoundExponentialBackoffRetryPolicy(conf.getZkTimeout(), conf.getZkTimeout(), 0)) .statsLogger(statsLogger).build(); this.ownZKHandle = true; } else { if (!zkc.getState().isConnected()) { LOG.error("Unconnected zookeeper handle passed to bookkeeper"); throw KeeperException.create(KeeperException.Code.CONNECTIONLOSS); } this.zk = zkc; this.ownZKHandle = false; } // initialize channel factory if (null == channelFactory) { ThreadFactoryBuilder tfb = new ThreadFactoryBuilder(); this.channelFactory = new NioClientSocketChannelFactory( Executors.newCachedThreadPool(tfb.setNameFormat("BookKeeper-NIOBoss-%d").build()), Executors.newCachedThreadPool(tfb.setNameFormat("BookKeeper-NIOWorker-%d").build())); this.ownChannelFactory = true; } else { this.channelFactory = channelFactory; this.ownChannelFactory = false; } if (null == requestTimer) { this.requestTimer = new HashedWheelTimer( new ThreadFactoryBuilder().setNameFormat("BookieClientTimer-%d").build(), conf.getTimeoutTimerTickDurationMs(), TimeUnit.MILLISECONDS, conf.getTimeoutTimerNumTicks()); this.ownTimer = true; } else { this.requestTimer = requestTimer; this.ownTimer = false; } if (null == featureProvider) { this.featureProvider = SettableFeatureProvider.DISABLE_ALL; } else { this.featureProvider = featureProvider; } // initialize scheduler ThreadFactoryBuilder tfb = new ThreadFactoryBuilder().setNameFormat("BookKeeperClientScheduler-%d"); this.scheduler = Executors.newSingleThreadScheduledExecutor(tfb.build()); // initialize stats logger this.statsLogger = statsLogger.scope(BookKeeperClientStats.CLIENT_SCOPE); initOpLoggers(this.statsLogger); // initialize the ensemble placement this.placementPolicy = initializeEnsemblePlacementPolicy(conf, dnsResolver, this.requestTimer, this.featureProvider, this.statsLogger); // initialize main worker pool this.mainWorkerPool = OrderedSafeExecutor.newBuilder().name("BookKeeperClientWorker") .numThreads(conf.getNumWorkerThreads()).statsLogger(statsLogger) .traceTaskExecution(conf.getEnableTaskExecutionStats()) .traceTaskWarnTimeMicroSec(conf.getTaskExecutionWarnTimeMicros()).build(); // initialize bookie client this.bookieClient = new BookieClient(conf, this.channelFactory, this.mainWorkerPool, statsLogger); this.bookieWatcher = new BookieWatcher(conf, this.scheduler, this.placementPolicy, this); this.bookieWatcher.readBookiesBlocking(); // initialize ledger manager this.ledgerManagerFactory = LedgerManagerFactory.newLedgerManagerFactory(conf, this.zk); this.ledgerManager = new CleanupLedgerManager(ledgerManagerFactory.newLedgerManager()); this.ledgerIdGenerator = ledgerManagerFactory.newLedgerIdGenerator(); scheduleBookieHealthCheckIfEnabled(); }
From source file:org.acfun.flume.plugins.maidian.source.AcfunHttpSource.java
@Override public void start() { Preconditions.checkState(srv == null, "Running HTTP Server found in source: " + getName() + " before I started one." + "Will not attempt to start."); srv = new Server(); // Connector Array Connector[] connectors = new Connector[1]; if (sslEnabled) { SslSocketConnector sslSocketConnector = new HTTPSourceSocketConnector(excludedProtocols); sslSocketConnector.setKeystore(keyStorePath); sslSocketConnector.setKeyPassword(keyStorePassword); sslSocketConnector.setReuseAddress(true); connectors[0] = sslSocketConnector; } else {//ww w .j a v a 2 s .c om SelectChannelConnector connector = new SelectChannelConnector(); connector.setReuseAddress(true); connectors[0] = connector; } connectors[0].setHost(host); connectors[0].setPort(port); srv.setConnectors(connectors); try { org.mortbay.jetty.servlet.Context root = new org.mortbay.jetty.servlet.Context(srv, "/", org.mortbay.jetty.servlet.Context.SESSIONS); root.addServlet(new ServletHolder(new FlumeHTTPServlet()), "/"); // ServletHolder holderHome = new ServletHolder(new // DefaultServlet()); // holderHome.setInitParameter("resourceBase","/home/"); // holderHome.setInitParameter("dirAllowed","true"); // holderHome.setInitParameter("pathInfoOnly","true"); // root.addServlet(holderHome, "/crossdomain.xml"); HTTPServerConstraintUtil.enforceConstraints(root); srv.start(); Preconditions.checkArgument(srv.getHandler().equals(root)); } catch (Exception ex) { LOG.error("Error while starting HTTPSource. Exception follows.", ex); Throwables.propagate(ex); } Preconditions.checkArgument(srv.isRunning()); sourceCounter.start(); super.start(); executorService = Executors.newSingleThreadScheduledExecutor( new ThreadFactoryBuilder().setNameFormat("conf-file-poller-%d").build()); AcFunMaidianConfFileWatcherRunnable fileWatcherRunnable = new AcFunMaidianConfFileWatcherRunnable( fileHandlerMap); executorService.scheduleWithFixedDelay(fileWatcherRunnable, 0, 10, TimeUnit.SECONDS); }