List of usage examples for java.util.concurrent SynchronousQueue SynchronousQueue
public SynchronousQueue()
From source file:org.apache.hive.jdbc.TestJdbcWithMiniHS2.java
private static void startConcurrencyTest(Connection conn, String tableName, int numTasks) { // Start concurrent testing int POOL_SIZE = 100; int TASK_COUNT = numTasks; SynchronousQueue<Runnable> executorQueue = new SynchronousQueue<Runnable>(); ExecutorService workers = new ThreadPoolExecutor(1, POOL_SIZE, 20, TimeUnit.SECONDS, executorQueue); List<Future<Boolean>> list = new ArrayList<Future<Boolean>>(); int i = 0;//from w w w .j ava 2 s . c o m while (i < TASK_COUNT) { try { Future<Boolean> future = workers.submit(new JDBCTask(conn, i, tableName)); list.add(future); i++; } catch (RejectedExecutionException ree) { try { TimeUnit.MILLISECONDS.sleep(100); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } } } for (Future<Boolean> future : list) { try { Boolean result = future.get(30, TimeUnit.SECONDS); assertTrue(result); } catch (ExecutionException ee) { fail("Concurrent Statement failed: " + ee.getCause()); } catch (TimeoutException te) { System.out.println("Task was timeout after 30 second: " + te); } catch (CancellationException ce) { System.out.println("Task was interrupted: " + ce); } catch (InterruptedException ie) { System.out.println("Thread was interrupted: " + ie); } } workers.shutdown(); }
From source file:org.apache.hadoop.hbase.client.HTable.java
public static ThreadPoolExecutor getDefaultExecutor(Configuration conf) { int maxThreads = conf.getInt("hbase.htable.threads.max", Integer.MAX_VALUE); if (maxThreads == 0) { maxThreads = 1; // is there a better default? }/*from www .j a va 2s . c o m*/ long keepAliveTime = conf.getLong("hbase.htable.threads.keepalivetime", 60); // Using the "direct handoff" approach, new threads will only be created // if it is necessary and will grow unbounded. This could be bad but in HCM // we only create as many Runnables as there are region servers. It means // it also scales when new region servers are added. ThreadPoolExecutor pool = new ThreadPoolExecutor(1, maxThreads, keepAliveTime, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(), Threads.newDaemonThreadFactory("htable")); ((ThreadPoolExecutor) pool).allowCoreThreadTimeOut(true); return pool; }
From source file:com.offbynull.portmapper.upnpigd.UpnpIgdDiscovery.java
private static Map<UpnpIgdServiceReference, byte[]> getServiceDescriptions( Set<UpnpIgdServiceReference> services) throws InterruptedException { Map<UpnpIgdServiceReference, byte[]> serviceXmls = new HashMap(); ExecutorService executorService = null; try {/* www. jav a2 s . com*/ int maximumPoolSize = (int) ((double) Runtime.getRuntime().availableProcessors() / (1.0 - 0.95)); executorService = new ThreadPoolExecutor(0, maximumPoolSize, 1, TimeUnit.SECONDS, new SynchronousQueue<Runnable>()); List<HttpRequestCallable<UpnpIgdServiceReference>> tasks = new LinkedList<>(); for (UpnpIgdServiceReference service : services) { tasks.add(new HttpRequestCallable<>(service.getScpdUrl(), service)); } List<Future<Pair<UpnpIgdServiceReference, byte[]>>> results = executorService.invokeAll(tasks); for (Future<Pair<UpnpIgdServiceReference, byte[]>> result : results) { try { Pair<UpnpIgdServiceReference, byte[]> data = result.get(); serviceXmls.put(data.getKey(), data.getValue()); } catch (InterruptedException | ExecutionException | CancellationException e) { // NOPMD // do nothing, skip } } } finally { if (executorService != null) { executorService.shutdownNow(); } } return serviceXmls; }
From source file:com.emarsys.dyson.storage.DefaultDysonStorage.java
/** * Creates and starts the {@link #storageService * storage's executor service} as well as the * {@link #deliveredMailProcessor} in its own {@link Thread}. * //from w w w .j a va2 s . c o m */ protected void startStorageServices() { if (this.storageService == null) { this.storageService = new ThreadPoolExecutor(0, Integer.MAX_VALUE, 5L, TimeUnit.SECONDS, new SynchronousQueue<Runnable>()); } if (this.deliveredMailProcessor == null) { this.deliveredMailProcessor = new DeliveredMailProcessor(); } if (!this.deliveredMailProcessor.isRunning()) { Thread th = new Thread(this.deliveredMailProcessor, "DeliveredMailProcessor"); th.start(); } }
From source file:com.alibaba.wasp.fserver.FServer.java
/** * Starts a FServer at the default location * /*w w w . j a va2 s . c om*/ * @param conf * @throws java.io.IOException * @throws InterruptedException */ public FServer(Configuration conf) throws IOException, InterruptedException { this.conf = conf; this.isOnline = false; // Set how many times to retry talking to another server over FConnection. FConnectionManager.setServerSideFConnectionRetries(this.conf, LOG); // Config'ed params this.msgInterval = conf.getInt("wasp.fserver.msginterval", 3 * 1000); this.sleeper = new Sleeper(this.msgInterval, this); this.numEntityGroupsToReport = conf.getInt("wasp.fserver.numentitygroupstoreport", 10); this.rpcTimeout = conf.getInt(FConstants.WASP_RPC_TIMEOUT_KEY, FConstants.DEFAULT_WASP_RPC_TIMEOUT); this.abortRequested = false; this.stopped = false; this.actionManager = new StorageActionManager(conf); // Server to handle client requests. String hostname = Strings .domainNamePointerToHostName(DNS.getDefaultHost(conf.get("wasp.fserver.dns.interface", "default"), conf.get("wasp.fserver.dns.nameserver", "default"))); int port = conf.getInt(FConstants.FSERVER_PORT, FConstants.DEFAULT_FSERVER_PORT); // Creation of a HSA will force a resolve. InetSocketAddress initialIsa = new InetSocketAddress(hostname, port); if (initialIsa.getAddress() == null) { throw new IllegalArgumentException("Failed resolve of " + initialIsa); } this.rpcServer = WaspRPC.getServer(FServer.class, this, new Class<?>[] { ClientProtocol.class, AdminProtocol.class, WaspRPCErrorHandler.class, OnlineEntityGroups.class }, initialIsa.getHostName(), // BindAddress is // IP we got for // this server. initialIsa.getPort(), conf); // Set our address. this.isa = this.rpcServer.getListenerAddress(); this.leases = new Leases(conf.getInt(FConstants.THREAD_WAKE_FREQUENCY, 10 * 1000)); this.startcode = System.currentTimeMillis(); int maxThreads = conf.getInt("wasp.transaction.threads.max", 150); this.pool = new ThreadPoolExecutor(1, maxThreads, 60, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(), new DaemonThreadFactory("thread factory")); ((ThreadPoolExecutor) this.pool).allowCoreThreadTimeOut(true); this.scannerLeaseTimeoutPeriod = conf.getInt(FConstants.WASP_CLIENT_SCANNER_TIMEOUT_PERIOD, FConstants.DEFAULT_WASP_CLIENT_SCANNER_TIMEOUT_PERIOD); this.driver = new BaseDriver(this); this.splitThread = new SplitThread(this); this.globalEntityGroup = new GlobalEntityGroup(this); }
From source file:org.apache.solr.cloud.OverseerCollectionProcessor.java
@Override public void run() { log.info("Process current queue of collection creations"); LeaderStatus isLeader = amILeader(); while (isLeader == LeaderStatus.DONT_KNOW) { log.debug("am_i_leader unclear {}", isLeader); isLeader = amILeader(); // not a no, not a yes, try ask again }/*from w ww . j a v a 2 s . co m*/ String oldestItemInWorkQueue = null; // hasLeftOverItems - used for avoiding re-execution of async tasks that were processed by a previous Overseer. // This variable is set in case there's any task found on the workQueue when the OCP starts up and // the id for the queue tail is used as a marker to check for the task in completed/failed map in zk. // Beyond the marker, all tasks can safely be assumed to have never been executed. boolean hasLeftOverItems = true; try { oldestItemInWorkQueue = workQueue.getTailId(); } catch (KeeperException e) { // We don't need to handle this. This is just a fail-safe which comes in handy in skipping already processed // async calls. SolrException.log(log, "", e); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } if (oldestItemInWorkQueue == null) hasLeftOverItems = false; else log.debug("Found already existing elements in the work-queue. Last element: {}", oldestItemInWorkQueue); try { prioritizeOverseerNodes(); } catch (Exception e) { log.error("Unable to prioritize overseer ", e); } // TODO: Make maxThreads configurable. this.tpe = new ExecutorUtil.MDCAwareThreadPoolExecutor(5, 100, 0L, TimeUnit.MILLISECONDS, new SynchronousQueue<Runnable>(), new DefaultSolrThreadFactory("OverseerThreadFactory")); try { while (!this.isClosed) { try { isLeader = amILeader(); if (LeaderStatus.NO == isLeader) { break; } else if (LeaderStatus.YES != isLeader) { log.debug("am_i_leader unclear {}", isLeader); continue; // not a no, not a yes, try asking again } log.debug("Cleaning up work-queue. #Running tasks: {}", runningTasks.size()); cleanUpWorkQueue(); printTrackingMaps(); boolean waited = false; while (runningTasks.size() > maxParallelThreads) { synchronized (waitLock) { waitLock.wait(100);//wait for 100 ms or till a task is complete } waited = true; } if (waited) cleanUpWorkQueue(); List<QueueEvent> heads = workQueue.peekTopN(maxParallelThreads, runningZKTasks, 2000L); if (heads == null) continue; log.debug("Got {} tasks from work-queue : [{}]", heads.size(), heads.toString()); if (isClosed) break; for (QueueEvent head : heads) { final ZkNodeProps message = ZkNodeProps.load(head.getBytes()); String collectionName = message.containsKey(COLLECTION_PROP) ? message.getStr(COLLECTION_PROP) : message.getStr(NAME); final String asyncId = message.getStr(ASYNC); if (hasLeftOverItems) { if (head.getId().equals(oldestItemInWorkQueue)) hasLeftOverItems = false; if (asyncId != null && (completedMap.contains(asyncId) || failureMap.contains(asyncId))) { log.debug("Found already processed task in workQueue, cleaning up. AsyncId [{}]", asyncId); workQueue.remove(head); continue; } } if (!checkExclusivity(message, head.getId())) { log.debug("Exclusivity check failed for [{}]", message.toString()); continue; } try { markTaskAsRunning(head, collectionName, asyncId, message); log.debug("Marked task [{}] as running", head.getId()); } catch (KeeperException.NodeExistsException e) { // This should never happen log.error("Tried to pick up task [{}] when it was already running!", head.getId()); } catch (InterruptedException e) { log.error("Thread interrupted while trying to pick task for execution.", head.getId()); Thread.currentThread().interrupt(); } log.info("Overseer Collection Processor: Get the message id:" + head.getId() + " message:" + message.toString()); String operation = message.getStr(Overseer.QUEUE_OPERATION); Runner runner = new Runner(message, operation, head); tpe.execute(runner); } } catch (KeeperException e) { if (e.code() == KeeperException.Code.SESSIONEXPIRED) { log.warn("Overseer cannot talk to ZK"); return; } SolrException.log(log, "", e); } catch (InterruptedException e) { Thread.currentThread().interrupt(); return; } catch (Exception e) { SolrException.log(log, "", e); } } } finally { this.close(); } }
From source file:org.springframework.xd.shell.command.JobCommandTests.java
@Test public void testLaunchJobTwiceWhereMakeUniqueIsFalse() throws Exception { logger.info("Launch batch job (makeUnique=false) twice"); String jobName = generateJobName(); // Batch 3.0 requires at least one parameter to reject duplicate executions of an instance String myJobParams = "{\"-param(long)\":\"12345\"}"; JobParametersHolder.reset();//w ww .ja v a 2s . c o m final JobParametersHolder jobParametersHolder = new JobParametersHolder(); executeJobCreate(jobName, JOB_WITH_PARAMETERS_DESCRIPTOR + " --makeUnique=false"); checkForJobInList(jobName, JOB_WITH_PARAMETERS_DESCRIPTOR + " --makeUnique=false", true); executeJobLaunch(jobName, myJobParams); assertTrue("The countdown latch expired and did not count down.", jobParametersHolder.isDone()); final SynchronousQueue<Message<?>> rendezvous = new SynchronousQueue<Message<?>>(); MessageHandler handler = new MessageHandler() { @Override public void handleMessage(Message<?> message) throws MessagingException { rendezvous.add(message); } }; getErrorChannel().subscribe(handler); executeCommand("job launch --name " + jobName + " --params " + myJobParams); Message<?> error = rendezvous.poll(5, TimeUnit.SECONDS); getErrorChannel().unsubscribe(handler); assertNotNull("expected an error message", error); Object payload = error.getPayload(); assertTrue("payload should be a MessagingException", payload instanceof MessagingException); assertEquals(JobInstanceAlreadyCompleteException.class, ((MessagingException) payload).getCause().getClass()); }
From source file:org.apache.hedwig.server.netty.PubSubServer.java
public void start() throws Exception { final SynchronousQueue<Either<Object, Exception>> queue = new SynchronousQueue<Either<Object, Exception>>(); new Thread(tg, new Runnable() { @Override/*from w w w. j ava2 s . c o m*/ public void run() { try { // Since zk is needed by almost everyone,try to see if we // need that first ThreadFactoryBuilder tfb = new ThreadFactoryBuilder(); scheduler = Executors.newSingleThreadScheduledExecutor( tfb.setNameFormat("PubSubServerScheduler-%d").build()); serverChannelFactory = new NioServerSocketChannelFactory( Executors.newCachedThreadPool(tfb.setNameFormat("PubSub-Server-NIOBoss-%d").build()), Executors.newCachedThreadPool(tfb.setNameFormat("PubSub-Server-NIOWorker-%d").build())); clientChannelFactory = new NioClientSocketChannelFactory( Executors.newCachedThreadPool(tfb.setNameFormat("PubSub-Client-NIOBoss-%d").build()), Executors.newCachedThreadPool(tfb.setNameFormat("PubSub-Client-NIOWorker-%d").build())); instantiateZookeeperClient(); instantiateMetadataManagerFactory(); tm = instantiateTopicManager(); pm = instantiatePersistenceManager(tm); dm = new FIFODeliveryManager(tm, pm, conf); dm.start(); sm = instantiateSubscriptionManager(tm, pm, dm); rm = instantiateRegionManager(pm, scheduler); sm.addListener(rm); allChannels = new DefaultChannelGroup("hedwig"); // Initialize the Netty Handlers (used by the // UmbrellaHandler) once so they can be shared by // both the SSL and non-SSL channels. SubscriptionChannelManager subChannelMgr = new SubscriptionChannelManager(); subChannelMgr.addSubChannelDisconnectedListener((SubChannelDisconnectedListener) dm); Map<OperationType, Handler> handlers = initializeNettyHandlers(tm, dm, pm, sm, subChannelMgr); // Initialize Netty for the regular non-SSL channels initializeNetty(null, handlers, subChannelMgr); if (conf.isSSLEnabled()) { initializeNetty(new SslServerContextFactory(conf), handlers, subChannelMgr); } // register jmx registerJMX(subChannelMgr); } catch (Exception e) { ConcurrencyUtils.put(queue, Either.right(e)); return; } ConcurrencyUtils.put(queue, Either.of(new Object(), (Exception) null)); } }).start(); Either<Object, Exception> either = ConcurrencyUtils.take(queue); if (either.left() == null) { throw either.right(); } }
From source file:org.schedulesdirect.grabber.Grabber.java
private ThreadPoolExecutor createThreadPoolExecutor() { return new ThreadPoolExecutor(0, globalOpts.getMaxThreads(), 10, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(), new ThreadPoolExecutor.CallerRunsPolicy()) { @Override//from ww w . j av a2s .c o m protected void afterExecute(Runnable r, Throwable t) { //super.afterExecute(r, t); if (t != null) { Logger log = Logger.getLogger(r.getClass()); log.error("Task failed!", t); if (!(r instanceof LogoTask)) failedTask = true; } } }; }
From source file:org.apache.drill.exec.client.DrillClient.java
public synchronized void connect(String connect, Properties props) throws RpcException { if (connected) { return;/* ww w . j a v a 2s . com*/ } final List<DrillbitEndpoint> endpoints = new ArrayList<>(); if (isDirectConnection) { // Populate the endpoints list with all the drillbit information provided in the connection string endpoints.addAll(parseAndVerifyEndpoints(props.getProperty("drillbit"), config.getString(ExecConstants.INITIAL_USER_PORT))); } else { if (ownsZkConnection) { try { this.clusterCoordinator = new ZKClusterCoordinator(this.config, connect); this.clusterCoordinator.start(10000); } catch (Exception e) { throw new RpcException("Failure setting up ZK for client.", e); } } endpoints.addAll(clusterCoordinator.getAvailableEndpoints()); // Make sure we have at least one endpoint in the list checkState(!endpoints.isEmpty(), "No active Drillbit endpoint found from ZooKeeper"); } // shuffle the collection then get the first endpoint Collections.shuffle(endpoints); final DrillbitEndpoint endpoint = endpoints.get(0); if (props != null) { final UserProperties.Builder upBuilder = UserProperties.newBuilder(); for (final String key : props.stringPropertyNames()) { upBuilder.addProperties(Property.newBuilder().setKey(key).setValue(props.getProperty(key))); } this.props = upBuilder.build(); } eventLoopGroup = createEventLoop(config.getInt(ExecConstants.CLIENT_RPC_THREADS), "Client-"); executor = new ThreadPoolExecutor(0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(), new NamedThreadFactory("drill-client-executor-")) { @Override protected void afterExecute(final Runnable r, final Throwable t) { if (t != null) { logger.error("{}.run() leaked an exception.", r.getClass().getName(), t); } super.afterExecute(r, t); } }; client = new UserClient(clientName, config, supportComplexTypes, allocator, eventLoopGroup, executor); logger.debug("Connecting to server {}:{}", endpoint.getAddress(), endpoint.getUserPort()); connect(endpoint); connected = true; }