List of usage examples for java.util.concurrent ExecutorCompletionService ExecutorCompletionService
public ExecutorCompletionService(Executor executor)
From source file:com.alibaba.otter.shared.communication.core.impl.DefaultCommunicationClientImpl.java
public Object call(final String[] addrs, final Event event) { Assert.notNull(this.factory, "No factory specified"); if (addrs == null || addrs.length == 0) { throw new IllegalArgumentException("addrs example: 127.0.0.1:1099"); }//from w ww . j a va 2 s. c o m ExecutorCompletionService completionService = new ExecutorCompletionService(executor); List<Future<Object>> futures = new ArrayList<Future<Object>>(addrs.length); List result = new ArrayList(10); for (final String addr : addrs) { futures.add(completionService.submit((new Callable<Object>() { @Override public Object call() throws Exception { return DefaultCommunicationClientImpl.this.call(addr, event); } }))); } Exception ex = null; int errorIndex = 0; while (errorIndex < futures.size()) { try { Future future = completionService.take();// ? future.get(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); ex = e; break; } catch (ExecutionException e) { ex = e; break; } errorIndex++; } if (errorIndex < futures.size()) { for (int index = 0; index < futures.size(); index++) { Future<Object> future = futures.get(index); if (future.isDone() == false) { future.cancel(true); } } } else { for (int index = 0; index < futures.size(); index++) { Future<Object> future = futures.get(index); try { result.add(future.get()); } catch (InterruptedException e) { // ignore Thread.currentThread().interrupt(); } catch (ExecutionException e) { // ignore } } } if (ex != null) { throw new CommunicationException( String.format("call addr[%s] error by %s", addrs[errorIndex], ex.getMessage()), ex); } else { return result; } }
From source file:org.springframework.integration.groovy.GroovyExpressionTests.java
@Test public void testScriptFactoryCustomizerStatic() throws Exception { final Customizer customizer = new Customizer(Collections.singletonMap("name", (Object) "foo")); final GroovyScriptFactory factory = new GroovyScriptFactory("Groovy Script", customizer); final ResourceScriptSource scriptSource = new ResourceScriptSource( new NamedByteArrayResource("\"name=${name}\"".getBytes(), "InlineScript")); Object scriptedObject = factory.getScriptedObject(scriptSource, null); assertEquals("name=foo", scriptedObject.toString()); CompletionService<String> completionService = new ExecutorCompletionService<String>( Executors.newFixedThreadPool(10)); for (int i = 0; i < 100; i++) { final String name = "bar" + i; completionService.submit(new Callable<String>() { public String call() throws Exception { Object scriptedObject = factory.getScriptedObject(scriptSource, null); String result = scriptedObject.toString(); logger.debug("Result=" + result + " with name=" + name); if (!("name=foo").equals(result)) { throw new IllegalStateException("Wrong value (" + result + ") for: " + name); }/*from w w w . j a v a2 s . c om*/ return name; } }); } Set<String> set = new HashSet<String>(); for (int i = 0; i < 100; i++) { set.add(completionService.take().get()); } assertEquals(100, set.size()); }
From source file:org.apache.hadoop.hbase.util.ModifyRegionUtils.java
/** * Create new set of regions on the specified file-system. * NOTE: that you should add the regions to hbase:meta after this operation. * * @param exec Thread Pool Executor//ww w . j a va2s . c o m * @param conf {@link Configuration} * @param rootDir Root directory for HBase instance * @param tableDir table directory * @param hTableDescriptor description of the table * @param newRegions {@link HRegionInfo} that describes the regions to create * @param task {@link RegionFillTask} custom code to populate region after creation * @throws IOException */ public static List<HRegionInfo> createRegions(final ThreadPoolExecutor exec, final Configuration conf, final Path rootDir, final Path tableDir, final HTableDescriptor hTableDescriptor, final HRegionInfo[] newRegions, final RegionFillTask task) throws IOException { if (newRegions == null) return null; int regionNumber = newRegions.length; CompletionService<HRegionInfo> completionService = new ExecutorCompletionService<HRegionInfo>(exec); List<HRegionInfo> regionInfos = new ArrayList<HRegionInfo>(); for (final HRegionInfo newRegion : newRegions) { completionService.submit(new Callable<HRegionInfo>() { @Override public HRegionInfo call() throws IOException { return createRegion(conf, rootDir, tableDir, hTableDescriptor, newRegion, task); } }); } try { // wait for all regions to finish creation for (int i = 0; i < regionNumber; i++) { Future<HRegionInfo> future = completionService.take(); HRegionInfo regionInfo = future.get(); regionInfos.add(regionInfo); } } catch (InterruptedException e) { LOG.error("Caught " + e + " during region creation"); throw new InterruptedIOException(e.getMessage()); } catch (ExecutionException e) { throw new IOException(e); } return regionInfos; }
From source file:com.alibaba.otter.node.etl.extract.extractor.DatabaseExtractor.java
@Override public void extract(DbBatch dbBatch) throws ExtractException { Assert.notNull(dbBatch);//from w w w . j ava 2 s .c o m Assert.notNull(dbBatch.getRowBatch()); // ?? Pipeline pipeline = getPipeline(dbBatch.getRowBatch().getIdentity().getPipelineId()); boolean mustDb = pipeline.getParameters().getSyncConsistency().isMedia(); boolean isRow = pipeline.getParameters().getSyncMode().isRow();// ??? // ?? adjustPoolSize(pipeline.getParameters().getExtractPoolSize()); // Extractor? ExecutorCompletionService completionService = new ExecutorCompletionService(executor); // ??? ExtractException exception = null; // ?? List<DataItem> items = new ArrayList<DataItem>(); List<Future> futures = new ArrayList<Future>(); List<EventData> eventDatas = dbBatch.getRowBatch().getDatas(); for (EventData eventData : eventDatas) { if (eventData.getEventType().isDdl()) { continue; } DataItem item = new DataItem(eventData); // row??????row??? boolean flag = mustDb || (eventData.getSyncConsistency() != null && eventData.getSyncConsistency().isMedia()); // ?case, oracle erosa?????? if (!flag && CollectionUtils.isEmpty(eventData.getUpdatedColumns())) { DataMedia dataMedia = ConfigHelper.findDataMedia(pipeline, eventData.getTableId()); if (dataMedia.getSource().getType().isOracle()) { flag |= true; eventData.setRemedy(true);// ???erosa????? } } if (isRow && !flag) { // ????? // view?? flag = checkNeedDbForRowMode(pipeline, eventData); } if (flag && (eventData.getEventType().isInsert() || eventData.getEventType().isUpdate())) {// ???? Future future = completionService.submit(new DatabaseExtractWorker(pipeline, item), null); // ?? if (future.isDone()) { // ?CallerRun???? try { future.get(); } catch (InterruptedException e) { cancel(futures);// ?? throw new ExtractException(e); } catch (ExecutionException e) { cancel(futures); // ?? throw new ExtractException(e); } } futures.add(future);// } items.add(item);// ? } // ? int index = 0; while (index < futures.size()) { // ?? try { Future future = completionService.take();// ? future.get(); } catch (InterruptedException e) { exception = new ExtractException(e); break;// future } catch (ExecutionException e) { exception = new ExtractException(e); break;// future } index++; } if (index < futures.size()) { // ???cancel????? cancel(futures); throw exception; } else { // ?, ???? for (int i = 0; i < items.size(); i++) { DataItem item = items.get(i); if (item.filter) { // ??????? eventDatas.remove(item.getEventData()); } } } }
From source file:org.apache.syncope.core.provisioning.java.propagation.PriorityPropagationTaskExecutor.java
@Override protected void doExecute(final Collection<PropagationTask> tasks, final PropagationReporter reporter, final boolean nullPriorityAsync) { List<PropagationTask> prioritizedTasks = CollectionUtils.select(tasks, new Predicate<PropagationTask>() { @Override// w w w. j av a 2s . com public boolean evaluate(final PropagationTask task) { return task.getResource().getPropagationPriority() != null; } }, new ArrayList<PropagationTask>()); Collections.sort(prioritizedTasks, new PriorityComparator()); LOG.debug("Propagation tasks sorted by priority, for serial execution: {}", prioritizedTasks); Collection<PropagationTask> concurrentTasks = CollectionUtils.subtract(tasks, prioritizedTasks); LOG.debug("Propagation tasks for concurrent execution: {}", concurrentTasks); // first process priority resources sequentially and fail as soon as any propagation failure is reported for (PropagationTask task : prioritizedTasks) { TaskExec execution = null; PropagationTaskExecStatus execStatus; try { execution = newPropagationTaskCallable(task, reporter).call(); execStatus = PropagationTaskExecStatus.valueOf(execution.getStatus()); } catch (Exception e) { LOG.error("Unexpected exception", e); execStatus = PropagationTaskExecStatus.FAILURE; } if (execStatus != PropagationTaskExecStatus.SUCCESS) { throw new PropagationException(task.getResource().getKey(), execution == null ? null : execution.getMessage()); } } // then process non-priority resources concurrently... final CompletionService<TaskExec> completionService = new ExecutorCompletionService<>(executor); Map<PropagationTask, Future<TaskExec>> nullPriority = new HashMap<>(concurrentTasks.size()); for (PropagationTask task : concurrentTasks) { try { nullPriority.put(task, completionService.submit(newPropagationTaskCallable(task, reporter))); } catch (Exception e) { LOG.error("Unexpected exception", e); } } // ...waiting for all callables to complete, if async processing was not required if (!nullPriority.isEmpty()) { if (nullPriorityAsync) { for (Map.Entry<PropagationTask, Future<TaskExec>> entry : nullPriority.entrySet()) { reporter.onSuccessOrNonPriorityResourceFailures(entry.getKey(), PropagationTaskExecStatus.CREATED, null, null, null); } } else { final Set<Future<TaskExec>> nullPriorityFutures = new HashSet<>(nullPriority.values()); try { executor.submit(new Runnable() { @Override public void run() { while (!nullPriorityFutures.isEmpty()) { try { nullPriorityFutures.remove(completionService.take()); } catch (Exception e) { LOG.error("Unexpected exception", e); } } } }).get(60, TimeUnit.SECONDS); } catch (Exception e) { LOG.error("Unexpected exception", e); } finally { for (Future<TaskExec> future : nullPriorityFutures) { future.cancel(true); } nullPriorityFutures.clear(); nullPriority.clear(); } } } }
From source file:com.amazon.s3.S3ParserTest.java
@Test(enabled = false) void testAmazonParseListBucketResultParallelResponseTime() throws InterruptedException, ExecutionException { CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>(exec); for (int i = 0; i < LOOP_COUNT; i++) completer.submit(new Callable<Boolean>() { public Boolean call() throws IOException { runAmazonParseListBucketResult(); return true; }/*w ww . ja v a 2 s . c o m*/ }); for (int i = 0; i < LOOP_COUNT; i++) assert completer.take().get(); }
From source file:pl.edu.icm.cermine.libsvm.SVMParameterFinder.java
public void run(String inputFile, String ext, int threads, int kernel, int degree) throws AnalysisException, IOException, TransformationException, CloneNotSupportedException, InterruptedException, ExecutionException { List<TrainingSample<BxZoneLabel>> samples = getSamples(inputFile, ext); ExecutorService executor = Executors.newFixedThreadPool(3); CompletionService<EvaluationParams> completionService = new ExecutorCompletionService<EvaluationParams>( executor);//from w w w . j ava 2s . com double bestRate = 0; int bestclog = 0; int bestglog = 0; int submitted = 0; for (int clog = -5; clog <= 15; clog++) { for (int glog = 3; glog >= -15; glog--) { completionService.submit(new Evaluator(samples, new EvaluationParams(clog, glog), kernel, degree)); submitted++; } } while (submitted > 0) { Future<EvaluationParams> f1 = completionService.take(); EvaluationParams p = f1.get(); if (p.rate > bestRate) { bestRate = p.rate; bestclog = p.clog; bestglog = p.glog; } System.out.println("Gamma: " + p.glog + ", C: " + p.clog + ", rate: " + p.rate + " (Best: " + bestglog + " " + bestclog + " " + bestRate + ")"); submitted--; } executor.shutdown(); }
From source file:org.apache.hama.bsp.LocalBSPRunner.java
@Override public JobStatus submitJob(BSPJobID jobID, String jobFile) throws IOException { this.jobFile = jobFile; if (fs == null) { this.fs = FileSystem.get(conf); }/*from www. j av a 2 s .com*/ // add the resource to the current configuration, because add resouce in // HamaConfigurations constructor (ID,FILE) does not take local->HDFS // connections into account. This leads to not serializing the // configuration, which yields into failure. conf.addResource(fs.open(new Path(jobFile))); conf.setClass(MessageManagerFactory.MESSAGE_MANAGER_CLASS, LocalMessageManager.class, MessageManager.class); conf.setClass(SyncServiceFactory.SYNC_PEER_CLASS, LocalSyncClient.class, SyncClient.class); BSPJob job = new BSPJob(new HamaConfiguration(conf), jobID); currentJobStatus = new JobStatus(jobID, System.getProperty("user.name"), 0L, JobStatus.RUNNING, globalCounters); int numBspTask = job.getNumBspTask(); String jobSplit = conf.get("bsp.job.split.file"); BSPJobClient.RawSplit[] splits = null; if (jobSplit != null) { DataInputStream splitFile = fs.open(new Path(jobSplit)); try { splits = BSPJobClient.readSplitFile(splitFile); } finally { splitFile.close(); } } threadPool = (ThreadPoolExecutor) Executors.newFixedThreadPool(numBspTask); @SuppressWarnings("rawtypes") ExecutorCompletionService<BSPPeerImpl> completionService = new ExecutorCompletionService<BSPPeerImpl>( threadPool); peerNames = new String[numBspTask]; for (int i = 0; i < numBspTask; i++) { peerNames[i] = "local:" + i; completionService.submit(new BSPRunner(new HamaConfiguration(conf), job, i, splits)); globalCounters.incrCounter(JobInProgress.JobCounter.LAUNCHED_TASKS, 1L); } new Thread(new ThreadObserver(numBspTask, completionService)).start(); return currentJobStatus; }
From source file:org.apache.solr.cloud.BasicDistributedZkTest.java
public BasicDistributedZkTest() { sliceCount = 2; completionService = new ExecutorCompletionService<>(executor); pending = new HashSet<>(); }
From source file:com.alibaba.otter.manager.biz.monitor.impl.GlobalMonitor.java
private void concurrentProcess(List<Long> channelIds) { ExecutorCompletionService completionExecutor = new ExecutorCompletionService(executor); List<Future> futures = new ArrayList<Future>(); for (final Long channelId : channelIds) { futures.add(completionExecutor.submit(new Callable<Object>() { @Override/* ww w . j a v a 2s. co m*/ public Object call() throws Exception { ChannelStatus status = arbitrateManageService.channelEvent().status(channelId); if (status.isPause()) { restartAlarmRecovery.recovery(channelId); } return null; } })); } List<Throwable> exceptions = new ArrayList<Throwable>(); int index = 0; int size = futures.size(); while (index < size) { try { Future<?> future = completionExecutor.take(); future.get(); } catch (InterruptedException e) { exceptions.add(e); } catch (ExecutionException e) { exceptions.add(e); } index++; } if (!exceptions.isEmpty()) { StringBuilder sb = new StringBuilder(exceptions.size() + " exception happens in global monitor\n"); sb.append("exception stack start :\n"); for (Throwable t : exceptions) { sb.append(ExceptionUtils.getStackTrace(t)); } sb.append("exception stack end \n"); throw new IllegalStateException(sb.toString()); } }