Example usage for java.util.concurrent ExecutorCompletionService ExecutorCompletionService

List of usage examples for java.util.concurrent ExecutorCompletionService ExecutorCompletionService

Introduction

In this page you can find the example usage for java.util.concurrent ExecutorCompletionService ExecutorCompletionService.

Prototype

public ExecutorCompletionService(Executor executor) 

Source Link

Document

Creates an ExecutorCompletionService using the supplied executor for base task execution and a LinkedBlockingQueue as a completion queue.

Usage

From source file:com.alibaba.otter.shared.communication.core.impl.DefaultCommunicationClientImpl.java

public Object call(final String[] addrs, final Event event) {
    Assert.notNull(this.factory, "No factory specified");
    if (addrs == null || addrs.length == 0) {
        throw new IllegalArgumentException("addrs example: 127.0.0.1:1099");
    }//from  w ww  . j  a va 2 s. c  o m

    ExecutorCompletionService completionService = new ExecutorCompletionService(executor);
    List<Future<Object>> futures = new ArrayList<Future<Object>>(addrs.length);
    List result = new ArrayList(10);
    for (final String addr : addrs) {
        futures.add(completionService.submit((new Callable<Object>() {

            @Override
            public Object call() throws Exception {
                return DefaultCommunicationClientImpl.this.call(addr, event);
            }
        })));
    }

    Exception ex = null;
    int errorIndex = 0;
    while (errorIndex < futures.size()) {
        try {
            Future future = completionService.take();// ?
            future.get();
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            ex = e;
            break;
        } catch (ExecutionException e) {
            ex = e;
            break;
        }

        errorIndex++;
    }

    if (errorIndex < futures.size()) {
        for (int index = 0; index < futures.size(); index++) {
            Future<Object> future = futures.get(index);
            if (future.isDone() == false) {
                future.cancel(true);
            }
        }
    } else {
        for (int index = 0; index < futures.size(); index++) {
            Future<Object> future = futures.get(index);
            try {
                result.add(future.get());
            } catch (InterruptedException e) {
                // ignore
                Thread.currentThread().interrupt();
            } catch (ExecutionException e) {
                // ignore
            }
        }
    }

    if (ex != null) {
        throw new CommunicationException(
                String.format("call addr[%s] error by %s", addrs[errorIndex], ex.getMessage()), ex);
    } else {
        return result;
    }
}

From source file:org.springframework.integration.groovy.GroovyExpressionTests.java

@Test
public void testScriptFactoryCustomizerStatic() throws Exception {
    final Customizer customizer = new Customizer(Collections.singletonMap("name", (Object) "foo"));
    final GroovyScriptFactory factory = new GroovyScriptFactory("Groovy Script", customizer);
    final ResourceScriptSource scriptSource = new ResourceScriptSource(
            new NamedByteArrayResource("\"name=${name}\"".getBytes(), "InlineScript"));
    Object scriptedObject = factory.getScriptedObject(scriptSource, null);
    assertEquals("name=foo", scriptedObject.toString());
    CompletionService<String> completionService = new ExecutorCompletionService<String>(
            Executors.newFixedThreadPool(10));
    for (int i = 0; i < 100; i++) {
        final String name = "bar" + i;
        completionService.submit(new Callable<String>() {
            public String call() throws Exception {
                Object scriptedObject = factory.getScriptedObject(scriptSource, null);
                String result = scriptedObject.toString();
                logger.debug("Result=" + result + " with name=" + name);
                if (!("name=foo").equals(result)) {
                    throw new IllegalStateException("Wrong value (" + result + ") for: " + name);
                }/*from w  w w . j a  v  a2  s  .  c  om*/
                return name;
            }
        });
    }
    Set<String> set = new HashSet<String>();
    for (int i = 0; i < 100; i++) {
        set.add(completionService.take().get());
    }
    assertEquals(100, set.size());
}

From source file:org.apache.hadoop.hbase.util.ModifyRegionUtils.java

/**
 * Create new set of regions on the specified file-system.
 * NOTE: that you should add the regions to hbase:meta after this operation.
 *
 * @param exec Thread Pool Executor//ww  w  . j a va2s  .  c  o  m
 * @param conf {@link Configuration}
 * @param rootDir Root directory for HBase instance
 * @param tableDir table directory
 * @param hTableDescriptor description of the table
 * @param newRegions {@link HRegionInfo} that describes the regions to create
 * @param task {@link RegionFillTask} custom code to populate region after creation
 * @throws IOException
 */
public static List<HRegionInfo> createRegions(final ThreadPoolExecutor exec, final Configuration conf,
        final Path rootDir, final Path tableDir, final HTableDescriptor hTableDescriptor,
        final HRegionInfo[] newRegions, final RegionFillTask task) throws IOException {
    if (newRegions == null)
        return null;
    int regionNumber = newRegions.length;
    CompletionService<HRegionInfo> completionService = new ExecutorCompletionService<HRegionInfo>(exec);
    List<HRegionInfo> regionInfos = new ArrayList<HRegionInfo>();
    for (final HRegionInfo newRegion : newRegions) {
        completionService.submit(new Callable<HRegionInfo>() {
            @Override
            public HRegionInfo call() throws IOException {
                return createRegion(conf, rootDir, tableDir, hTableDescriptor, newRegion, task);
            }
        });
    }
    try {
        // wait for all regions to finish creation
        for (int i = 0; i < regionNumber; i++) {
            Future<HRegionInfo> future = completionService.take();
            HRegionInfo regionInfo = future.get();
            regionInfos.add(regionInfo);
        }
    } catch (InterruptedException e) {
        LOG.error("Caught " + e + " during region creation");
        throw new InterruptedIOException(e.getMessage());
    } catch (ExecutionException e) {
        throw new IOException(e);
    }
    return regionInfos;
}

From source file:com.alibaba.otter.node.etl.extract.extractor.DatabaseExtractor.java

@Override
public void extract(DbBatch dbBatch) throws ExtractException {
    Assert.notNull(dbBatch);//from  w  w w .  j  ava 2  s .c o m
    Assert.notNull(dbBatch.getRowBatch());
    // ??
    Pipeline pipeline = getPipeline(dbBatch.getRowBatch().getIdentity().getPipelineId());
    boolean mustDb = pipeline.getParameters().getSyncConsistency().isMedia();
    boolean isRow = pipeline.getParameters().getSyncMode().isRow();// ???
    // ??
    adjustPoolSize(pipeline.getParameters().getExtractPoolSize()); // Extractor?
    ExecutorCompletionService completionService = new ExecutorCompletionService(executor);

    // ???
    ExtractException exception = null;
    // ??
    List<DataItem> items = new ArrayList<DataItem>();
    List<Future> futures = new ArrayList<Future>();
    List<EventData> eventDatas = dbBatch.getRowBatch().getDatas();
    for (EventData eventData : eventDatas) {
        if (eventData.getEventType().isDdl()) {
            continue;
        }

        DataItem item = new DataItem(eventData);
        // row??????row???
        boolean flag = mustDb
                || (eventData.getSyncConsistency() != null && eventData.getSyncConsistency().isMedia());

        // ?case, oracle erosa??????
        if (!flag && CollectionUtils.isEmpty(eventData.getUpdatedColumns())) {
            DataMedia dataMedia = ConfigHelper.findDataMedia(pipeline, eventData.getTableId());
            if (dataMedia.getSource().getType().isOracle()) {
                flag |= true;
                eventData.setRemedy(true);// ???erosa?????
            }
        }

        if (isRow && !flag) {
            // ?????
            // view??
            flag = checkNeedDbForRowMode(pipeline, eventData);
        }

        if (flag && (eventData.getEventType().isInsert() || eventData.getEventType().isUpdate())) {// ????
            Future future = completionService.submit(new DatabaseExtractWorker(pipeline, item), null); // ??
            if (future.isDone()) {
                // ?CallerRun????
                try {
                    future.get();
                } catch (InterruptedException e) {
                    cancel(futures);// ??
                    throw new ExtractException(e);
                } catch (ExecutionException e) {
                    cancel(futures); // ??
                    throw new ExtractException(e);
                }
            }

            futures.add(future);// 
        }

        items.add(item);// ?
    }

    // ?
    int index = 0;
    while (index < futures.size()) { // ??
        try {
            Future future = completionService.take();// ?
            future.get();
        } catch (InterruptedException e) {
            exception = new ExtractException(e);
            break;// future
        } catch (ExecutionException e) {
            exception = new ExtractException(e);
            break;// future
        }

        index++;
    }

    if (index < futures.size()) {
        // ???cancel?????
        cancel(futures);
        throw exception;
    } else {
        // ?, ????
        for (int i = 0; i < items.size(); i++) {
            DataItem item = items.get(i);
            if (item.filter) { // ???????
                eventDatas.remove(item.getEventData());
            }
        }
    }

}

From source file:org.apache.syncope.core.provisioning.java.propagation.PriorityPropagationTaskExecutor.java

@Override
protected void doExecute(final Collection<PropagationTask> tasks, final PropagationReporter reporter,
        final boolean nullPriorityAsync) {

    List<PropagationTask> prioritizedTasks = CollectionUtils.select(tasks, new Predicate<PropagationTask>() {

        @Override//  w  w  w. j  av  a  2s .  com
        public boolean evaluate(final PropagationTask task) {
            return task.getResource().getPropagationPriority() != null;
        }
    }, new ArrayList<PropagationTask>());
    Collections.sort(prioritizedTasks, new PriorityComparator());
    LOG.debug("Propagation tasks sorted by priority, for serial execution: {}", prioritizedTasks);

    Collection<PropagationTask> concurrentTasks = CollectionUtils.subtract(tasks, prioritizedTasks);
    LOG.debug("Propagation tasks for concurrent execution: {}", concurrentTasks);

    // first process priority resources sequentially and fail as soon as any propagation failure is reported
    for (PropagationTask task : prioritizedTasks) {
        TaskExec execution = null;
        PropagationTaskExecStatus execStatus;
        try {
            execution = newPropagationTaskCallable(task, reporter).call();
            execStatus = PropagationTaskExecStatus.valueOf(execution.getStatus());
        } catch (Exception e) {
            LOG.error("Unexpected exception", e);
            execStatus = PropagationTaskExecStatus.FAILURE;
        }
        if (execStatus != PropagationTaskExecStatus.SUCCESS) {
            throw new PropagationException(task.getResource().getKey(),
                    execution == null ? null : execution.getMessage());
        }
    }

    // then process non-priority resources concurrently...
    final CompletionService<TaskExec> completionService = new ExecutorCompletionService<>(executor);
    Map<PropagationTask, Future<TaskExec>> nullPriority = new HashMap<>(concurrentTasks.size());
    for (PropagationTask task : concurrentTasks) {
        try {
            nullPriority.put(task, completionService.submit(newPropagationTaskCallable(task, reporter)));
        } catch (Exception e) {
            LOG.error("Unexpected exception", e);
        }
    }
    // ...waiting for all callables to complete, if async processing was not required
    if (!nullPriority.isEmpty()) {
        if (nullPriorityAsync) {
            for (Map.Entry<PropagationTask, Future<TaskExec>> entry : nullPriority.entrySet()) {
                reporter.onSuccessOrNonPriorityResourceFailures(entry.getKey(),
                        PropagationTaskExecStatus.CREATED, null, null, null);
            }
        } else {
            final Set<Future<TaskExec>> nullPriorityFutures = new HashSet<>(nullPriority.values());
            try {
                executor.submit(new Runnable() {

                    @Override
                    public void run() {
                        while (!nullPriorityFutures.isEmpty()) {
                            try {
                                nullPriorityFutures.remove(completionService.take());
                            } catch (Exception e) {
                                LOG.error("Unexpected exception", e);
                            }
                        }
                    }
                }).get(60, TimeUnit.SECONDS);
            } catch (Exception e) {
                LOG.error("Unexpected exception", e);
            } finally {
                for (Future<TaskExec> future : nullPriorityFutures) {
                    future.cancel(true);
                }
                nullPriorityFutures.clear();
                nullPriority.clear();
            }
        }
    }
}

From source file:com.amazon.s3.S3ParserTest.java

@Test(enabled = false)
void testAmazonParseListBucketResultParallelResponseTime() throws InterruptedException, ExecutionException {
    CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>(exec);

    for (int i = 0; i < LOOP_COUNT; i++)
        completer.submit(new Callable<Boolean>() {
            public Boolean call() throws IOException {
                runAmazonParseListBucketResult();
                return true;
            }/*w  ww  .  ja  v  a  2 s . c  o  m*/
        });
    for (int i = 0; i < LOOP_COUNT; i++)
        assert completer.take().get();
}

From source file:pl.edu.icm.cermine.libsvm.SVMParameterFinder.java

public void run(String inputFile, String ext, int threads, int kernel, int degree)
        throws AnalysisException, IOException, TransformationException, CloneNotSupportedException,
        InterruptedException, ExecutionException {
    List<TrainingSample<BxZoneLabel>> samples = getSamples(inputFile, ext);

    ExecutorService executor = Executors.newFixedThreadPool(3);
    CompletionService<EvaluationParams> completionService = new ExecutorCompletionService<EvaluationParams>(
            executor);//from  w w  w .  j  ava  2s .  com

    double bestRate = 0;
    int bestclog = 0;
    int bestglog = 0;

    int submitted = 0;

    for (int clog = -5; clog <= 15; clog++) {
        for (int glog = 3; glog >= -15; glog--) {
            completionService.submit(new Evaluator(samples, new EvaluationParams(clog, glog), kernel, degree));
            submitted++;
        }
    }

    while (submitted > 0) {
        Future<EvaluationParams> f1 = completionService.take();
        EvaluationParams p = f1.get();
        if (p.rate > bestRate) {
            bestRate = p.rate;
            bestclog = p.clog;
            bestglog = p.glog;
        }
        System.out.println("Gamma: " + p.glog + ", C: " + p.clog + ", rate: " + p.rate + " (Best: " + bestglog
                + " " + bestclog + " " + bestRate + ")");
        submitted--;
    }

    executor.shutdown();
}

From source file:org.apache.hama.bsp.LocalBSPRunner.java

@Override
public JobStatus submitJob(BSPJobID jobID, String jobFile) throws IOException {
    this.jobFile = jobFile;

    if (fs == null) {
        this.fs = FileSystem.get(conf);
    }/*from  www. j av  a  2 s .com*/

    // add the resource to the current configuration, because add resouce in
    // HamaConfigurations constructor (ID,FILE) does not take local->HDFS
    // connections into account. This leads to not serializing the
    // configuration, which yields into failure.
    conf.addResource(fs.open(new Path(jobFile)));

    conf.setClass(MessageManagerFactory.MESSAGE_MANAGER_CLASS, LocalMessageManager.class, MessageManager.class);
    conf.setClass(SyncServiceFactory.SYNC_PEER_CLASS, LocalSyncClient.class, SyncClient.class);

    BSPJob job = new BSPJob(new HamaConfiguration(conf), jobID);
    currentJobStatus = new JobStatus(jobID, System.getProperty("user.name"), 0L, JobStatus.RUNNING,
            globalCounters);

    int numBspTask = job.getNumBspTask();

    String jobSplit = conf.get("bsp.job.split.file");

    BSPJobClient.RawSplit[] splits = null;
    if (jobSplit != null) {

        DataInputStream splitFile = fs.open(new Path(jobSplit));

        try {
            splits = BSPJobClient.readSplitFile(splitFile);
        } finally {
            splitFile.close();
        }
    }

    threadPool = (ThreadPoolExecutor) Executors.newFixedThreadPool(numBspTask);
    @SuppressWarnings("rawtypes")
    ExecutorCompletionService<BSPPeerImpl> completionService = new ExecutorCompletionService<BSPPeerImpl>(
            threadPool);

    peerNames = new String[numBspTask];
    for (int i = 0; i < numBspTask; i++) {
        peerNames[i] = "local:" + i;
        completionService.submit(new BSPRunner(new HamaConfiguration(conf), job, i, splits));
        globalCounters.incrCounter(JobInProgress.JobCounter.LAUNCHED_TASKS, 1L);
    }

    new Thread(new ThreadObserver(numBspTask, completionService)).start();
    return currentJobStatus;
}

From source file:org.apache.solr.cloud.BasicDistributedZkTest.java

public BasicDistributedZkTest() {
    sliceCount = 2;
    completionService = new ExecutorCompletionService<>(executor);
    pending = new HashSet<>();

}

From source file:com.alibaba.otter.manager.biz.monitor.impl.GlobalMonitor.java

private void concurrentProcess(List<Long> channelIds) {
    ExecutorCompletionService completionExecutor = new ExecutorCompletionService(executor);
    List<Future> futures = new ArrayList<Future>();
    for (final Long channelId : channelIds) {
        futures.add(completionExecutor.submit(new Callable<Object>() {

            @Override/* ww  w  .  j  a v a  2s.  co m*/
            public Object call() throws Exception {
                ChannelStatus status = arbitrateManageService.channelEvent().status(channelId);
                if (status.isPause()) {
                    restartAlarmRecovery.recovery(channelId);
                }
                return null;
            }
        }));
    }

    List<Throwable> exceptions = new ArrayList<Throwable>();
    int index = 0;
    int size = futures.size();
    while (index < size) {
        try {
            Future<?> future = completionExecutor.take();
            future.get();
        } catch (InterruptedException e) {
            exceptions.add(e);
        } catch (ExecutionException e) {
            exceptions.add(e);
        }
        index++;
    }

    if (!exceptions.isEmpty()) {
        StringBuilder sb = new StringBuilder(exceptions.size() + " exception happens in global monitor\n");
        sb.append("exception stack start :\n");
        for (Throwable t : exceptions) {
            sb.append(ExceptionUtils.getStackTrace(t));
        }
        sb.append("exception stack end \n");
        throw new IllegalStateException(sb.toString());
    }
}