Example usage for java.util.concurrent ExecutorCompletionService ExecutorCompletionService

Introduction

In this page you can find the example usage for java.util.concurrent ExecutorCompletionService ExecutorCompletionService.

Prototype

public ExecutorCompletionService(Executor executor) 

Source Link

Document

Creates an ExecutorCompletionService using the supplied executor for base task execution and a LinkedBlockingQueue as a completion queue.

Usage

From source file:org.apache.hadoop.hbase.util.TestDrainableQueue.java

@Test(timeout = 30 * 1000)
public void testDrainableQueue() throws Exception {
    for (int attempt = 0; attempt < NUM_ATTEMPTS; ++attempt) {
        final int totalEvents = NUM_PRODUCERS * NUM_EVENTS_PER_BATCH;
        final int drainAfterNEvents = totalEvents / 2;
        shouldDrain = new CountDownLatch(drainAfterNEvents);
        numEnqueued.set(0);//from   ww w . ja  va 2  s . c  o m
        q = new DrainableQueue<Integer>("queue");
        ExecutorService exec = Executors.newFixedThreadPool(NUM_PRODUCERS);
        CompletionService<Void> cs = new ExecutorCompletionService<Void>(exec);
        List<Future<Void>> futures = new ArrayList<Future<Void>>();
        for (int producer = 0; producer < NUM_PRODUCERS; ++producer) {
            futures.add(cs.submit(new Producer(producer)));
        }
        shouldDrain.await();
        eventsProcessed = 0;
        LOG.info("Starting draining the queue");
        q.drain(this);
        LOG.info("Finished draining the queue");
        assertEquals(numEnqueued.get(), eventsProcessed);
        LOG.info("Events processed: " + eventsProcessed + ", drainAfterNEvents: " + drainAfterNEvents);
        assertTrue(eventsProcessed >= drainAfterNEvents);
        for (Future<Void> f : futures) {
            try {
                f.get();
            } catch (ExecutionException ex) {
                LOG.error("Exception from producer thread", ex);
                if (ex.getCause() instanceof AssertionError) {
                    throw (AssertionError) ex.getCause();
                }
                throw ex;
            }
        }
        exec.shutdown();
        assertTrue(exec.awaitTermination(5, TimeUnit.SECONDS));
    }
}

---

From source file:org.apache.solr.update.SolrCmdDistributor.java

public SolrCmdDistributor(UpdateShardHandler updateShardHandler) {
    this.clients = new StreamingSolrClients(updateShardHandler);
    this.updateExecutor = updateShardHandler.getUpdateExecutor();
    this.completionService = new ExecutorCompletionService<>(updateExecutor);
}

---

From source file:com.linkedin.pinot.common.http.MultiGetRequest.java

/**
 * GET urls in parallel using the executor service.
 * @param urls absolute URLs to GET//from w w  w .j a  v a 2s.  c om
 * @param timeoutMs timeout in milliseconds for each GET request
 * @return instance of CompletionService. Completion service will provide
 *   results as they arrive. The order is NOT same as the order of URLs
 */
public CompletionService<GetMethod> execute(@Nonnull List<String> urls, final int timeoutMs) {
    Preconditions.checkNotNull(urls);
    Preconditions.checkArgument(timeoutMs > 0, "Timeout value for multi-get must be greater than 0");

    CompletionService<GetMethod> completionService = new ExecutorCompletionService<>(executor);
    for (final String url : urls) {
        completionService.submit(new Callable<GetMethod>() {
            @Override
            public GetMethod call() throws Exception {
                HttpClient client = new HttpClient(connectionManager);
                GetMethod getMethod = new GetMethod(url);
                getMethod.getParams().setSoTimeout(timeoutMs);
                // if all connections in the connection manager are busy this will wait to retrieve a connection
                // set time to wait to retrieve a connection from connection manager
                client.getParams().setConnectionManagerTimeout(timeoutMs);
                client.executeMethod(getMethod);
                return getMethod;
            }
        });
    }
    return completionService;
}

---

From source file:com.uber.hoodie.common.util.queue.BoundedInMemoryExecutor.java

/**
 * Start all Producers/*w w w.  j a v  a  2s .  c o m*/
 */
public ExecutorCompletionService<Boolean> startProducers() {
    // Latch to control when and which producer thread will close the queue
    final CountDownLatch latch = new CountDownLatch(producers.size());
    final ExecutorCompletionService<Boolean> completionService = new ExecutorCompletionService<Boolean>(
            executorService);
    producers.stream().map(producer -> {
        return completionService.submit(() -> {
            try {
                preExecute();
                producer.produce(queue);
            } catch (Exception e) {
                logger.error("error consuming records", e);
                queue.markAsFailed(e);
                throw e;
            } finally {
                synchronized (latch) {
                    latch.countDown();
                    if (latch.getCount() == 0) {
                        // Mark production as done so that consumer will be able to exit
                        queue.close();
                    }
                }
            }
            return true;
        });
    }).collect(Collectors.toList());
    return completionService;
}

---

From source file:nl.b3p.viewer.image.ImageManager.java

public void process() throws Exception {
    ExecutorService threadPool = Executors.newFixedThreadPool(MAX_TREADS);
    CompletionService<ImageCollector> pool = new ExecutorCompletionService<ImageCollector>(threadPool);
    for (ImageCollector ic : ics) {
        if (ic.getStatus() == ImageCollector.NEW) {
            pool.submit(ic);//  www . j ava2  s.c o m
        }
    }
    //wait for all to complete. Wait max 5 min
    for (int i = 0; i < ics.size(); i++) {
        pool.poll(5, TimeUnit.MINUTES).get();
    }
}

---

From source file:org.jclouds.aws.s3.xml.S3ParserTest.java

@Test
void testParseListAllMyBucketsParallelResponseTime() throws InterruptedException, ExecutionException {
    CompletionService<SortedSet<BucketMetadata>> completer = new ExecutorCompletionService<SortedSet<BucketMetadata>>(
            exec);/*from  ww w  .  j  av  a 2s.c om*/
    for (int i = 0; i < LOOP_COUNT; i++)
        completer.submit(new Callable<SortedSet<BucketMetadata>>() {
            public SortedSet<BucketMetadata> call() throws IOException, SAXException, HttpException {
                return runParseListAllMyBuckets();
            }
        });
    for (int i = 0; i < LOOP_COUNT; i++)
        assert completer.take().get() != null;
}

---

From source file:io.scigraph.owlapi.loader.BatchOwlLoader.java

public void loadOntology() throws InterruptedException, ExecutionException {
    CompletionService<Long> completionService = new ExecutorCompletionService<Long>(exec);
    Set<Future<?>> futures = new HashSet<>();
    if (!ontologies.isEmpty()) {
        for (int i = 0; i < numConsumers; i++) {
            futures.add(completionService.submit(consumerProvider.get()));
        }// w w  w  . ja  v a 2 s.c  o m
        for (int i = 0; i < numProducers; i++) {
            futures.add(completionService.submit(producerProvider.get()));
        }
        for (OntologySetup ontology : ontologies) {
            urlQueue.offer(ontology);
        }
        for (int i = 0; i < numProducers; i++) {
            urlQueue.offer(POISON_STR);
        }
    }

    while (futures.size() > 0) {
        Future<?> completedFuture = completionService.take();
        futures.remove(completedFuture);
        try {
            completedFuture.get();
        } catch (ExecutionException e) {
            logger.log(Level.SEVERE, "Stopping batchLoading due to: " + e.getMessage(), e);
            e.printStackTrace();
            exec.shutdownNow();
            throw new InterruptedException(e.getCause().getMessage());
        }
    }

    exec.shutdown();
    exec.awaitTermination(10, TimeUnit.DAYS);
    graph.shutdown();
    logger.info("Postprocessing...");
    postprocessorProvider.get().postprocess();

    if (cliqueConfiguration.isPresent()) {
        postprocessorProvider.runCliquePostprocessor(cliqueConfiguration.get());
    }

    postprocessorProvider.shutdown();

}

---

From source file:com.laudandjolynn.mytv.crawler.CrawlerGroup.java

@Override
public List<TvStation> crawlAllTvStation() {
    List<TvStation> resultList = new ArrayList<TvStation>();
    int size = crawlers.size();
    int maxThreadNum = Constant.CPU_PROCESSOR_NUM;
    ThreadFactory threadFactory = new BasicThreadFactory.Builder().namingPattern("Mytv_CrawlerGroup_%d")
            .build();//from   w  w  w.j  a v  a 2  s  .  co m
    ExecutorService executorService = Executors.newFixedThreadPool(size > maxThreadNum ? maxThreadNum : size,
            threadFactory);
    CompletionService<List<TvStation>> completionService = new ExecutorCompletionService<List<TvStation>>(
            executorService);
    for (final Crawler crawler : crawlers) {
        Callable<List<TvStation>> task = new Callable<List<TvStation>>() {
            @Override
            public List<TvStation> call() throws Exception {
                return crawler.crawlAllTvStation();
            }
        };
        completionService.submit(task);
    }
    executorService.shutdown();
    int count = 0;
    while (count < size) {
        try {
            List<TvStation> stationList = completionService.take().get();
            if (stationList != null) {
                resultList.addAll(stationList);
            }
        } catch (InterruptedException e) {
            logger.error("crawl task of all tv station interrupted.", e);
        } catch (ExecutionException e) {
            logger.error("crawl task of all tv station executed fail.", e);
        }
        count++;
    }

    for (CrawlEventListener listener : listeners) {
        listener.crawlEnd(new AllTvStationCrawlEndEvent(this, resultList));
    }
    return resultList;
}

---

From source file:org.springframework.batch.item.database.JpaPagingItemReaderAsyncTests.java

/**
 * @throws Exception//from  w  w  w.ja va  2 s.  c o  m
 * @throws InterruptedException
 * @throws ExecutionException
 */
private void doTest() throws Exception, InterruptedException, ExecutionException {
    final JpaPagingItemReader<Foo> reader = getItemReader();
    CompletionService<List<Foo>> completionService = new ExecutorCompletionService<List<Foo>>(
            Executors.newFixedThreadPool(THREAD_COUNT));
    for (int i = 0; i < THREAD_COUNT; i++) {
        completionService.submit(new Callable<List<Foo>>() {
            @Override
            public List<Foo> call() throws Exception {
                List<Foo> list = new ArrayList<Foo>();
                Foo next = null;
                do {
                    next = reader.read();
                    Thread.sleep(10L);
                    logger.debug("Reading item: " + next);
                    if (next != null) {
                        list.add(next);
                    }
                } while (next != null);
                return list;
            }
        });
    }
    int count = 0;
    Set<Foo> results = new HashSet<Foo>();
    for (int i = 0; i < THREAD_COUNT; i++) {
        List<Foo> items = completionService.take().get();
        count += items.size();
        logger.debug("Finished items count: " + items.size());
        logger.debug("Finished items: " + items);
        assertNotNull(items);
        results.addAll(items);
    }
    assertEquals(ITEM_COUNT, count);
    assertEquals(ITEM_COUNT, results.size());
    reader.close();
}

---

From source file:org.springframework.amqp.rabbit.test.RepeatProcessor.java

public Statement apply(final Statement base, FrameworkMethod method, final Object target) {

    Repeat repeat = AnnotationUtils.findAnnotation(method.getMethod(), Repeat.class);
    if (repeat == null) {
        return base;
    }//  w ww .j a va 2s.  c  o m

    final int repeats = repeat.value();
    if (repeats <= 1) {
        return base;
    }

    initializeIfNecessary(target);

    if (concurrency <= 0) {
        return new Statement() {
            @Override
            public void evaluate() throws Throwable {
                try {
                    for (int i = 0; i < repeats; i++) {
                        try {
                            base.evaluate();
                        } catch (Throwable t) {
                            throw new IllegalStateException(
                                    "Failed on iteration: " + i + " of " + repeats + " (started at 0)", t);
                        }
                    }
                } finally {
                    finalizeIfNecessary(target);
                }
            }
        };
    }
    return new Statement() {
        @Override
        public void evaluate() throws Throwable {
            List<Future<Boolean>> results = new ArrayList<Future<Boolean>>();
            ExecutorService executor = Executors.newFixedThreadPool(concurrency);
            CompletionService<Boolean> completionService = new ExecutorCompletionService<Boolean>(executor);
            try {
                for (int i = 0; i < repeats; i++) {
                    final int count = i;
                    results.add(completionService.submit(new Callable<Boolean>() {
                        public Boolean call() {
                            try {
                                base.evaluate();
                            } catch (Throwable t) {
                                throw new IllegalStateException("Failed on iteration: " + count, t);
                            }
                            return true;
                        }
                    }));
                }
                for (int i = 0; i < repeats; i++) {
                    Future<Boolean> future = completionService.take();
                    assertTrue("Null result from completer", future.get());
                }
            } finally {
                executor.shutdownNow();
                finalizeIfNecessary(target);
            }
        }
    };
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• »