Example usage for java.util.concurrent CompletionService take

List of usage examples for java.util.concurrent CompletionService take

  1. HOME
  2. Java
  3. java
  4. java.util.concurrent.*
  5. CompletionService
  6. take

Introduction

In this page you can find the example usage for java.util.concurrent CompletionService take.

Prototype

Future<V> take() throws InterruptedException;

Source Link

Document

Retrieves and removes the Future representing the next completed task, waiting if none are yet present.

Usage

From source file:org.nickelproject.nickel.blobStore.S3BlobStore.java

private void putMultiPartByteArray(final BlobRef blobRef, final byte[] pBytes) {
    final CompletionService<PartETag> completionService = new ExecutorCompletionService<PartETag>(executor);
    final String key = blobRef.toString();
    final PartTransferInterface partGetter = RetryProxy.newInstance(PartTransferInterface.class,
            new PartTransfer());
    final String uploadId = partGetter.initiateUpload(key);
    int partNumber = 1;
    for (int start = 0; start < pBytes.length; start += uploadPartSize) {
        final long length = Math.min(uploadPartSize, pBytes.length - start);
        completionService.submit(new PutPartCallable(start, (int) length, pBytes, partNumber++, key, uploadId,
                start + uploadPartSize >= pBytes.length));
    }// www .ja  v a  2  s .  c o  m
    final List<PartETag> partETags = Lists.newArrayList();
    for (int i = 1; i < partNumber; i++) {
        try {
            partETags.add(completionService.take().get());
        } catch (final InterruptedException e1) {
            Thread.currentThread().interrupt();
            throw RethrownException.rethrow(e1);
        } catch (final ExecutionException e2) {
            throw RethrownException.rethrow(e2);
        }
    }
    partGetter.completeUpload(key, uploadId, partETags);
}

From source file:com.threadswarm.imagefeedarchiver.driver.CommandLineDriver.java

@Override
public void run() {
    //setup filters
    List<RssItemFilter> filterList = new LinkedList<RssItemFilter>();
    filterList.add(new PreviouslyDownloadedItemFilter(processedRssItemDAO));
    RssItemFilter chainedItemFilter = new ChainedRssItemFilter(filterList);

    RssChannel rssChannel = null;//from  ww  w .  ja  va  2  s .  c o m
    try {
        rssChannel = fetchRssChannel(rssFeedUri);
    } catch (IOException | FeedParserException e) {
        LOGGER.error(
                "An Exception was thrown while attempting to download and parse the target RSS feed.. exiting",
                e);
        System.exit(1);
    }

    List<RssItem> filteredItemList = new LinkedList<RssItem>();
    if (rssChannel != null && rssChannel.getItems() != null) {
        for (RssItem rssItem : rssChannel.getItems()) {
            rssItem = chainedItemFilter.filter(rssItem);
            if (rssItem != null)
                filteredItemList.add(rssItem);
        }
    }

    if (!filteredItemList.isEmpty()) {
        //create list of headers to be used when downloading images
        List<Header> headerList = new ArrayList<Header>(2);
        if (doNotTrackRequested) {
            LOGGER.debug("Adding 'DNT' header to worker requests");
            headerList.add(DNT_HEADER);
        }
        headerList.add(new BasicHeader(HttpHeaders.REFERER, rssFeedUri.toString()));
        headerList = Collections.unmodifiableList(headerList);

        ExecutorService executorService = null;
        try {
            executorService = Executors.newFixedThreadPool(threadCount);
            CompletionService<ProcessedRssItem> completionService = new ExecutorCompletionService<ProcessedRssItem>(
                    executorService);
            Set<URI> processedURISet = new ConcurrentSkipListSet<URI>();
            int itemCount = 0;
            for (RssItem rssItem : filteredItemList) {
                completionService.submit(new RssItemProcessor(httpClient, rssItem, processedRssItemDAO,
                        outputDirectory, headerList, processedURISet, downloadDelay, forceHttps));
                itemCount++;
            }

            LOGGER.info("{} jobs submitted for execution", itemCount);

            for (int x = 0; x < itemCount; x++) {
                ProcessedRssItem processedItem = completionService.take().get();
                LOGGER.info("Item status: {} --> [{}]", processedItem.getRssItem().getTitle(),
                        processedItem.getDownloadStatus());
            }
        } catch (InterruptedException e) {
            LOGGER.warn("Thread interrupted while blocking", e);
            Thread.currentThread().interrupt(); // restore interrupt
        } catch (ExecutionException e) {
            LOGGER.error("An Exception was thrown during worker execution and subsequently propagated", e);
            e.printStackTrace();
        } finally {
            executorService.shutdown();
            try {
                executorService.awaitTermination(10, TimeUnit.SECONDS);
            } catch (InterruptedException e) {
                LOGGER.warn("Thread interrupted while blocking", e);
                Thread.currentThread().interrupt(); // restore interrupt
            }
            httpClient.getConnectionManager().shutdown();
        }
    }
}

From source file:org.springframework.batch.item.database.JdbcPagingItemReaderAsyncTests.java

/**
 * @throws Exception/* www  . java2s.c  o m*/
 * @throws InterruptedException
 * @throws ExecutionException
 */
private void doTest() throws Exception, InterruptedException, ExecutionException {
    final ItemReader<Foo> reader = getItemReader();
    CompletionService<List<Foo>> completionService = new ExecutorCompletionService<List<Foo>>(
            Executors.newFixedThreadPool(THREAD_COUNT));
    for (int i = 0; i < THREAD_COUNT; i++) {
        completionService.submit(new Callable<List<Foo>>() {
            @Override
            public List<Foo> call() throws Exception {
                List<Foo> list = new ArrayList<Foo>();
                Foo next = null;
                do {
                    next = reader.read();
                    Thread.sleep(10L);
                    logger.debug("Reading item: " + next);
                    if (next != null) {
                        list.add(next);
                    }
                } while (next != null);
                return list;
            }
        });
    }
    int count = 0;
    Set<Foo> results = new HashSet<Foo>();
    for (int i = 0; i < THREAD_COUNT; i++) {
        List<Foo> items = completionService.take().get();
        count += items.size();
        logger.debug("Finished items count: " + items.size());
        logger.debug("Finished items: " + items);
        assertNotNull(items);
        results.addAll(items);
    }
    assertEquals(ITEM_COUNT, count);
    assertEquals(ITEM_COUNT, results.size());
}

From source file:org.springframework.batch.item.database.IbatisPagingItemReaderAsyncTests.java

/**
 * @throws Exception//from  w  ww.j  a v  a2  s .c o m
 * @throws InterruptedException
 * @throws ExecutionException
 */
private void doTest() throws Exception, InterruptedException, ExecutionException {
    final IbatisPagingItemReader<Foo> reader = getItemReader();
    reader.setDataSource(dataSource);
    CompletionService<List<Foo>> completionService = new ExecutorCompletionService<List<Foo>>(
            Executors.newFixedThreadPool(THREAD_COUNT));
    for (int i = 0; i < THREAD_COUNT; i++) {
        completionService.submit(new Callable<List<Foo>>() {
            @Override
            public List<Foo> call() throws Exception {
                List<Foo> list = new ArrayList<Foo>();
                Foo next = null;
                do {
                    next = reader.read();
                    Thread.sleep(10L); // try to make it fairer
                    logger.debug("Reading item: " + next);
                    if (next != null) {
                        list.add(next);
                    }
                } while (next != null);
                return list;
            }
        });
    }
    int count = 0;
    Set<Foo> results = new HashSet<Foo>();
    for (int i = 0; i < THREAD_COUNT; i++) {
        List<Foo> items = completionService.take().get();
        count += items.size();
        logger.debug("Finished items count: " + items.size());
        logger.debug("Finished items: " + items);
        assertNotNull(items);
        results.addAll(items);
    }
    assertEquals(ITEM_COUNT, count);
    assertEquals(ITEM_COUNT, results.size());
    reader.close();
}

From source file:org.m2x.rssreader.service.FetcherService.java

private int refreshFeeds() {
    ContentResolver cr = getContentResolver();
    final Cursor cursor = cr.query(FeedColumns.CONTENT_URI, FeedColumns.PROJECTION_ID, null, null, null);
    int nbFeed = cursor.getCount();

    ExecutorService executor = Executors.newFixedThreadPool(THREAD_NUMBER, new ThreadFactory() {
        @Override//from  w  w w . ja va  2  s  . c o  m
        public Thread newThread(Runnable r) {
            Thread t = new Thread(r);
            t.setPriority(Thread.MIN_PRIORITY);
            return t;
        }
    });

    CompletionService<Integer> completionService = new ExecutorCompletionService<Integer>(executor);
    while (cursor.moveToNext()) {
        final String feedId = cursor.getString(0);
        completionService.submit(new Callable<Integer>() {
            @Override
            public Integer call() {
                int result = 0;
                try {
                    result = refreshFeed(feedId);
                } catch (Exception ignored) {
                }
                return result;
            }
        });
    }
    cursor.close();

    int globalResult = 0;
    for (int i = 0; i < nbFeed; i++) {
        try {
            Future<Integer> f = completionService.take();
            globalResult += f.get();
        } catch (Exception ignored) {
        }
    }

    executor.shutdownNow(); // To purge all threads

    return globalResult;
}

From source file:org.springframework.integration.groovy.GroovyExpressionTests.java

@Test
public void testScriptFactoryCustomizerThreadSafety() throws Exception {
    final Customizer customizer = new Customizer(Collections.singletonMap("name", (Object) "foo"));
    final GroovyScriptFactory factory = new GroovyScriptFactory("Groovy Script", customizer);
    final ResourceScriptSource scriptSource = new ResourceScriptSource(
            new NamedByteArrayResource("\"name=${name}\"".getBytes(), "InlineScript"));
    Object scriptedObject = factory.getScriptedObject(scriptSource, null);
    assertEquals("name=foo", scriptedObject.toString());
    CompletionService<String> completionService = new ExecutorCompletionService<String>(
            Executors.newFixedThreadPool(10));
    for (int i = 0; i < 100; i++) {
        final String name = "bar" + i;
        completionService.submit(new Callable<String>() {
            public String call() throws Exception {
                Object scriptedObject;
                synchronized (customizer) {
                    customizer.setMap(Collections.singletonMap("name", (Object) name));
                    scriptedObject = factory.getScriptedObject(scriptSource, null);
                }/*w w  w . j a  va2 s  .c  om*/
                String result = scriptedObject.toString();
                logger.debug("Result=" + result + " with name=" + name);
                if (!("name=" + name).equals(result)) {
                    throw new IllegalStateException("Wrong value (" + result + ") for: " + name);
                }
                return name;
            }
        });
    }
    Set<String> set = new HashSet<String>();
    for (int i = 0; i < 100; i++) {
        set.add(completionService.take().get());
    }
    assertEquals(100, set.size());
}

From source file:org.opoo.press.maven.wagon.github.GitHub.java

private List<TreeEntry> createEntriesInThreads(List<TreeEntry> entries, final String prefix,
        final String[] paths, final DataService service, final RepositoryId repository,
        final File outputDirectory, int numThreads) throws GitHubException {
    ExecutorService threadPool = Executors.newFixedThreadPool(numThreads);//.newCachedThreadPool();  
    CompletionService<TreeEntry> cs = new ExecutorCompletionService<TreeEntry>(threadPool);

    for (final String path : paths) {
        cs.submit(new Callable<TreeEntry>() {
            @Override/*  w  w w.ja va2 s.  c  o  m*/
            public TreeEntry call() throws Exception {
                return createEntry(prefix, path, service, repository, outputDirectory);
            }
        });
    }

    try {
        //BUG: wait for ever??
        //         Future<TreeEntry> future = cs.take();
        //         while(future != null){
        //            entries.add(future.get());
        //            future = cs.take();
        //         }

        for (int i = 0; i < paths.length; i++) {
            entries.add(cs.take().get());
        }
        log.info("All entries created: " + paths.length);
    } catch (InterruptedException e) {
        throw new GitHubException("", e);
    } catch (ExecutionException e) {
        throw new GitHubException("", e);
    }
    return entries;
}

From source file:org.apache.hadoop.hbase.wal.LogRecoveredEditsOutputSink.java

/**
 * @param completionService threadPool to execute the closing tasks
 * @param thrown store the exceptions/*from  ww w . j av  a  2  s.  c  o m*/
 * @param paths arrayList to store the paths written
 * @return if close tasks executed successful
 */
boolean executeCloseTask(CompletionService<Void> completionService, List<IOException> thrown, List<Path> paths)
        throws InterruptedException, ExecutionException {
    for (final Map.Entry<String, WALSplitter.SinkWriter> writersEntry : writers.entrySet()) {
        if (LOG.isTraceEnabled()) {
            LOG.trace("Submitting close of " + ((WALSplitter.WriterAndPath) writersEntry.getValue()).path);
        }
        completionService.submit(new Callable<Void>() {
            @Override
            public Void call() throws Exception {
                WALSplitter.WriterAndPath wap = (WALSplitter.WriterAndPath) writersEntry.getValue();
                Path dst = closeWriter(writersEntry.getKey(), wap, thrown);
                paths.add(dst);
                return null;
            }
        });
    }
    boolean progress_failed = false;
    for (int i = 0, n = this.writers.size(); i < n; i++) {
        Future<Void> future = completionService.take();
        future.get();
        if (!progress_failed && reporter != null && !reporter.progress()) {
            progress_failed = true;
        }
    }
    return progress_failed;
}

From source file:org.springframework.integration.groovy.GroovyExpressionTests.java

@Test
public void testScriptFactoryCustomizerThreadSafetyWithNewScript() throws Exception {
    final Customizer customizer = new Customizer(Collections.singletonMap("name", (Object) "foo"));
    final GroovyScriptFactory factory = new GroovyScriptFactory("Groovy Script", customizer);
    CompletionService<String> completionService = new ExecutorCompletionService<String>(
            Executors.newFixedThreadPool(5));
    for (int i = 0; i < 100; i++) {
        final String name = "Bar" + i;
        completionService.submit(new Callable<String>() {
            public String call() throws Exception {
                Object scriptedObject;
                synchronized (customizer) {
                    customizer.setMap(Collections.singletonMap("name", (Object) name));
                    ResourceScriptSource scriptSource = new ResourceScriptSource(
                            new NamedByteArrayResource("\"name=${name}\"".getBytes(), "InlineScript" + name));
                    scriptedObject = factory.getScriptedObject(scriptSource, null);
                }//from   www  . j  av a 2  s . co m
                String result = scriptedObject.toString();
                logger.debug("Result=" + result + " with name=" + name);
                if (!("name=" + name).equals(result)) {
                    throw new IllegalStateException("Wrong value (" + result + ") for: " + name);
                }
                return name;
            }
        });
    }
    Set<String> set = new HashSet<String>();
    for (int i = 0; i < 100; i++) {
        set.add(completionService.take().get());
    }
    assertEquals(100, set.size());
}

From source file:org.paxle.filter.robots.impl.RobotsTxtManager.java

/**
 * Check a list of {@link URI URI} against the robots.txt file of the servers hosting the {@link URI}.
 * @param hostPort the web-server hosting the {@link URI URIs}
 * @param urlList a list of {@link URI}/* ww  w.  ja  v  a2  s. co m*/
 * 
 * @return all {@link URI} that are blocked by the servers
 */
public List<URI> isDisallowed(Collection<URI> urlList) {
    if (urlList == null)
        throw new NullPointerException("The URI-list is null.");

    // group the URL list based on hostname:port
    HashMap<URI, List<URI>> uriBlocks = this.groupURI(urlList);
    ArrayList<URI> disallowedURI = new ArrayList<URI>();

    /*
     * Asynchronous execution and parallel check of all blocks 
     */
    final CompletionService<Collection<URI>> execCompletionService = new ExecutorCompletionService<Collection<URI>>(
            this.execService);

    // loop through the blocks and start a worker for each block
    for (Entry<URI, List<URI>> uriBlock : uriBlocks.entrySet()) {
        URI baseUri = uriBlock.getKey();
        List<URI> uriList = uriBlock.getValue();
        execCompletionService.submit(new RobotsTxtManagerCallable(baseUri, uriList));
    }

    // wait for the worker-threads to finish execution
    for (int i = 0; i < uriBlocks.size(); ++i) {
        try {
            Collection<URI> disallowedInGroup = execCompletionService.take().get();
            if (disallowedInGroup != null) {
                disallowedURI.addAll(disallowedInGroup);
            }
        } catch (InterruptedException e) {
            this.logger.info(String.format("Interruption detected while waiting for robots.txt-check result."));
            // XXX should we break here?
        } catch (ExecutionException e) {
            this.logger.error(
                    String.format("Unexpected '%s' while performing robots.txt check.", e.getClass().getName()),
                    e);
        }
    }

    return disallowedURI;
}