Example usage for java.util.concurrent CompletionService take

List of usage examples for java.util.concurrent CompletionService take

  1. HOME
  2. Java
  3. java
  4. java.util.concurrent.*
  5. CompletionService
  6. take

Introduction

In this page you can find the example usage for java.util.concurrent CompletionService take.

Prototype

Future<V> take() throws InterruptedException;

Source Link

Document

Retrieves and removes the Future representing the next completed task, waiting if none are yet present.

Usage

From source file:org.apache.syncope.core.provisioning.java.propagation.PriorityPropagationTaskExecutor.java

@Override
protected void doExecute(final Collection<PropagationTask> tasks, final PropagationReporter reporter,
        final boolean nullPriorityAsync) {

    List<PropagationTask> prioritizedTasks = CollectionUtils.select(tasks, new Predicate<PropagationTask>() {

        @Override//from   www. j a va2s . co m
        public boolean evaluate(final PropagationTask task) {
            return task.getResource().getPropagationPriority() != null;
        }
    }, new ArrayList<PropagationTask>());
    Collections.sort(prioritizedTasks, new PriorityComparator());
    LOG.debug("Propagation tasks sorted by priority, for serial execution: {}", prioritizedTasks);

    Collection<PropagationTask> concurrentTasks = CollectionUtils.subtract(tasks, prioritizedTasks);
    LOG.debug("Propagation tasks for concurrent execution: {}", concurrentTasks);

    // first process priority resources sequentially and fail as soon as any propagation failure is reported
    for (PropagationTask task : prioritizedTasks) {
        TaskExec execution = null;
        PropagationTaskExecStatus execStatus;
        try {
            execution = newPropagationTaskCallable(task, reporter).call();
            execStatus = PropagationTaskExecStatus.valueOf(execution.getStatus());
        } catch (Exception e) {
            LOG.error("Unexpected exception", e);
            execStatus = PropagationTaskExecStatus.FAILURE;
        }
        if (execStatus != PropagationTaskExecStatus.SUCCESS) {
            throw new PropagationException(task.getResource().getKey(),
                    execution == null ? null : execution.getMessage());
        }
    }

    // then process non-priority resources concurrently...
    final CompletionService<TaskExec> completionService = new ExecutorCompletionService<>(executor);
    Map<PropagationTask, Future<TaskExec>> nullPriority = new HashMap<>(concurrentTasks.size());
    for (PropagationTask task : concurrentTasks) {
        try {
            nullPriority.put(task, completionService.submit(newPropagationTaskCallable(task, reporter)));
        } catch (Exception e) {
            LOG.error("Unexpected exception", e);
        }
    }
    // ...waiting for all callables to complete, if async processing was not required
    if (!nullPriority.isEmpty()) {
        if (nullPriorityAsync) {
            for (Map.Entry<PropagationTask, Future<TaskExec>> entry : nullPriority.entrySet()) {
                reporter.onSuccessOrNonPriorityResourceFailures(entry.getKey(),
                        PropagationTaskExecStatus.CREATED, null, null, null);
            }
        } else {
            final Set<Future<TaskExec>> nullPriorityFutures = new HashSet<>(nullPriority.values());
            try {
                executor.submit(new Runnable() {

                    @Override
                    public void run() {
                        while (!nullPriorityFutures.isEmpty()) {
                            try {
                                nullPriorityFutures.remove(completionService.take());
                            } catch (Exception e) {
                                LOG.error("Unexpected exception", e);
                            }
                        }
                    }
                }).get(60, TimeUnit.SECONDS);
            } catch (Exception e) {
                LOG.error("Unexpected exception", e);
            } finally {
                for (Future<TaskExec> future : nullPriorityFutures) {
                    future.cancel(true);
                }
                nullPriorityFutures.clear();
                nullPriority.clear();
            }
        }
    }
}

From source file:com.rapid7.diskstorage.dynamodb.DynamoDBDelegate.java

public void parallelMutate(List<MutateWorker> workers) throws BackendException {
    CompletionService<Void> completion = new ExecutorCompletionService<>(clientThreadPool);
    List<Future<Void>> futures = Lists.newLinkedList();
    for (MutateWorker worker : workers) {
        futures.add(completion.submit(worker));
    }//from   w ww  .  j  ava 2  s  .c  o m

    //block on the futures all getting or throwing instead of using a latch as i need to check future status anyway
    boolean interrupted = false;
    try {
        for (int i = 0; i < workers.size(); i++) {
            try {
                completion.take().get(); //Void
            } catch (InterruptedException e) {
                interrupted = true;
                // fail out because titan does not poll this thread for interrupted anywhere
                throw new BackendRuntimeException("was interrupted during parallelMutate");
            } catch (ExecutionException e) {
                throw unwrapExecutionException(e, MUTATE_ITEM);
            }
        }
    } finally {
        for (Future<Void> future : futures) {
            if (!future.isDone()) {
                future.cancel(interrupted /* mayInterruptIfRunning */);
            }
        }
        if (interrupted) {
            // set interrupted on this thread
            Thread.currentThread().interrupt();
        }
    }
}

From source file:com.amazon.janusgraph.diskstorage.dynamodb.DynamoDBDelegate.java

public void parallelMutate(List<MutateWorker> workers) throws BackendException {
    CompletionService<Void> completion = new ExecutorCompletionService<>(clientThreadPool);
    List<Future<Void>> futures = Lists.newLinkedList();
    for (MutateWorker worker : workers) {
        futures.add(completion.submit(worker));
    }/*from w  w w.  j a  v a  2s  .c o  m*/

    //block on the futures all getting or throwing instead of using a latch as i need to check future status anyway
    boolean interrupted = false;
    try {
        for (int i = 0; i < workers.size(); i++) {
            try {
                completion.take().get(); //Void
            } catch (InterruptedException e) {
                interrupted = true;
                // fail out because janusgraph does not poll this thread for interrupted anywhere
                throw new BackendRuntimeException("was interrupted during parallelMutate");
            } catch (ExecutionException e) {
                throw unwrapExecutionException(e, MUTATE_ITEM);
            }
        }
    } finally {
        for (Future<Void> future : futures) {
            if (!future.isDone()) {
                future.cancel(interrupted /* mayInterruptIfRunning */);
            }
        }
        if (interrupted) {
            // set interrupted on this thread
            Thread.currentThread().interrupt();
        }
    }
}

From source file:com.rapid7.diskstorage.dynamodb.DynamoDBDelegate.java

public List<QueryResultWrapper> parallelQuery(List<QueryWorker> queryWorkers) throws BackendException {
    CompletionService<QueryResultWrapper> completionService = new ExecutorCompletionService<>(clientThreadPool);

    List<Future<QueryResultWrapper>> futures = Lists.newLinkedList();
    for (QueryWorker worker : queryWorkers) {
        futures.add(completionService.submit(worker));
    }//from w ww  . j a  va  2  s.  c  o  m

    boolean interrupted = false;
    List<QueryResultWrapper> results = Lists.newLinkedList();
    try {
        for (int i = 0; i < queryWorkers.size(); i++) {
            try {
                QueryResultWrapper result = completionService.take().get();
                results.add(result);
            } catch (InterruptedException e) {
                interrupted = true;
                // fail out because titan does not poll this thread for interrupted anywhere
                throw new BackendRuntimeException("was interrupted during parallelQuery");
            } catch (ExecutionException e) {
                throw unwrapExecutionException(e, QUERY);
            }
        }
    } finally {
        for (Future<QueryResultWrapper> future : futures) {
            if (!future.isDone()) {
                future.cancel(interrupted /* mayInterruptIfRunning */);
            }
        }

        if (interrupted) {
            // set interrupted on this thread and fail out
            Thread.currentThread().interrupt();
        }
    }
    return results;
}

From source file:net.sourceforge.seqware.pipeline.plugins.PluginRunnerIT.java

public void testLatestWorkflowsInternal(List<Integer> accessions) throws IOException {
    String output = ITUtility.runSeqWareJar(
            "-p net.sourceforge.seqware.pipeline.plugins.BundleManager -- --list-installed",
            ReturnValue.SUCCESS);//from   ww  w .ja  va2 s  .  c  om
    Assert.assertTrue("output should include installed workflows", output.contains("INSTALLED WORKFLOWS"));
    Map<String, WorkflowInfo> latestWorkflows = new HashMap<String, WorkflowInfo>();
    String[] lines = output.split(System.getProperty("line.separator"));
    for (String line : lines) {
        String[] lineParts = line.split("\t");
        try {
            int workflow_accession = Integer.valueOf(lineParts[3]);
            String workflowName = lineParts[0];
            String path = lineParts[4];
            if (path.equals("null")) {
                continue;
            }
            WorkflowInfo wi = new WorkflowInfo(workflow_accession, path, workflowName, lineParts[1]);

            //TODO: check that the permanent workflow actually exists, if not warn and skip
            File fileAtPath = new File(path);
            if (!fileAtPath.exists()) {
                Log.warn("Skipping " + workflowName + ":" + workflow_accession
                        + " , bundle path does not exist at " + path);
                continue;
            }

            if (!latestWorkflows.containsKey(workflowName)) {
                latestWorkflows.put(workflowName, wi);
            } else {
                // contained
                int old = latestWorkflows.get(workflowName).sw_accession;
                if (workflow_accession > old) {
                    latestWorkflows.put(workflowName, wi);
                }
            }
        } catch (Exception e) {
            /**
             * do nothing and skip this line of the BundleManager output
             */
        }
    }
    // setup thread pool
    ExecutorService threadPool = Executors.newFixedThreadPool(latestWorkflows.size());
    CompletionService<String> pool = new ExecutorCompletionService<String>(threadPool);
    for (Entry<String, WorkflowInfo> e : latestWorkflows.entrySet()) {
        System.out.println("Testing " + e.getKey() + " " + e.getValue().sw_accession);

        // if we have an accession list, skip accessions that are not in it
        if (accessions.size() > 0) {
            Integer acc = e.getValue().sw_accession;
            if (!accessions.contains(acc)) {
                System.out.println(
                        "Skipping " + e.getKey() + " " + e.getValue().sw_accession + " due to accession list");
                continue;
            }
        }

        StringBuilder params = new StringBuilder();
        params.append("--bundle ").append(e.getValue().path).append(" ");
        params.append("--version ").append(e.getValue().version).append(" ");
        params.append("--test ");
        File tempFile = File.createTempFile(e.getValue().name, ".out");
        pool.submit(new TestingThread(params.toString(), tempFile));
    }
    for (Entry<String, WorkflowInfo> e : latestWorkflows.entrySet()) {
        try {
            pool.take().get();
        } catch (InterruptedException ex) {
            Log.error(ex);
        } catch (ExecutionException ex) {
            Log.error(ex);
        }
    }
    threadPool.shutdown();
}

From source file:com.amazon.janusgraph.diskstorage.dynamodb.DynamoDBDelegate.java

public List<QueryResultWrapper> parallelQuery(List<QueryWorker> queryWorkers) throws BackendException {
    CompletionService<QueryResultWrapper> completionService = new ExecutorCompletionService<>(clientThreadPool);

    List<Future<QueryResultWrapper>> futures = Lists.newLinkedList();
    for (QueryWorker worker : queryWorkers) {
        futures.add(completionService.submit(worker));
    }//from www. j  av  a  2 s .  c o  m

    boolean interrupted = false;
    List<QueryResultWrapper> results = Lists.newLinkedList();
    try {
        for (int i = 0; i < queryWorkers.size(); i++) {
            try {
                QueryResultWrapper result = completionService.take().get();
                results.add(result);
            } catch (InterruptedException e) {
                interrupted = true;
                // fail out because janusgraph does not poll this thread for interrupted anywhere
                throw new BackendRuntimeException("was interrupted during parallelQuery");
            } catch (ExecutionException e) {
                throw unwrapExecutionException(e, QUERY);
            }
        }
    } finally {
        for (Future<QueryResultWrapper> future : futures) {
            if (!future.isDone()) {
                future.cancel(interrupted /* mayInterruptIfRunning */);
            }
        }

        if (interrupted) {
            // set interrupted on this thread and fail out
            Thread.currentThread().interrupt();
        }
    }
    return results;
}

From source file:nl.privacybarometer.privacyvandaag.service.FetcherService.java

private int refreshFeeds(final long keepDateBorderTime) {
    ContentResolver cr = getContentResolver();
    final Cursor cursor = cr.query(FeedColumns.CONTENT_URI, FeedColumns.PROJECTION_ID, null, null, null);
    int nbFeed = (cursor != null) ? cursor.getCount() : 0;

    ExecutorService executor = Executors.newFixedThreadPool(THREAD_NUMBER, new ThreadFactory() {
        @Override/*from www.  j  a  va2s. c om*/
        public Thread newThread(Runnable r) {
            Thread t = new Thread(r);
            t.setPriority(Thread.MIN_PRIORITY);
            return t;
        }
    });

    CompletionService<Integer> completionService = new ExecutorCompletionService<>(executor);
    while (cursor != null && cursor.moveToNext()) {
        final String feedId = cursor.getString(0);
        completionService.submit(new Callable<Integer>() {
            @Override
            public Integer call() {
                int result = 0;
                try {
                    result = refreshFeed(feedId, keepDateBorderTime);
                } catch (Exception e) {
                    Log.e(TAG, "Error refreshing feed " + e.getMessage());
                }
                return result;
            }
        });
    }
    if (cursor != null)
        cursor.close();

    int globalResult = 0;
    for (int i = 0; i < nbFeed; i++) {
        try {
            Future<Integer> f = completionService.take(); // ModPrivacyVandaag: the count of new articles after a feed is refreshed
            globalResult += f.get();
        } catch (Exception e) {
            Log.e(TAG, "Error counting new articles " + e.getMessage());
        }
    }

    executor.shutdownNow(); // To purge all threads

    return globalResult; // ModPrivacyVandaag: As far as I can see: this contains the number of new articles from a refresh of the feeds.
}

From source file:net.sourceforge.seqware.pipeline.plugins.PluginRunnerET.java

public void testLatestWorkflowsInternal(List<Integer> accessions) throws IOException {
    String output = ITUtility.runSeqWareJar(
            "-p net.sourceforge.seqware.pipeline.plugins.BundleManager -- --list-installed",
            ReturnValue.SUCCESS, null);//  w  w w.j ava2s  . com
    Assert.assertTrue("output should include installed workflows", output.contains("INSTALLED WORKFLOWS"));
    Map<String, WorkflowInfo> latestWorkflows = new HashMap<>();
    String[] lines = output.split(System.getProperty("line.separator"));
    for (String line : lines) {
        String[] lineParts = line.split("\t");
        try {
            int workflow_accession = Integer.valueOf(lineParts[3]);
            String workflowName = lineParts[0];
            String path = lineParts[lineParts.length - 2];
            if (path.equals("null")) {
                continue;
            }
            WorkflowInfo wi = new WorkflowInfo(workflow_accession, path, workflowName, lineParts[1]);

            //TODO: check that the permanent workflow actually exists, if not warn and skip
            File fileAtPath = new File(path);
            if (!fileAtPath.exists()) {
                Log.warn("Skipping " + workflowName + ":" + workflow_accession
                        + " , bundle path does not exist at " + path);
                continue;
            }

            if (!latestWorkflows.containsKey(workflowName)) {
                latestWorkflows.put(workflowName, wi);
            } else {
                // contained
                int old = latestWorkflows.get(workflowName).sw_accession;
                if (workflow_accession > old) {
                    latestWorkflows.put(workflowName, wi);
                }
            }
        } catch (Exception e) {
            /**
             * do nothing and skip this line of the BundleManager output
             */
        }
    }
    // setup thread pool
    ExecutorService threadPool = Executors.newFixedThreadPool(latestWorkflows.size());
    CompletionService<String> pool = new ExecutorCompletionService<>(threadPool);
    for (Entry<String, WorkflowInfo> e : latestWorkflows.entrySet()) {
        System.out.println("Testing " + e.getKey() + " " + e.getValue().sw_accession);

        // if we have an accession list, skip accessions that are not in it
        if (accessions.size() > 0) {
            Integer acc = e.getValue().sw_accession;
            if (!accessions.contains(acc)) {
                System.out.println(
                        "Skipping " + e.getKey() + " " + e.getValue().sw_accession + " due to accession list");
                continue;
            }
        }

        StringBuilder params = new StringBuilder();
        params.append("--bundle ").append(e.getValue().path).append(" ");
        params.append("--version ").append(e.getValue().version).append(" ");
        params.append("--test ");
        File tempFile = File.createTempFile(e.getValue().name, ".out");
        pool.submit(new TestingThread(params.toString(), tempFile));
    }
    for (Entry<String, WorkflowInfo> e : latestWorkflows.entrySet()) {
        try {
            pool.take().get();
        } catch (InterruptedException ex) {
            Log.error(ex);
        } catch (ExecutionException ex) {
            Log.error(ex);
        }
    }
    threadPool.shutdown();
}

From source file:com.rapid7.diskstorage.dynamodb.DynamoDBDelegate.java

public Map<StaticBuffer, GetItemResult> parallelGetItem(List<GetItemWorker> workers) throws BackendException {
    final CompletionService<GetItemResultWrapper> completionService = new ExecutorCompletionService<>(
            clientThreadPool);//  ww w  .jav a2s  . c om

    final List<Future<GetItemResultWrapper>> futures = Lists.newLinkedList();
    for (GetItemWorker worker : workers) {
        futures.add(completionService.submit(worker));
    }

    boolean interrupted = false;
    final Map<StaticBuffer, GetItemResult> results = Maps.newHashMap();
    try {
        for (int i = 0; i < workers.size(); i++) {
            try {
                GetItemResultWrapper result = completionService.take().get();
                results.put(result.getTitanKey(), result.getDynamoDBResult());
            } catch (InterruptedException e) {
                interrupted = true;
                throw new BackendRuntimeException("was interrupted during parallelGet");
            } catch (ExecutionException e) {
                throw unwrapExecutionException(e, GET_ITEM);
            }
        }
    } finally {
        for (Future<GetItemResultWrapper> future : futures) {
            if (!future.isDone()) {
                future.cancel(interrupted /* mayInterruptIfRunning */);
            }
        }

        if (interrupted) {
            // set interrupted on this thread and fail out
            Thread.currentThread().interrupt();
        }
    }
    return results;
}

From source file:com.amazon.janusgraph.diskstorage.dynamodb.DynamoDBDelegate.java

public Map<StaticBuffer, GetItemResult> parallelGetItem(List<GetItemWorker> workers) throws BackendException {
    final CompletionService<GetItemResultWrapper> completionService = new ExecutorCompletionService<>(
            clientThreadPool);//from   ww w.j a va  2s .co m

    final List<Future<GetItemResultWrapper>> futures = Lists.newLinkedList();
    for (GetItemWorker worker : workers) {
        futures.add(completionService.submit(worker));
    }

    boolean interrupted = false;
    final Map<StaticBuffer, GetItemResult> results = Maps.newHashMap();
    try {
        for (int i = 0; i < workers.size(); i++) {
            try {
                GetItemResultWrapper result = completionService.take().get();
                results.put(result.getJanusGraphKey(), result.getDynamoDBResult());
            } catch (InterruptedException e) {
                interrupted = true;
                throw new BackendRuntimeException("was interrupted during parallelGet");
            } catch (ExecutionException e) {
                throw unwrapExecutionException(e, GET_ITEM);
            }
        }
    } finally {
        for (Future<GetItemResultWrapper> future : futures) {
            if (!future.isDone()) {
                future.cancel(interrupted /* mayInterruptIfRunning */);
            }
        }

        if (interrupted) {
            // set interrupted on this thread and fail out
            Thread.currentThread().interrupt();
        }
    }
    return results;
}