Example usage for java.util.concurrent ExecutorService invokeAll

List of usage examples for java.util.concurrent ExecutorService invokeAll

  1. HOME
  2. Java
  3. java
  4. java.util.concurrent.*
  5. ExecutorService
  6. invokeAll

Introduction

In this page you can find the example usage for java.util.concurrent ExecutorService invokeAll.

Prototype

<T> List<Future<T>> invokeAll(Collection<? extends Callable<T>> tasks) throws InterruptedException;

Source Link

Document

Executes the given tasks, returning a list of Futures holding their status and results when all complete.

Usage

From source file:edu.cmu.cs.lti.ark.fn.identification.training.AlphabetCreationThreaded.java

/**
 * Splits frameElementLines into numThreads equally-sized batches and creates an alphabet
 * file for each one./*from  ww w . java 2  s  . com*/
 *
 * @throws IOException
 */
public Multiset<String> createAlphabet() throws IOException, ExecutionException, InterruptedException {
    final List<String> frameLines = Files.readLines(new File(frameElementsFile), Charsets.UTF_8)
            .subList(startIndex, endIndex);
    final int batchSize = (int) Math.ceil(frameLines.size() / (double) numThreads);
    final List<List<String>> frameLinesPartition = Lists.partition(frameLines, batchSize);
    final List<String> parseLines = Files.readLines(new File(parseFile), Charsets.UTF_8);
    final Multiset<String> alphabet = ConcurrentHashMultiset.create();
    final List<Callable<Integer>> jobs = Lists.newArrayListWithExpectedSize(numThreads);
    for (final int i : xrange(numThreads)) {
        jobs.add(newJob(i, frameLinesPartition.get(i), parseLines, alphabet));
    }
    final ExecutorService threadPool = Executors.newFixedThreadPool(numThreads);
    final List<Future<Integer>> results = threadPool.invokeAll(jobs);
    threadPool.shutdown();
    try {
        for (Integer i : xrange(results.size())) {
            logger.info(String.format("Thread %d successfully processed %d lines", i, results.get(i).get()));
        }
    } finally {
        threadPool.shutdownNow();
    }
    return alphabet;
}

From source file:io.ecarf.core.cloud.task.processor.files.ProcessFilesTask.java

@Override
public void run() throws IOException {

    log.info("START: processing files");

    Stopwatch stopwatch = Stopwatch.createStarted();

    Set<String> filesSet = ObjectUtils.csvToSet(files);
    log.info("Processing files: " + filesSet);

    List<Callable<T>> tasks = getSubTasks(filesSet);

    int processors = Runtime.getRuntime().availableProcessors();

    try {/*from  ww w  . jav a 2s  . c o  m*/

        // check if we only have one file to process
        if (tasks.size() == 1) {

            this.processSingleOutput(tasks.get(0).call());

        } else if (processors == 1) {
            // only one process then process synchronously
            List<T> output = new ArrayList<>();
            for (Callable<T> task : tasks) {
                output.add(task.call());
            }

            this.processMultiOutput(output);

        } else {

            // multiple cores
            ExecutorService executor = Utils.createFixedThreadPool(processors);

            try {

                List<Future<T>> results = executor.invokeAll(tasks);
                List<T> output = new ArrayList<>();

                for (Future<T> result : results) {
                    output.add(result.get());
                }

                this.processMultiOutput(output);

            } finally {
                executor.shutdown();
            }
        }

    } catch (Exception e) {
        log.error("Failed to process multiple files", e);
        throw new IOException(e);

    }

    log.info("TIMER# All files are processed successfully, elapsed time: " + stopwatch);
}

From source file:com.ibm.bi.dml.runtime.matrix.data.LibMatrixDatagen.java

/**
 * Function to generate a matrix of random numbers. This is invoked both
 * from CP as well as from MR. In case of CP, it generates an entire matrix
 * block-by-block. A <code>bigrand</code> is passed so that block-level
 * seeds are generated internally. In case of MR, it generates a single
 * block for given block-level seed <code>bSeed</code>.
 * /*from ww  w  .  ja  va2 s  . c o m*/
 * When pdf="uniform", cell values are drawn from uniform distribution in
 * range <code>[min,max]</code>.
 * 
 * When pdf="normal", cell values are drawn from standard normal
 * distribution N(0,1). The range of generated values will always be
 * (-Inf,+Inf).
 * 
 * @param rows
 * @param cols
 * @param rowsInBlock
 * @param colsInBlock
 * @param sparsity
 * @param min
 * @param max
 * @param bigrand
 * @param bSeed
 * @return
 * @throws DMLRuntimeException
 */
public static void generateRandomMatrix(MatrixBlock out, RandomMatrixGenerator rgen, long[] nnzInBlocks,
        Well1024a bigrand, long bSeed, int k) throws DMLRuntimeException {
    int rows = rgen._rows;
    int cols = rgen._cols;
    int rpb = rgen._rowsPerBlock;
    int cpb = rgen._colsPerBlock;
    double sparsity = rgen._sparsity;

    if (rows == 1) {
        generateRandomMatrix(out, rgen, nnzInBlocks, bigrand, bSeed);
        return;
    }

    boolean invokedFromCP = true;
    if (bigrand == null && nnzInBlocks != null)
        invokedFromCP = false;

    // sanity check valid dimensions and sparsity
    checkMatrixDimensionsAndSparsity(rows, cols, sparsity);

    /*
     * Setup min and max for distributions other than "uniform". Min and Max
     * are set up in such a way that the usual logic of
     * (max-min)*prng.nextDouble() is still valid. This is done primarily to
     * share the same code across different distributions.
     */
    double min = 0, max = 1;
    if (rgen._pdf.equalsIgnoreCase(RAND_PDF_UNIFORM)) {
        min = rgen._min;
        max = rgen._max;
    }

    // Determine the sparsity of output matrix
    // if invoked from CP: estimated NNZ is for entire matrix (nnz=0, if 0 initialized)
    // if invoked from MR: estimated NNZ is for one block
    final long estnnz = (invokedFromCP ? ((min == 0.0 && max == 0.0) ? 0 : (long) (sparsity * rows * cols))
            : nnzInBlocks[0]);
    boolean lsparse = MatrixBlock.evalSparseFormatInMemory(rows, cols, estnnz);
    out.reset(rows, cols, lsparse);

    // Special case shortcuts for efficiency
    if (rgen._pdf.equalsIgnoreCase(RAND_PDF_UNIFORM)) {
        //specific cases for efficiency
        if (min == 0.0 && max == 0.0) { //all zeros
            // nothing to do here
            out.nonZeros = 0;
            return;
        } else if (!out.sparse && sparsity == 1.0d && min == max) //equal values
        {
            out.init(min, out.rlen, out.clen);
            return;
        }
    }

    // Allocate memory
    if (out.sparse) {
        //note: individual sparse rows are allocated on demand,
        //for consistency with memory estimates and prevent OOMs.
        out.allocateSparseRowsBlock();
    } else {
        out.allocateDenseBlock();
    }

    int nrb = (int) Math.ceil((double) rows / rpb);
    int ncb = (int) Math.ceil((double) cols / cpb);

    try {
        ExecutorService pool = Executors.newFixedThreadPool(k);
        ArrayList<RandTask> tasks = new ArrayList<RandTask>();
        int blklen = ((int) (Math.ceil((double) nrb / k)));
        for (int i = 0; i < k & i * blklen < nrb; i++) {
            long[] seeds = generateSeedsForCP(bigrand, blklen, ncb);
            tasks.add(new RandTask(invokedFromCP, i * blklen, Math.min((i + 1) * blklen, nrb), out, rgen,
                    nnzInBlocks, bSeed, seeds));
        }
        pool.invokeAll(tasks);
        pool.shutdown();

        //early error notify in case not all tasks successful
        for (RandTask rt : tasks)
            if (!rt.getReturnCode())
                throw new DMLRuntimeException("RandGen task failed: " + rt.getErrMsg());
    } catch (Exception e) {
        throw new DMLRuntimeException(e);
    }

    out.recomputeNonZeros();
}

From source file:com.lumata.lib.lupa.extractor.internal.HtmlBiggestImageExtractor.java

@Override
public Image extractBestImage(URL sourceUrl, Elements htmlSection, ImageExtractionRequirements requirements) {
    Map<String, Image> imagesToExplore = new HashMap<String, Image>();
    Set<ImageDownloadTask> imagesToDownload = new HashSet<ImageDownloadTask>();
    Iterator<org.jsoup.nodes.Element> it = htmlSection.iterator();

    // collect valid images
    while (it.hasNext() && imagesToExplore.size() < requirements.getMaxImagesToExplore()) {
        Element imageElement = it.next();
        String imageUrl = imageElement.absUrl("src");

        // Do not process empty img tags, duplicated images or tracking
        // pixels and other assorted ads
        if (imageUrl == null || imagesToExplore.containsKey(imageUrl) || isTrackingPixelOrAd(imageUrl)) {
            continue;
        }//from   w  w  w. j av a 2 s. co m

        // remember this image
        Image imageContent = new Image(imageUrl);
        if (imageElement.hasAttr(WIDTH_ATTRIBUTE)) {
            // TODO: We need to convert other picture size units supported by html (there must be a lib for this)
            imageContent.setWidth(Integer.parseInt(imageElement.attr(WIDTH_ATTRIBUTE).replace("px", "")));
        }
        if (imageElement.hasAttr(HEIGHT_ATTRIBUTE)) {
            imageContent.setHeight(Integer.parseInt(imageElement.attr(HEIGHT_ATTRIBUTE).replace("px", "")));
        }
        if (imageContent.getWidth() == null || imageContent.getHeight() == null) {// mark image to download
            imagesToDownload.add(new ImageDownloadTask(imageContent));
        }
        imagesToExplore.put(imageUrl, imageContent);
    }

    // if dimensions are empty -> download image
    if (CollectionUtils.isNotEmpty(imagesToDownload)) {
        try {
            ExecutorService pool = Executors.newFixedThreadPool(imagesToDownload.size(),
                    getThreadFactory(sourceUrl));
            pool.invokeAll(imagesToDownload);
            pool.shutdown();
        } catch (InterruptedException e) {
            LOG.error("InterruptedException while downloading images", e);
        }
    }

    // select biggest image
    Image biggestImage = null;
    try {
        biggestImage = Collections.max(imagesToExplore.values(), new Comparator<Image>() {
            @Override
            public int compare(Image o1, Image o2) {
                return getSquarePixels(o1) - getSquarePixels(o2);
            }
        });
    } catch (NoSuchElementException e) {
        return null;
    }

    // if image is too small, discard
    return (biggestImage.getWidth() < requirements.getMinImageSize()
            || biggestImage.getHeight() < requirements.getMinImageSize()) ? null : biggestImage;
}

From source file:org.pentaho.di.core.database.ConnectionPoolUtilIntegrationIT.java

private void testGetConnectionFromPool(final int threadCount) throws Exception {
    ArgumentCaptor<String> captorLogMessage = ArgumentCaptor.forClass(String.class);

    final DatabaseMeta dbMeta = mock(DatabaseMeta.class, RETURNS_MOCKS);
    when(dbMeta.getDriverClass()).thenReturn(driver.getClass().getCanonicalName());
    when(dbMeta.getConnectionPoolingProperties()).thenReturn(new Properties());
    when(dbMeta.environmentSubstitute(anyString())).thenAnswer(new Answer<Object>() {
        @Override/*from ww  w. j  a v  a  2 s .c o m*/
        public Object answer(InvocationOnMock invocation) throws Throwable {
            return invocation.getArguments()[0];
        }
    });
    when(dbMeta.getName()).thenReturn("CP1");
    when(dbMeta.getPassword()).thenReturn(PASSWORD);

    Callable<Connection> task = new Callable<Connection>() {
        @Override
        public Connection call() throws Exception {
            return ConnectionPoolUtil.getConnection(logChannelInterface, dbMeta, "", INITIAL_POOL_SIZE,
                    threadCount);
        }
    };
    List<Callable<Connection>> tasks = Collections.nCopies(threadCount, task);
    ExecutorService executorService = Executors.newFixedThreadPool(threadCount);
    List<Future<Connection>> futures = executorService.invokeAll(tasks);

    assertNotNull(futures);
    assertEquals(threadCount, futures.size());

    //pool should be creates only once for KettleClientEnvironment
    verify(logChannelInterface, atMost(2)).logBasic(captorLogMessage.capture());
    List<String> capturedLogEntry = captorLogMessage.getAllValues();
    if (capturedLogEntry != null && !capturedLogEntry.isEmpty()) {
        assertEquals(BaseMessages.getString(PKG, "Database.CreatingConnectionPool", dbMeta.getName()),
                capturedLogEntry.get(0));
        assertEquals(BaseMessages.getString(PKG, "Database.CreatedConnectionPool", dbMeta.getName()),
                capturedLogEntry.get(1));
    }
}

From source file:com.aliyun.odps.ship.download.DshipDownload.java

private void multiThreadDownload() throws TunnelException {
    ArrayList<Callable<Long>> callList = new ArrayList<Callable<Long>>();
    for (final FileDownloader downloader : workItems) {
        Callable<Long> call = new Callable<Long>() {
            @Override/*w  w  w  .  java  2s  .  c om*/
            public Long call() throws Exception {
                downloader.download();
                return downloader.getWrittenBytes();
            }
        };
        callList.add(call);
    }

    ExecutorService executors = Executors.newFixedThreadPool(threads);
    try {
        List<Future<Long>> futures = executors.invokeAll(callList);
        ArrayList<String> failedThread = new ArrayList<String>();
        for (int i = 0; i < futures.size(); ++i) {
            try {
                writtenBytes += futures.get(i).get();
            } catch (ExecutionException e) {
                e.printStackTrace();
                failedThread.add(String.valueOf(i));
            }
        }
        if (!failedThread.isEmpty()) {
            throw new TunnelException("Slice ID:" + StringUtils.join(failedThread, ",") + " Failed.");
        }
    } catch (InterruptedException e) {
        throw new UserInterruptException(e.getMessage());
    }
}

From source file:org.silverpeas.migration.publication.HtmlUnescaper.java

public void unescapeHtml() throws Exception {
    ExecutorService executor = Executors.newFixedThreadPool(2);
    // use futures to block at the tasks termination to ensure no other sql executions are
    // interleaved within the execution of these tasks.
    List<Future<Void>> futures = executor.invokeAll(Arrays.asList(unescapePublications(), unescapeComments()));
    for (Future<Void> future : futures) {
        future.get();/*from  w ww .ja v a2  s.  com*/
    }
    executor.shutdown();
}

From source file:io.ecarf.core.cloud.task.processor.old.ProcessLoadTask1Old.java

@Override
public void run() throws IOException {

    log.info("START: processing files for bigquery import");

    EcarfGoogleCloudService cloudService = (EcarfGoogleCloudService) this.getCloudService();

    //String bucket = metadata.getBucket();

    // get the schema terms if provided
    //String schemaTermsFile = metadata.getSchemaTermsFile();
    Set<String> schemaTerms = cloudService.getSetFromCloudStorageFile(schemaTermsFile, bucket);

    //Set<String> files = metadata.getFiles();
    Set<String> filesSet = ObjectUtils.csvToSet(files);
    log.info("Loading files: " + filesSet);
    Map<String, Integer> count = new HashMap<>();
    List<ProcessFilesForBigQuerySubTask> tasks = new ArrayList<>();
    int processors = Runtime.getRuntime().availableProcessors();

    for (final String file : filesSet) {

        TermCounter counter = null;/*from  w w w  .  ja  v a 2  s . co m*/

        if (schemaTerms != null) {
            counter = new TermCounter();
            counter.setTermsToCount(schemaTerms);
        }

        ProcessFilesForBigQuerySubTask task = new ProcessFilesForBigQuerySubTask(file, bucket, bucket, counter,
                null, false, false, this.getCloudService());
        tasks.add(task);

    }

    // check if we only have one file to process
    if (tasks.size() == 1) {

        TermCounter counter = tasks.get(0).call();
        if (counter != null) {
            count = counter.getCount();
        }

    } else if (processors == 1) {
        // only one process then process synchronously
        for (ProcessFilesForBigQuerySubTask task : tasks) {
            TermCounter counter = task.call();
            if (counter != null) {
                Utils.mergeCountMaps(count, counter.getCount());
            }
        }

    } else {

        // multiple cores
        ExecutorService executor = Utils.createFixedThreadPool(processors);

        try {

            List<Future<TermCounter>> results = executor.invokeAll(tasks);

            for (Future<TermCounter> result : results) {
                TermCounter counter = result.get();
                if (counter != null) {
                    Utils.mergeCountMaps(count, counter.getCount());
                }
            }

        } catch (Exception e) {
            log.error("Failed to process multiple files", e);
            throw new IOException(e);

        } finally {
            executor.shutdown();
        }
    }
    // write term stats to file and upload
    if ((count != null) && !count.isEmpty()) {
        log.info("Saving terms stats");
        String countStatsFile = Utils.TEMP_FOLDER + cloudService.getInstanceId() + Constants.DOT_JSON;
        FileUtils.objectToJsonFile(countStatsFile, count);

        cloudService.uploadFileToCloudStorage(countStatsFile, bucket);
    }

    log.info("FINISH: All files are processed and uploaded successfully");
}

From source file:de.tbuchloh.kiskis.cracker.PasswordCracker.java

public String crackPassword() {
    final Long totalEstimation = _passwordCreator.estimateTotalCount();
    _progressListener.notifyTotalCount(totalEstimation);
    _progressListener.notifyStartTime(System.currentTimeMillis());

    final AtomicBoolean found = new AtomicBoolean(false);
    final Callable<String> callable = new Callable<String>() {

        @Override// w w  w.  ja  v a2  s . c  o  m
        public String call() throws Exception {
            String guess;
            while (!found.get() && (guess = _passwordCreator.create()) != null) {
                _progressListener.notifyTry(guess);
                if (_tester.test(guess)) {
                    found.set(true);
                    return guess;
                }
            }
            return null;
        }
    };

    final int cpus = ManagementFactory.getOperatingSystemMXBean().getAvailableProcessors();
    LOG.info(String.format("Found %1$d cpus ...", cpus));
    final ExecutorService threadPool = Executors.newFixedThreadPool(cpus);
    final Collection<Callable<String>> tasks = new ArrayList<Callable<String>>();
    for (int i = 0; i < cpus; ++i) {
        tasks.add(callable);
    }
    try {
        final List<Future<String>> futures = threadPool.invokeAll(tasks);
        for (final Future<String> future : futures) {
            final String guessedPwd = future.get();
            if (guessedPwd != null) {
                return guessedPwd;
            }
        }
        return null;
    } catch (final InterruptedException e) {
        throw new KisKisRuntimeException("InterruptedException", e);
    } catch (final ExecutionException e) {
        throw new KisKisRuntimeException("ExecutionException", e);
    }
}

From source file:com.intuit.karate.cucumber.CucumberRunner.java

public static KarateStats parallel(Class clazz, int threadCount, String reportDir) {
    KarateStats stats = KarateStats.startTimer();
    ExecutorService executor = Executors.newFixedThreadPool(threadCount);
    CucumberRunner runner = new CucumberRunner(clazz);
    List<FeatureFile> featureFiles = runner.getFeatureFiles();
    List<Callable<KarateJunitFormatter>> callables = new ArrayList<>(featureFiles.size());
    int count = featureFiles.size();
    for (int i = 0; i < count; i++) {
        int index = i + 1;
        FeatureFile featureFile = featureFiles.get(i);
        callables.add(() -> {//w w  w. j a va2s  .co  m
            String threadName = Thread.currentThread().getName();
            KarateJunitFormatter formatter = getFormatter(reportDir, featureFile);
            logger.info(">>>> feature {} of {} on thread {}: {}", index, count, threadName,
                    featureFile.feature.getPath());
            runner.run(featureFile, formatter);
            logger.info("<<<< feature {} of {} on thread {}: {}", index, count, threadName,
                    featureFile.feature.getPath());
            formatter.done();
            return formatter;
        });
    }
    try {
        List<Future<KarateJunitFormatter>> futures = executor.invokeAll(callables);
        stats.stopTimer();
        for (Future<KarateJunitFormatter> future : futures) {
            KarateJunitFormatter formatter = future.get();
            stats.addToTestCount(formatter.getTestCount());
            stats.addToFailCount(formatter.getFailCount());
            stats.addToSkipCount(formatter.getSkipCount());
            stats.addToTimeTaken(formatter.getTimeTaken());
            if (formatter.isFail()) {
                stats.addToFailedList(formatter.getFeaturePath());
            }
        }
        stats.printStats(threadCount);
        return stats;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}