Example usage for java.util.concurrent ExecutorService isTerminated

List of usage examples for java.util.concurrent ExecutorService isTerminated

  1. HOME
  2. Java
  3. java
  4. java.util.concurrent.*
  5. ExecutorService
  6. isTerminated

Introduction

In this page you can find the example usage for java.util.concurrent ExecutorService isTerminated.

Prototype

boolean isTerminated();

Source Link

Document

Returns true if all tasks have completed following shut down.

Usage

From source file:com.linkedin.pinot.tools.perf.QueryRunner.java

/**
 * Use multiple threads to run query at an increasing target QPS.
 * <p>Use a concurrent linked queue to buffer the queries to be sent. Use the main thread to insert queries into the
 * queue at the target QPS, and start <code>numThreads</code> worker threads to fetch queries from the queue and send
 * them./*from w ww  . j a v  a2s.co m*/
 * <p>We start with the start QPS, and keep adding delta QPS to the start QPS during the test.
 * <p>The main thread is responsible for collecting and logging the statistic information periodically.
 * <p>Queries are picked sequentially from the query file.
 * <p>Query runner will stop when all queries in the query file has been executed number of times configured.
 *
 * @param conf perf benchmark driver config.
 * @param queryFile query file.
 * @param numTimesToRunQueries number of times to run all queries in the query file, 0 means infinite times.
 * @param numThreads number of threads sending queries.
 * @param startQPS start QPS.
 * @param deltaQPS delta QPS.
 * @param reportIntervalMs report interval in milliseconds.
 * @param numIntervalsToReportAndClearStatistics number of report intervals to report detailed statistics and clear
 *                                               them, 0 means never.
 * @param numIntervalsToIncreaseQPS number of intervals to increase QPS.
 * @throws Exception
 */

public static void increasingQPSQueryRunner(PerfBenchmarkDriverConf conf, String queryFile,
        int numTimesToRunQueries, int numThreads, double startQPS, double deltaQPS, int reportIntervalMs,
        int numIntervalsToReportAndClearStatistics, int numIntervalsToIncreaseQPS) throws Exception {
    List<String> queries;
    try (FileInputStream input = new FileInputStream(new File(queryFile))) {
        queries = IOUtils.readLines(input);
    }

    PerfBenchmarkDriver driver = new PerfBenchmarkDriver(conf);
    ConcurrentLinkedQueue<String> queryQueue = new ConcurrentLinkedQueue<>();
    AtomicInteger numQueriesExecuted = new AtomicInteger(0);
    AtomicLong totalBrokerTime = new AtomicLong(0L);
    AtomicLong totalClientTime = new AtomicLong(0L);
    List<Statistics> statisticsList = Collections.singletonList(new Statistics(CLIENT_TIME_STATISTICS));

    ExecutorService executorService = Executors.newFixedThreadPool(numThreads);
    for (int i = 0; i < numThreads; i++) {
        executorService.submit(new Worker(driver, queryQueue, numQueriesExecuted, totalBrokerTime,
                totalClientTime, statisticsList));
    }
    executorService.shutdown();

    long startTime = System.currentTimeMillis();
    long reportStartTime = startTime;
    int numReportIntervals = 0;
    int numTimesExecuted = 0;
    double currentQPS = startQPS;
    int queryIntervalMs = (int) (MILLIS_PER_SECOND / currentQPS);
    while (numTimesToRunQueries == 0 || numTimesExecuted < numTimesToRunQueries) {
        if (executorService.isTerminated()) {
            LOGGER.error("All threads got exception and already dead.");
            return;
        }

        for (String query : queries) {
            queryQueue.add(query);
            Thread.sleep(queryIntervalMs);

            long currentTime = System.currentTimeMillis();
            if (currentTime - reportStartTime >= reportIntervalMs) {
                long timePassed = currentTime - startTime;
                reportStartTime = currentTime;
                numReportIntervals++;

                if (numReportIntervals == numIntervalsToIncreaseQPS) {
                    // Try to find the next interval.
                    double newQPS = currentQPS + deltaQPS;
                    int newQueryIntervalMs;
                    // Skip the target QPS with the same interval as the previous one.
                    while ((newQueryIntervalMs = (int) (MILLIS_PER_SECOND / newQPS)) == queryIntervalMs) {
                        newQPS += deltaQPS;
                    }
                    if (newQueryIntervalMs == 0) {
                        LOGGER.warn("Due to sleep granularity of millisecond, cannot further increase QPS.");
                    } else {
                        // Find the next interval.
                        LOGGER.info(
                                "--------------------------------------------------------------------------------");
                        LOGGER.info("REPORT FOR TARGET QPS: {}", currentQPS);
                        int numQueriesExecutedInt = numQueriesExecuted.get();
                        LOGGER.info(
                                "Current Target QPS: {}, Time Passed: {}ms, Queries Executed: {}, Average QPS: {}, "
                                        + "Average Broker Time: {}ms, Average Client Time: {}ms, Queries Queued: {}.",
                                currentQPS, timePassed, numQueriesExecutedInt,
                                numQueriesExecutedInt / ((double) timePassed / MILLIS_PER_SECOND),
                                totalBrokerTime.get() / (double) numQueriesExecutedInt,
                                totalClientTime.get() / (double) numQueriesExecutedInt, queryQueue.size());
                        numReportIntervals = 0;
                        startTime = currentTime;
                        reportAndClearStatistics(numQueriesExecuted, totalBrokerTime, totalClientTime,
                                statisticsList);

                        currentQPS = newQPS;
                        queryIntervalMs = newQueryIntervalMs;
                        LOGGER.info(
                                "Increase target QPS to: {}, the following statistics are for the new target QPS.",
                                currentQPS);
                    }
                } else {
                    int numQueriesExecutedInt = numQueriesExecuted.get();
                    LOGGER.info(
                            "Current Target QPS: {}, Time Passed: {}ms, Queries Executed: {}, Average QPS: {}, "
                                    + "Average Broker Time: {}ms, Average Client Time: {}ms, Queries Queued: {}.",
                            currentQPS, timePassed, numQueriesExecutedInt,
                            numQueriesExecutedInt / ((double) timePassed / MILLIS_PER_SECOND),
                            totalBrokerTime.get() / (double) numQueriesExecutedInt,
                            totalClientTime.get() / (double) numQueriesExecutedInt, queryQueue.size());

                    if ((numIntervalsToReportAndClearStatistics != 0)
                            && (numReportIntervals % numIntervalsToReportAndClearStatistics == 0)) {
                        startTime = currentTime;
                        reportAndClearStatistics(numQueriesExecuted, totalBrokerTime, totalClientTime,
                                statisticsList);
                    }
                }
            }
        }
        numTimesExecuted++;
    }

    // Wait for all queries getting executed.
    while (queryQueue.size() != 0) {
        Thread.sleep(1);
    }
    executorService.shutdownNow();
    while (!executorService.isTerminated()) {
        Thread.sleep(1);
    }

    long timePassed = System.currentTimeMillis() - startTime;
    int numQueriesExecutedInt = numQueriesExecuted.get();
    LOGGER.info("--------------------------------------------------------------------------------");
    LOGGER.info("FINAL REPORT:");
    LOGGER.info(
            "Current Target QPS: {}, Time Passed: {}ms, Queries Executed: {}, Average QPS: {}, "
                    + "Average Broker Time: {}ms, Average Client Time: {}ms.",
            currentQPS, timePassed, numQueriesExecutedInt,
            numQueriesExecutedInt / ((double) timePassed / MILLIS_PER_SECOND),
            totalBrokerTime.get() / (double) numQueriesExecutedInt,
            totalClientTime.get() / (double) numQueriesExecutedInt);
    for (Statistics statistics : statisticsList) {
        statistics.report();
    }
}

From source file:org.rhq.metrics.simulator.Simulator.java

private void shutdown(ExecutorService service, String serviceName, int wait) {
    log.info("Shutting down " + serviceName);
    service.shutdown();/*from   ww  w .  jav  a 2  s .  co  m*/
    try {
        service.awaitTermination(wait, TimeUnit.SECONDS);
    } catch (InterruptedException e) {
    }
    if (!service.isTerminated()) {
        log.info("Forcing " + serviceName + " shutdown.");
        service.shutdownNow();
    }
    log.info(serviceName + " shut down complete");
}

From source file:org.yamj.core.service.ScanningScheduler.java

@Scheduled(initialDelay = 5000, fixedDelay = 45000)
public void scanMediaFiles() throws Exception {
    int maxThreads = configService.getIntProperty("yamj3.scheduler.mediafilescan.maxThreads", 1);
    if (maxThreads <= 0 || !mediaInfoService.isMediaInfoActivated()) {
        if (!messageDisabledMediaFiles) {
            messageDisabledMediaFiles = Boolean.TRUE;
            LOG.info("Media file scanning is disabled");
        }//w  ww.j a  v  a2s .  c o m
        return;
    } else {
        messageDisabledMediaFiles = Boolean.FALSE;
    }

    int maxResults = configService.getIntProperty("yamj3.scheduler.mediafilescan.maxResults", 20);
    List<QueueDTO> queueElements = mediaStorageService.getMediaFileQueueForScanning(maxResults);
    if (CollectionUtils.isEmpty(queueElements)) {
        LOG.debug("No media files found to scan");
        return;
    }

    LOG.info("Found {} media files to process; scan with {} threads", queueElements.size(), maxThreads);
    BlockingQueue<QueueDTO> queue = new LinkedBlockingQueue<QueueDTO>(queueElements);

    ExecutorService executor = Executors.newFixedThreadPool(maxThreads);
    for (int i = 0; i < maxThreads; i++) {
        MediaInfoRunner worker = new MediaInfoRunner(queue, mediaInfoService);
        executor.execute(worker);
    }
    executor.shutdown();

    // run until all workers have finished
    while (!executor.isTerminated()) {
        try {
            TimeUnit.SECONDS.sleep(5);
        } catch (InterruptedException ignore) {
        }
    }

    LOG.debug("Finished media file scanning");
}

From source file:org.yamj.core.service.ScanningScheduler.java

@Scheduled(initialDelay = 5000, fixedDelay = 45000)
public void scanMediaData() throws Exception {
    int maxThreads = configService.getIntProperty("yamj3.scheduler.mediadatascan.maxThreads", 1);
    if (maxThreads <= 0) {
        if (!messageDisabledMediaData) {
            messageDisabledMediaData = Boolean.TRUE;
            LOG.info("Media data scanning is disabled");
        }/*from  w  w w.  ja v a2s.co m*/
        return;
    } else {
        messageDisabledMediaData = Boolean.FALSE;
    }

    int maxResults = configService.getIntProperty("yamj3.scheduler.mediadatascan.maxResults", 20);
    List<QueueDTO> queueElements = metadataStorageService.getMediaQueueForScanning(maxResults);
    if (CollectionUtils.isEmpty(queueElements)) {
        LOG.debug("No media data found to scan");
        return;
    }

    LOG.info("Found {} media data objects to process; scan with {} threads", queueElements.size(), maxThreads);
    BlockingQueue<QueueDTO> queue = new LinkedBlockingQueue<QueueDTO>(queueElements);

    ExecutorService executor = Executors.newFixedThreadPool(maxThreads);
    for (int i = 0; i < maxThreads; i++) {
        PluginMetadataRunner worker = new PluginMetadataRunner(queue, pluginMetadataService);
        executor.execute(worker);
    }
    executor.shutdown();

    // run until all workers have finished
    while (!executor.isTerminated()) {
        try {
            TimeUnit.SECONDS.sleep(5);
        } catch (InterruptedException ignore) {
        }
    }

    LOG.debug("Finished media data scanning");
}

From source file:org.yamj.core.service.ScanningScheduler.java

@Scheduled(initialDelay = 15000, fixedDelay = 45000)
public void scanArtwork() throws Exception {
    int maxThreads = configService.getIntProperty("yamj3.scheduler.artworkscan.maxThreads", 1);
    if (maxThreads <= 0) {
        if (!messageDisabledArtwork) {
            messageDisabledArtwork = Boolean.TRUE;
            LOG.info("Artwork scanning is disabled");
        }//from   ww  w  .j  a va 2  s .com
        return;
    } else {
        messageDisabledArtwork = Boolean.FALSE;
    }

    int maxResults = configService.getIntProperty("yamj3.scheduler.artworkscan.maxResults", 30);
    List<QueueDTO> queueElements = artworkStorageService.getArtworkQueueForScanning(maxResults);
    if (CollectionUtils.isEmpty(queueElements)) {
        LOG.debug("No artwork found to scan");
        return;
    }

    LOG.info("Found {} artwork objects to process; scan with {} threads", queueElements.size(), maxThreads);
    BlockingQueue<QueueDTO> queue = new LinkedBlockingQueue<QueueDTO>(queueElements);

    ExecutorService executor = Executors.newFixedThreadPool(maxThreads);
    for (int i = 0; i < maxThreads; i++) {
        ArtworkScannerRunner worker = new ArtworkScannerRunner(queue, artworkScannerService);
        executor.execute(worker);
    }
    executor.shutdown();

    // run until all workers have finished
    while (!executor.isTerminated()) {
        try {
            TimeUnit.SECONDS.sleep(5);
        } catch (InterruptedException ignore) {
        }
    }

    LOG.debug("Finished artwork scanning");
}

From source file:org.yamj.core.service.ScanningScheduler.java

@Scheduled(initialDelay = 10000, fixedDelay = 45000)
public void scanPeopleData() throws Exception {
    int maxThreads = configService.getIntProperty("yamj3.scheduler.peoplescan.maxThreads", 1);
    if (maxThreads <= 0) {
        if (!messageDisabledPeople) {
            messageDisabledPeople = Boolean.TRUE;
            LOG.info("People scanning is disabled");
        }//from  w  w  w . ja  v a  2 s . c  o  m
        return;
    } else {
        messageDisabledPeople = Boolean.FALSE;
    }

    int maxResults = configService.getIntProperty("yamj3.scheduler.peoplescan.maxResults", 50);
    List<QueueDTO> queueElements = metadataStorageService.getPersonQueueForScanning(maxResults);
    if (CollectionUtils.isEmpty(queueElements)) {
        LOG.debug("No people data found to scan");
        return;
    }

    LOG.info("Found {} people objects to process; scan with {} threads", queueElements.size(), maxThreads);
    BlockingQueue<QueueDTO> queue = new LinkedBlockingQueue<QueueDTO>(queueElements);

    ExecutorService executor = Executors.newFixedThreadPool(maxThreads);
    for (int i = 0; i < maxThreads; i++) {
        PluginMetadataRunner worker = new PluginMetadataRunner(queue, pluginMetadataService);
        executor.execute(worker);
    }
    executor.shutdown();

    // run until all workers have finished
    while (!executor.isTerminated()) {
        try {
            TimeUnit.SECONDS.sleep(5);
        } catch (InterruptedException ignore) {
        }

    }

    LOG.debug("Finished people data scanning");
}

From source file:com.amour.imagecrawler.ImagesManager.java

/**
 * Run crawler operation in multi-thread
 * @param propertiesManager The Properties-Manager
 * @throws IOException /*  ww  w. j ava 2  s. c o  m*/
 * @throws java.security.NoSuchAlgorithmException 
 */
public void run(Properties propertiesManager) throws IOException, NoSuchAlgorithmException, Exception {

    ExecutorService executor = Executors.newFixedThreadPool(
            Integer.parseInt(propertiesManager.getProperty(Crawler.NUMBER_OF_WORKER_THREADS_KEY)));
    for (String imageUrl : this.imagesList) {

        Runnable worker = new ImageRunable(imageUrl, propertiesManager);
        executor.execute(worker);
    }
    executor.shutdown();
    while (!executor.isTerminated()) {
    }
}

From source file:it.geosolutions.tools.io.file.Copy.java

/**
 * Copy a list of files asynchronously to a destination (which can be on
 * nfs) each thread wait (at least) 'seconds' seconds for each file
 * propagation./*from   ww  w . jav a  2 s.  c  o  m*/
 * 
 * @param ex
 *            the thread pool executor
 * @param baseDestDir
 * @param overwrite
 *            if false and destination exists() do not overwrite the file
 * @param seconds
 * @return the resulting moved file list or null
 * @note this function could not use overwrite boolean flag to avoid file
 *       lock on the same section when 2 thread are called to write the same
 *       file name
 */
public static List<FutureTask<File>> asynchCopyListFileToNFS(final ExecutorService ex, final List<File> list,
        final File baseDestDir, final int seconds) {
    // list
    if (list == null) {
        if (LOGGER.isErrorEnabled())
            LOGGER.error("Failed to copy file list using a NULL list");
        return null;
    }
    final int size = list.size();
    if (size == 0) {
        if (LOGGER.isErrorEnabled())
            LOGGER.error("Failed to copy file list using an empty list");
        return null;
    }
    // baseDestDir
    if (baseDestDir == null) {
        if (LOGGER.isErrorEnabled())
            LOGGER.error("Failed to copy file list using a NULL baseDestDir");
        return null;
    } else if (!baseDestDir.isDirectory() || !baseDestDir.canWrite()) {
        if (LOGGER.isErrorEnabled())
            LOGGER.error("Failed to copy file list using a not " + "writeable directory as baseDestDir: "
                    + baseDestDir.getAbsolutePath());
        return null;
    }
    // Asynch executor check
    if (ex == null || ex.isTerminated()) {
        if (LOGGER.isErrorEnabled())
            LOGGER.error("Unable to run asynchronously using a terminated or null ThreadPoolExecutor");
        return null;
    }

    final List<FutureTask<File>> asyncRes = new ArrayList<FutureTask<File>>(size);
    for (File file : list) {
        if (file != null) {
            if (file.exists()) {
                try {
                    asyncRes.add(asynchFileCopyToNFS(ex, file, new File(baseDestDir, file.getName()), seconds));
                } catch (RejectedExecutionException e) {
                    if (LOGGER.isWarnEnabled())
                        LOGGER.warn("SKIPPING file:\n" + file.getAbsolutePath() + ".\nError: "
                                + e.getLocalizedMessage());
                } catch (IllegalArgumentException e) {
                    if (LOGGER.isWarnEnabled())
                        LOGGER.warn("SKIPPING file:\n" + file.getAbsolutePath() + ".\nError: "
                                + e.getLocalizedMessage());
                }

            } else {
                if (LOGGER.isWarnEnabled())
                    LOGGER.warn("SKIPPING file:\n" + file.getAbsolutePath()
                            + "\nUnable to copy a not existent file.");
            }
        }
    }

    return asyncRes;
}

From source file:ke.co.binary.app.restoppress.restoppress.Service.ImplGenRequests.java

@Override
public void sendRequests(int number, String url, Reply json_obj, String method) {

    long starttime = System.currentTimeMillis();
    ExecutorService executor = Executors.newCachedThreadPool();
    for (int i = 0; i < number; i++) {
        //generate random ref code and order info
        json_obj.getMessage().setReferenceNo(generateRandom());
        json_obj.getMessage().setOrderInfo(generateRandom());
        //System.out.println("Request Method: "+json_obj.getMessage());
        Runnable worker = new ProcessRequest(url, method, json_obj);
        executor.execute(worker);/*  w w  w  . ja v a 2  s  .c o m*/
    }
    executor.shutdown();
    //wait for all the threads to terminate
    while (!executor.isTerminated()) {
    }
    long finishtime = System.currentTimeMillis();
    time_taken = (finishtime - starttime);

}

From source file:xbird.xquery.expr.ext.BDQExpr.java

private static Sequence invokeRequestsInParallel(final List<String> endpoints, final BindingVariable hostVar,
        final PreparedQueryRequest request, final DynamicContext dynEnv) {
    final int numEndpoints = endpoints.size();
    final ExecutorService ex = ExecutorFactory.newFixedThreadPool(numEndpoints, "parallelRemoteExec");
    final Sequence[] results = new Sequence[numEndpoints];
    //final AtomicReferenceArray<Sequence> resultsRef = new AtomicReferenceArray<Sequence>(results);
    try {/*w  w w . java 2 s. c om*/
        for (int i = 0; i < numEndpoints; i++) {
            final String endpoint = endpoints.get(i);
            final int at = i;
            Runnable r = new Runnable() {
                public void run() {
                    final Sequence res;
                    try {
                        res = invokeRequest(endpoint, request);
                    } catch (XQueryException e) {
                        throw new IllegalStateException("An error caused while evaluating CompiledQueryRequest#"
                                + request.getIdentifier() + " at " + endpoint, e);
                    }
                    results[at] = res;
                }
            };
            ex.execute(r);
        }
    } finally {
        ExecutorUtils.shutdownAndAwaitTermination(ex);
    }
    assert (ex.isTerminated());
    return new PipelinedSequence(dynEnv, results);
}