Example usage for java.util.concurrent ExecutorCompletionService ExecutorCompletionService

List of usage examples for java.util.concurrent ExecutorCompletionService ExecutorCompletionService

Introduction

In this page you can find the example usage for java.util.concurrent ExecutorCompletionService ExecutorCompletionService.

Prototype

public ExecutorCompletionService(Executor executor) 

Source Link

Document

Creates an ExecutorCompletionService using the supplied executor for base task execution and a LinkedBlockingQueue as a completion queue.

Usage

From source file:com.siva.javamultithreading.MultiThreadExecutor.java

/**
 * This is sample.//from   w  w  w.  j  a va 2  s  .com
 */
private static void startProcess() {

    ExecutorService threadPool = Executors.newFixedThreadPool(4);
    CompletionService<HSSFWorkbook> pool = new ExecutorCompletionService<>(threadPool);
    // Excel creation through multiple threads
    long startTime = System.currentTimeMillis();
    pool.submit(new ExcelChunkSheetWriter(0, 1000));
    pool.submit(new ExcelChunkSheetWriter(1001, 20000));
    pool.submit(new ExcelChunkSheetWriter(2, 3000));
    pool.submit(new ExcelChunkSheetWriter(3, 40000));
    pool.submit(new ExcelChunkSheetWriter(4, 50000));

    HSSFWorkbook hSSFWorkbook = null;
    HSSFWorkbook book = new HSSFWorkbook();
    HSSFSheet sheet = book.createSheet("Report");

    try {
        for (int i = 0; i < 5; i++) {
            hSSFWorkbook = pool.take().get();
            System.out.println(
                    "sheet row count : sheet.PhysicalNumberOfRows() = " + sheet.getPhysicalNumberOfRows());
            int currentCount = sheet.getPhysicalNumberOfRows();
            int incomingCount = hSSFWorkbook.getSheetAt(0).getPhysicalNumberOfRows();
            if ((currentCount + incomingCount) > 60000) {
                sheet = book.createSheet("Report" + i);
            }
            ExcelUtil.copySheets(book, sheet, hSSFWorkbook.getSheetAt(0));
        }
    } catch (InterruptedException ex) {
        Logger.getLogger(MultiThreadExecutor.class.getName()).log(Level.SEVERE, null, ex);
    } catch (ExecutionException ex) {
        Logger.getLogger(MultiThreadExecutor.class.getName()).log(Level.SEVERE, null, ex);
    }

    try {
        writeFile(book, new FileOutputStream("Report.xls"));
    } catch (Exception e) {
        e.printStackTrace();
    }

    /*
     FileOutputStream fos = new FileOutputStream("all.zip");
     ZipOutputStream zos = new ZipOutputStream(fos);
     for (int i = 0; i < 5; i++) {
     try {
     hSSFWorkbook = pool.take().get();                
     ZipEntry ze = new ZipEntry("Excel" + i + ".xls");
     zos.putNextEntry(ze);
     hSSFWorkbook.write(zos);
     zos.closeEntry();
     } catch (InterruptedException ex) {
     Logger.getLogger(MultiThreadExecutor.class.getName()).log(Level.SEVERE, null, ex);
     }           
     }
     zos.close();
     */
    long endTime = System.currentTimeMillis();
    System.out.println("Time taken: " + (endTime - startTime) + " ms");
    threadPool.shutdown();
}

From source file:org.apache.hadoop.chukwa.dataloader.FSMDataLoader.java

public void load(ChukwaConfiguration conf, FileSystem fs, FileStatus[] fileList) throws IOException {

    if (executor == null) {
        try {/*www  . j  ava  2  s.co  m*/
            this.size = Integer.parseInt(conf.get(DATA_LOADER_THREAD_LIMIT));
        } catch (Exception e) {
            this.size = 1;
        }
        executor = Executors.newFixedThreadPool(size);
    }
    if (completion == null) {
        completion = new ExecutorCompletionService(executor);
    }

    try {
        // Locate directory output directories of the current demux, and create a unique directory list.
        HashSet<Path> inputPaths = new HashSet<Path>();
        HashSet<Path> outputPaths = new HashSet<Path>();
        int counter = 0;
        for (int i = 0; i < fileList.length; i++) {
            Path temp = fileList[i].getPath().getParent();
            if (!inputPaths.contains(temp)) {
                inputPaths.add(temp);
            }
        }
        String outputDir = conf.get("chukwa.tmp.data.dir") + File.separator + "fsm_"
                + System.currentTimeMillis() + "_";
        if (inputPaths.size() > 0) {
            Configuration fsmConf = new Configuration();
            // Run fsm map reduce job for dn, tt, and jobhist.
            for (String mapper : mappers) {
                String[] args = new String[inputPaths.size() + 3];
                args[0] = "-in";
                int k = 2;
                boolean hasData = false;
                for (Path temp : inputPaths) {
                    String tempPath = temp.toUri().toString();
                    if ((mapper.intern() == mappers[0].intern() && tempPath.indexOf("ClientTraceDetailed") > 0)
                            || (mapper.intern() == mappers[1].intern()
                                    && tempPath.indexOf("ClientTraceDetailed") > 0)
                            || (mapper.intern() == mappers[2].intern() && tempPath.indexOf("TaskData") > 0)
                            || (mapper.intern() == mappers[2].intern() && tempPath.indexOf("JobData") > 0)) {
                        args[k] = tempPath;
                        k++;
                        hasData = true;
                    }
                }
                args[1] = k - 2 + "";
                fsmConf.set("chukwa.salsa.fsm.mapclass", mapper);
                args[k] = outputDir + mapper;
                Path outputPath = new Path(args[k]);
                outputPaths.add(outputPath);
                if (hasData) {
                    int res = ToolRunner.run(fsmConf, new FSMBuilder(), args);
                }
            }
        }
        // Find the mapreduce output and load to MDL.
        for (Path outputPath : outputPaths) {
            Path searchDir = new Path(outputPath.toUri().toString() + "/*/*/*.evt");
            log.info("Search dir:" + searchDir.toUri().toString());
            FileStatus[] outputList = fs.globStatus(searchDir);
            if (outputList != null) {
                for (int j = 0; j < outputList.length; j++) {
                    String outputFile = outputList[j].getPath().toUri().toString();
                    log.info("FSM -> MDL loading: " + outputFile);
                    completion.submit(new MetricDataLoader(conf, fs, outputFile));
                    counter++;
                }
            } else {
                log.warn("No output to load.");
            }
        }
        for (int i = 0; i < counter; i++) {
            completion.take().get();
        }
        // Clean up mapreduce output of fsm.
        for (Path dir : outputPaths) {
            fs.delete(dir, true);
        }
    } catch (Exception e) {
        log.error(ExceptionUtil.getStackTrace(e));
        throw new IOException();
    } finally {
    }
}

From source file:com.laudandjolynn.mytv.proxy.MyTvProxyManager.java

public void prepareProxies(ProxyProvider... providers) {
    int length = providers == null ? 0 : providers.length;
    if (length > 0) {
        int maxThreadNum = Constant.CPU_PROCESSOR_NUM;
        ThreadFactory threadFactory = new BasicThreadFactory.Builder().namingPattern("MyTv_Find_Proxies_%d")
                .build();/*ww w.  j a  v  a2s  .  c  o m*/
        ExecutorService executorService = Executors
                .newFixedThreadPool(length > maxThreadNum ? maxThreadNum : length, threadFactory);
        CompletionService<List<Proxy>> completionService = new ExecutorCompletionService<List<Proxy>>(
                executorService);
        providerList.clear();
        for (int i = 0; i < length; i++) {
            final ProxyProvider provider = providers[i];
            providerList.add(provider);
            completionService.submit(new Callable<List<Proxy>>() {

                @Override
                public List<Proxy> call() throws Exception {
                    return provider.getProxies();
                }
            });
        }
        executorService.shutdown();

        int count = 0;
        List<Proxy> resultList = new ArrayList<Proxy>();
        while (count < length) {
            try {
                Future<List<Proxy>> future = completionService.take();
                List<Proxy> proxies = future.get();
                if (proxies != null) {
                    resultList.addAll(proxies);
                }
            } catch (InterruptedException e) {
                logger.error("get proxies thread has interrupted.", e);
            } catch (ExecutionException e) {
                logger.error("get proxies thread has execution fail.", e);
            }
            count++;
        }
        resultList.add(LOCALHOST_PROXY);
        PROXY_QUEUE.clear();
        PROXY_QUEUE.addAll(resultList);
    }
}

From source file:org.apache.hadoop.hbase.backup.regionserver.LogRollBackupSubprocedurePool.java

public LogRollBackupSubprocedurePool(String name, Configuration conf) {
    // configure the executor service
    long keepAlive = conf.getLong(LogRollRegionServerProcedureManager.BACKUP_TIMEOUT_MILLIS_KEY,
            LogRollRegionServerProcedureManager.BACKUP_TIMEOUT_MILLIS_DEFAULT);
    int threads = conf.getInt(CONCURENT_BACKUP_TASKS_KEY, DEFAULT_CONCURRENT_BACKUP_TASKS);
    this.name = name;
    executor = new ThreadPoolExecutor(1, threads, keepAlive, TimeUnit.SECONDS,
            new LinkedBlockingQueue<Runnable>(), new DaemonThreadFactory("rs(" + name + ")-backup-pool"));
    taskPool = new ExecutorCompletionService<Void>(executor);
}

From source file:biz.c24.io.spring.integration.samples.fpml.PreRenderingFpmlGenerator.java

private List<Generator> preRender() throws Exception {

    List<Generator> result = new ArrayList<Generator>(THREADS);

    final TradeConfirmed tradeConfirmed = readTradeConfirmed();

    ExecutorCompletionService<Generator> completionService = new ExecutorCompletionService<Generator>(
            Executors.newFixedThreadPool(THREADS));

    for (int i = 0; i < THREADS; i++) {
        completionService.submit(new Callable<Generator>() {

            public Generator call() throws Exception {
                System.out.println("Rendering... ");

                OutputType ot = OutputType.BYTE_ARRAY;
                Random rand = new Random();
                TradeConfirmed myTradeConfirmed = (TradeConfirmed) tradeConfirmed.cloneDeep();

                Generator gen = new Generator();

                List<Object> payloads = new ArrayList<Object>(ITERATIONS);

                for (int j = 0; j < ITERATIONS; j++) {

                    TradeConfirmed fpML = randomizeFpML(myTradeConfirmed);

                    if (rand.nextInt(100) == 0) {
                        breakFpml(fpML);
                    }//from   w w w.  j a  va 2s .c o  m

                    Sink sink = ot.getSink(sinkFactory);
                    sink.writeObject(fpML);
                    Object payload = ot.getOutput(sink);

                    payloads.add(payload);
                }

                gen.payloads = payloads;

                return gen;
            }
        });
    }

    for (int i = 0; i < THREADS; i++) {
        Future<Generator> future = completionService.take();
        result.add(future.get());
    }

    return result;

}

From source file:com.netflix.curator.framework.recipes.queue.TestBoundedDistributedQueue.java

@SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
@Test//from  ww w  .j ava 2 s  .c o m
public void testMulti() throws Exception {
    final String PATH = "/queue";
    final int CLIENT_QTY = 4;
    final int MAX_ITEMS = 10;
    final int ADD_ITEMS = MAX_ITEMS * 100;
    final int SLOP_FACTOR = 2;

    final QueueConsumer<String> consumer = new QueueConsumer<String>() {
        @Override
        public void consumeMessage(String message) throws Exception {
            Thread.sleep(10);
        }

        @Override
        public void stateChanged(CuratorFramework client, ConnectionState newState) {
        }
    };

    final Timing timing = new Timing();
    final ExecutorService executor = Executors.newCachedThreadPool();
    ExecutorCompletionService<Void> completionService = new ExecutorCompletionService<Void>(executor);

    final CuratorFramework client = CuratorFrameworkFactory.newClient(server.getConnectString(),
            timing.session(), timing.connection(), new RetryOneTime(1));
    try {
        client.start();
        client.create().forPath(PATH);

        final CountDownLatch isWaitingLatch = new CountDownLatch(1);
        final AtomicBoolean isDone = new AtomicBoolean(false);
        final List<Integer> counts = new CopyOnWriteArrayList<Integer>();
        final Object lock = new Object();
        executor.submit(new Callable<Void>() {
            @Override
            public Void call() throws Exception {
                Watcher watcher = new Watcher() {
                    @Override
                    public void process(WatchedEvent event) {
                        synchronized (lock) {
                            lock.notifyAll();
                        }
                    }
                };

                while (!Thread.currentThread().isInterrupted() && client.isStarted() && !isDone.get()) {
                    synchronized (lock) {
                        int size = client.getChildren().usingWatcher(watcher).forPath(PATH).size();
                        counts.add(size);
                        isWaitingLatch.countDown();
                        lock.wait();
                    }
                }
                return null;
            }
        });
        isWaitingLatch.await();

        for (int i = 0; i < CLIENT_QTY; ++i) {
            final int index = i;
            completionService.submit(new Callable<Void>() {
                @Override
                public Void call() throws Exception {
                    CuratorFramework client = null;
                    DistributedQueue<String> queue = null;

                    try {
                        client = CuratorFrameworkFactory.newClient(server.getConnectString(), timing.session(),
                                timing.connection(), new RetryOneTime(1));
                        client.start();
                        queue = QueueBuilder.builder(client, consumer, serializer, PATH).executor(executor)
                                .maxItems(MAX_ITEMS).putInBackground(false).lockPath("/locks").buildQueue();
                        queue.start();

                        for (int i = 0; i < ADD_ITEMS; ++i) {
                            queue.put("" + index + "-" + i);
                        }
                    } finally {
                        IOUtils.closeQuietly(queue);
                        IOUtils.closeQuietly(client);
                    }
                    return null;
                }
            });
        }

        for (int i = 0; i < CLIENT_QTY; ++i) {
            completionService.take().get();
        }

        isDone.set(true);
        synchronized (lock) {
            lock.notifyAll();
        }

        for (int count : counts) {
            Assert.assertTrue(counts.toString(), count <= (MAX_ITEMS * SLOP_FACTOR));
        }
    } finally {
        executor.shutdownNow();
        IOUtils.closeQuietly(client);
    }
}

From source file:org.tallison.cc.WReGetter.java

private void execute(String[] args) throws IOException {
    if (args.length != 3) {
        usage();//from w  ww  . j  av  a 2s .c o  m
        System.exit(1);
    }
    if (args[0].contains("-h")) {
        usage();
        System.exit(0);
    }
    int numThreads = Integer.parseInt(args[0]);
    BufferedReader r = Files.newBufferedReader(Paths.get(args[1]));
    ArrayBlockingQueue<DigestURLPair> queue = new ArrayBlockingQueue<DigestURLPair>(1000);
    QueueFiller filler = new QueueFiller(r, queue, numThreads);
    new Thread(filler).start();
    rootDir = Paths.get(args[2]);
    System.out.println("creating thread pool");
    ExecutorService executorService = Executors.newFixedThreadPool(numThreads);
    ExecutorCompletionService<Integer> executorCompletionService = new ExecutorCompletionService<Integer>(
            executorService);
    System.out.println("about to start");

    for (int i = 0; i < numThreads; i++) {
        System.out.println("submitted " + i);
        executorCompletionService.submit(new WGetter(queue));
    }

    int completed = 0;
    while (completed < numThreads) {
        try {
            Future<Integer> future = executorCompletionService.poll(1, TimeUnit.SECONDS);
            if (future != null) {
                completed++;
            }
        } catch (InterruptedException e) {

        }
    }
    executorService.shutdown();
    executorService.shutdownNow();
    System.exit(0);

}

From source file:com.flipkart.bifrost.CommunicationTest.java

@Test
public void testSendReceive() throws Exception {
    ObjectMapper mapper = new ObjectMapper();
    mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
    mapper.setSerializationInclusion(JsonInclude.Include.NON_EMPTY);

    Connection connection = new Connection(Lists.newArrayList("localhost"), "guest", "guest");
    connection.start();//from ww  w .  ja v  a2 s .c o  m

    BifrostExecutor<Void> executor = BifrostExecutor.<Void>builder(TestAction.class).connection(connection)
            .objectMapper(mapper).requestQueue("bifrost-send").responseQueue("bifrost-recv").concurrency(10)
            .executorService(Executors.newFixedThreadPool(10)).build();

    BifrostRemoteCallExecutionServer<Void> executionServer = BifrostRemoteCallExecutionServer
            .<Void>builder(TestAction.class).objectMapper(mapper).connection(connection).concurrency(10)
            .requestQueue("bifrost-send").build();
    executionServer.start();

    long startTime = System.currentTimeMillis();
    AtomicInteger counter = new AtomicInteger(0);
    int requestCount = 100;
    CompletionService<Void> ecs = new ExecutorCompletionService<>(Executors.newFixedThreadPool(50));
    List<Future<Void>> futures = Lists.newArrayListWithCapacity(requestCount);
    for (int i = 0; i < requestCount; i++) {
        futures.add(ecs.submit(new ServiceCaller(executor, counter)));
    }
    for (int i = 0; i < requestCount; i++) {
        try {
            ecs.take().get();
        } catch (ExecutionException e) {
            e.printStackTrace();
        }
    }
    System.out.println(
            String.format("Completed: %d in %d ms", counter.get(), (System.currentTimeMillis() - startTime)));
    executor.shutdown();
    executionServer.stop();
    connection.stop();

    Assert.assertEquals(requestCount, counter.get());
}

From source file:com.tascape.qa.th.SuiteRunner.java

public int startExecution() throws IOException, InterruptedException, SQLException, XMLStreamException {
    File dir = SYS_CONFIG.getLogPath().resolve(execId).toFile();
    LOG.info("Create suite execution log directory {}", dir);
    if (!dir.exists() && !dir.mkdirs()) {
        throw new IOException("Cannot create directory " + dir);
    }/*from  ww  w.  j  a  va  2 s.c  om*/
    this.logAppProperties(dir);

    int threadCount = SYS_CONFIG.getExecutionThreadCount();
    LOG.info("Start execution engine with {} thread(s)", threadCount);
    int len = (threadCount + "").length();
    ThreadFactory namedThreadFactory = new ThreadFactoryBuilder().setNameFormat("t%0" + len + "d").build();
    ExecutorService executorService = Executors.newFixedThreadPool(threadCount, namedThreadFactory);
    CompletionService<TestResult> completionService = new ExecutorCompletionService<>(executorService);

    LOG.info("Start to acquire test cases to execute");
    int numberOfFailures = 0;
    try {
        List<TestResult> tcrs = this.filter(this.db.getQueuedTestCaseResults(this.execId, 100));
        while (!tcrs.isEmpty()) {
            List<Future<TestResult>> futures = new ArrayList<>();

            for (TestResult tcr : tcrs) {
                LOG.info("Submit test case {}", tcr.getTestCase().format());
                futures.add(completionService.submit(new TestRunnerJUnit4(db, tcr)));
            }
            LOG.debug("Total {} test cases submitted", futures.size());

            for (Future<TestResult> f : futures) {
                try {
                    Future<TestResult> future = completionService.take();
                    TestResult tcr = future.get();
                    if (tcr == null) {
                        continue;
                    }
                    String result = tcr.getResult().result();
                    LOG.info("Get result of test case {} - {}", tcr.getTestCase().format(), result);
                    if (!ExecutionResult.PASS.name().equals(result) && !result.endsWith("/0")) {
                        numberOfFailures++;
                    }
                } catch (Throwable ex) {
                    LOG.error("Error executing test thread", ex);
                    numberOfFailures++;
                }
            }

            tcrs = this.filter(this.db.getQueuedTestCaseResults(this.execId, 100));
        }
    } finally {
        AbstractSuite.getSuites().stream().forEach((suite) -> {
            try {
                suite.tearDown();
            } catch (Exception ex) {
                LOG.warn("Error tearing down suite {} -  {}", suite.getClass(), ex.getMessage());
            }
        });
    }
    executorService.shutdown();

    LOG.info("No more test case to run on this host, updating suite execution result");
    this.db.updateSuiteExecutionResult(this.execId);
    this.db.saveJunitXml(this.execId);
    return numberOfFailures;
}

From source file:com.netflix.curator.framework.recipes.locks.TestInterProcessMutexBase.java

@Test
public void testWaitingProcessKilledServer() throws Exception {
    final Timing timing = new Timing();
    final CuratorFramework client = CuratorFrameworkFactory.newClient(server.getConnectString(),
            new RetryOneTime(1));
    try {// w w  w .  j  av a  2  s .  c  om
        client.start();

        final CountDownLatch latch = new CountDownLatch(1);
        ConnectionStateListener listener = new ConnectionStateListener() {
            @Override
            public void stateChanged(CuratorFramework client, ConnectionState newState) {
                if (newState == ConnectionState.LOST) {
                    latch.countDown();
                }
            }
        };
        client.getConnectionStateListenable().addListener(listener);

        final AtomicBoolean isFirst = new AtomicBoolean(true);
        ExecutorCompletionService<Object> service = new ExecutorCompletionService<Object>(
                Executors.newFixedThreadPool(2));
        for (int i = 0; i < 2; ++i) {
            service.submit(new Callable<Object>() {
                @Override
                public Object call() throws Exception {
                    InterProcessLock lock = makeLock(client);
                    lock.acquire();
                    try {
                        if (isFirst.compareAndSet(true, false)) {
                            timing.sleepABit();

                            server.stop();
                            Assert.assertTrue(timing.awaitLatch(latch));
                            server = new TestingServer(server.getPort(), server.getTempDirectory());
                        }
                    } finally {
                        try {
                            lock.release();
                        } catch (Exception e) {
                            // ignore
                        }
                    }
                    return null;
                }
            });
        }

        for (int i = 0; i < 2; ++i) {
            service.take().get(timing.forWaiting().milliseconds(), TimeUnit.MILLISECONDS);
        }
    } finally {
        IOUtils.closeQuietly(client);
    }
}