List of usage examples for java.util.concurrent ExecutorCompletionService ExecutorCompletionService
public ExecutorCompletionService(Executor executor)
From source file:com.siva.javamultithreading.MultiThreadExecutor.java
/** * This is sample.//from w w w. j a va 2 s .com */ private static void startProcess() { ExecutorService threadPool = Executors.newFixedThreadPool(4); CompletionService<HSSFWorkbook> pool = new ExecutorCompletionService<>(threadPool); // Excel creation through multiple threads long startTime = System.currentTimeMillis(); pool.submit(new ExcelChunkSheetWriter(0, 1000)); pool.submit(new ExcelChunkSheetWriter(1001, 20000)); pool.submit(new ExcelChunkSheetWriter(2, 3000)); pool.submit(new ExcelChunkSheetWriter(3, 40000)); pool.submit(new ExcelChunkSheetWriter(4, 50000)); HSSFWorkbook hSSFWorkbook = null; HSSFWorkbook book = new HSSFWorkbook(); HSSFSheet sheet = book.createSheet("Report"); try { for (int i = 0; i < 5; i++) { hSSFWorkbook = pool.take().get(); System.out.println( "sheet row count : sheet.PhysicalNumberOfRows() = " + sheet.getPhysicalNumberOfRows()); int currentCount = sheet.getPhysicalNumberOfRows(); int incomingCount = hSSFWorkbook.getSheetAt(0).getPhysicalNumberOfRows(); if ((currentCount + incomingCount) > 60000) { sheet = book.createSheet("Report" + i); } ExcelUtil.copySheets(book, sheet, hSSFWorkbook.getSheetAt(0)); } } catch (InterruptedException ex) { Logger.getLogger(MultiThreadExecutor.class.getName()).log(Level.SEVERE, null, ex); } catch (ExecutionException ex) { Logger.getLogger(MultiThreadExecutor.class.getName()).log(Level.SEVERE, null, ex); } try { writeFile(book, new FileOutputStream("Report.xls")); } catch (Exception e) { e.printStackTrace(); } /* FileOutputStream fos = new FileOutputStream("all.zip"); ZipOutputStream zos = new ZipOutputStream(fos); for (int i = 0; i < 5; i++) { try { hSSFWorkbook = pool.take().get(); ZipEntry ze = new ZipEntry("Excel" + i + ".xls"); zos.putNextEntry(ze); hSSFWorkbook.write(zos); zos.closeEntry(); } catch (InterruptedException ex) { Logger.getLogger(MultiThreadExecutor.class.getName()).log(Level.SEVERE, null, ex); } } zos.close(); */ long endTime = System.currentTimeMillis(); System.out.println("Time taken: " + (endTime - startTime) + " ms"); threadPool.shutdown(); }
From source file:org.apache.hadoop.chukwa.dataloader.FSMDataLoader.java
public void load(ChukwaConfiguration conf, FileSystem fs, FileStatus[] fileList) throws IOException { if (executor == null) { try {/*www . j ava 2 s.co m*/ this.size = Integer.parseInt(conf.get(DATA_LOADER_THREAD_LIMIT)); } catch (Exception e) { this.size = 1; } executor = Executors.newFixedThreadPool(size); } if (completion == null) { completion = new ExecutorCompletionService(executor); } try { // Locate directory output directories of the current demux, and create a unique directory list. HashSet<Path> inputPaths = new HashSet<Path>(); HashSet<Path> outputPaths = new HashSet<Path>(); int counter = 0; for (int i = 0; i < fileList.length; i++) { Path temp = fileList[i].getPath().getParent(); if (!inputPaths.contains(temp)) { inputPaths.add(temp); } } String outputDir = conf.get("chukwa.tmp.data.dir") + File.separator + "fsm_" + System.currentTimeMillis() + "_"; if (inputPaths.size() > 0) { Configuration fsmConf = new Configuration(); // Run fsm map reduce job for dn, tt, and jobhist. for (String mapper : mappers) { String[] args = new String[inputPaths.size() + 3]; args[0] = "-in"; int k = 2; boolean hasData = false; for (Path temp : inputPaths) { String tempPath = temp.toUri().toString(); if ((mapper.intern() == mappers[0].intern() && tempPath.indexOf("ClientTraceDetailed") > 0) || (mapper.intern() == mappers[1].intern() && tempPath.indexOf("ClientTraceDetailed") > 0) || (mapper.intern() == mappers[2].intern() && tempPath.indexOf("TaskData") > 0) || (mapper.intern() == mappers[2].intern() && tempPath.indexOf("JobData") > 0)) { args[k] = tempPath; k++; hasData = true; } } args[1] = k - 2 + ""; fsmConf.set("chukwa.salsa.fsm.mapclass", mapper); args[k] = outputDir + mapper; Path outputPath = new Path(args[k]); outputPaths.add(outputPath); if (hasData) { int res = ToolRunner.run(fsmConf, new FSMBuilder(), args); } } } // Find the mapreduce output and load to MDL. for (Path outputPath : outputPaths) { Path searchDir = new Path(outputPath.toUri().toString() + "/*/*/*.evt"); log.info("Search dir:" + searchDir.toUri().toString()); FileStatus[] outputList = fs.globStatus(searchDir); if (outputList != null) { for (int j = 0; j < outputList.length; j++) { String outputFile = outputList[j].getPath().toUri().toString(); log.info("FSM -> MDL loading: " + outputFile); completion.submit(new MetricDataLoader(conf, fs, outputFile)); counter++; } } else { log.warn("No output to load."); } } for (int i = 0; i < counter; i++) { completion.take().get(); } // Clean up mapreduce output of fsm. for (Path dir : outputPaths) { fs.delete(dir, true); } } catch (Exception e) { log.error(ExceptionUtil.getStackTrace(e)); throw new IOException(); } finally { } }
From source file:com.laudandjolynn.mytv.proxy.MyTvProxyManager.java
public void prepareProxies(ProxyProvider... providers) { int length = providers == null ? 0 : providers.length; if (length > 0) { int maxThreadNum = Constant.CPU_PROCESSOR_NUM; ThreadFactory threadFactory = new BasicThreadFactory.Builder().namingPattern("MyTv_Find_Proxies_%d") .build();/*ww w. j a v a2s . c o m*/ ExecutorService executorService = Executors .newFixedThreadPool(length > maxThreadNum ? maxThreadNum : length, threadFactory); CompletionService<List<Proxy>> completionService = new ExecutorCompletionService<List<Proxy>>( executorService); providerList.clear(); for (int i = 0; i < length; i++) { final ProxyProvider provider = providers[i]; providerList.add(provider); completionService.submit(new Callable<List<Proxy>>() { @Override public List<Proxy> call() throws Exception { return provider.getProxies(); } }); } executorService.shutdown(); int count = 0; List<Proxy> resultList = new ArrayList<Proxy>(); while (count < length) { try { Future<List<Proxy>> future = completionService.take(); List<Proxy> proxies = future.get(); if (proxies != null) { resultList.addAll(proxies); } } catch (InterruptedException e) { logger.error("get proxies thread has interrupted.", e); } catch (ExecutionException e) { logger.error("get proxies thread has execution fail.", e); } count++; } resultList.add(LOCALHOST_PROXY); PROXY_QUEUE.clear(); PROXY_QUEUE.addAll(resultList); } }
From source file:org.apache.hadoop.hbase.backup.regionserver.LogRollBackupSubprocedurePool.java
public LogRollBackupSubprocedurePool(String name, Configuration conf) { // configure the executor service long keepAlive = conf.getLong(LogRollRegionServerProcedureManager.BACKUP_TIMEOUT_MILLIS_KEY, LogRollRegionServerProcedureManager.BACKUP_TIMEOUT_MILLIS_DEFAULT); int threads = conf.getInt(CONCURENT_BACKUP_TASKS_KEY, DEFAULT_CONCURRENT_BACKUP_TASKS); this.name = name; executor = new ThreadPoolExecutor(1, threads, keepAlive, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(), new DaemonThreadFactory("rs(" + name + ")-backup-pool")); taskPool = new ExecutorCompletionService<Void>(executor); }
From source file:biz.c24.io.spring.integration.samples.fpml.PreRenderingFpmlGenerator.java
private List<Generator> preRender() throws Exception { List<Generator> result = new ArrayList<Generator>(THREADS); final TradeConfirmed tradeConfirmed = readTradeConfirmed(); ExecutorCompletionService<Generator> completionService = new ExecutorCompletionService<Generator>( Executors.newFixedThreadPool(THREADS)); for (int i = 0; i < THREADS; i++) { completionService.submit(new Callable<Generator>() { public Generator call() throws Exception { System.out.println("Rendering... "); OutputType ot = OutputType.BYTE_ARRAY; Random rand = new Random(); TradeConfirmed myTradeConfirmed = (TradeConfirmed) tradeConfirmed.cloneDeep(); Generator gen = new Generator(); List<Object> payloads = new ArrayList<Object>(ITERATIONS); for (int j = 0; j < ITERATIONS; j++) { TradeConfirmed fpML = randomizeFpML(myTradeConfirmed); if (rand.nextInt(100) == 0) { breakFpml(fpML); }//from w w w. j a va 2s .c o m Sink sink = ot.getSink(sinkFactory); sink.writeObject(fpML); Object payload = ot.getOutput(sink); payloads.add(payload); } gen.payloads = payloads; return gen; } }); } for (int i = 0; i < THREADS; i++) { Future<Generator> future = completionService.take(); result.add(future.get()); } return result; }
From source file:com.netflix.curator.framework.recipes.queue.TestBoundedDistributedQueue.java
@SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter") @Test//from ww w .j ava 2 s .c o m public void testMulti() throws Exception { final String PATH = "/queue"; final int CLIENT_QTY = 4; final int MAX_ITEMS = 10; final int ADD_ITEMS = MAX_ITEMS * 100; final int SLOP_FACTOR = 2; final QueueConsumer<String> consumer = new QueueConsumer<String>() { @Override public void consumeMessage(String message) throws Exception { Thread.sleep(10); } @Override public void stateChanged(CuratorFramework client, ConnectionState newState) { } }; final Timing timing = new Timing(); final ExecutorService executor = Executors.newCachedThreadPool(); ExecutorCompletionService<Void> completionService = new ExecutorCompletionService<Void>(executor); final CuratorFramework client = CuratorFrameworkFactory.newClient(server.getConnectString(), timing.session(), timing.connection(), new RetryOneTime(1)); try { client.start(); client.create().forPath(PATH); final CountDownLatch isWaitingLatch = new CountDownLatch(1); final AtomicBoolean isDone = new AtomicBoolean(false); final List<Integer> counts = new CopyOnWriteArrayList<Integer>(); final Object lock = new Object(); executor.submit(new Callable<Void>() { @Override public Void call() throws Exception { Watcher watcher = new Watcher() { @Override public void process(WatchedEvent event) { synchronized (lock) { lock.notifyAll(); } } }; while (!Thread.currentThread().isInterrupted() && client.isStarted() && !isDone.get()) { synchronized (lock) { int size = client.getChildren().usingWatcher(watcher).forPath(PATH).size(); counts.add(size); isWaitingLatch.countDown(); lock.wait(); } } return null; } }); isWaitingLatch.await(); for (int i = 0; i < CLIENT_QTY; ++i) { final int index = i; completionService.submit(new Callable<Void>() { @Override public Void call() throws Exception { CuratorFramework client = null; DistributedQueue<String> queue = null; try { client = CuratorFrameworkFactory.newClient(server.getConnectString(), timing.session(), timing.connection(), new RetryOneTime(1)); client.start(); queue = QueueBuilder.builder(client, consumer, serializer, PATH).executor(executor) .maxItems(MAX_ITEMS).putInBackground(false).lockPath("/locks").buildQueue(); queue.start(); for (int i = 0; i < ADD_ITEMS; ++i) { queue.put("" + index + "-" + i); } } finally { IOUtils.closeQuietly(queue); IOUtils.closeQuietly(client); } return null; } }); } for (int i = 0; i < CLIENT_QTY; ++i) { completionService.take().get(); } isDone.set(true); synchronized (lock) { lock.notifyAll(); } for (int count : counts) { Assert.assertTrue(counts.toString(), count <= (MAX_ITEMS * SLOP_FACTOR)); } } finally { executor.shutdownNow(); IOUtils.closeQuietly(client); } }
From source file:org.tallison.cc.WReGetter.java
private void execute(String[] args) throws IOException { if (args.length != 3) { usage();//from w ww . j av a 2s .c o m System.exit(1); } if (args[0].contains("-h")) { usage(); System.exit(0); } int numThreads = Integer.parseInt(args[0]); BufferedReader r = Files.newBufferedReader(Paths.get(args[1])); ArrayBlockingQueue<DigestURLPair> queue = new ArrayBlockingQueue<DigestURLPair>(1000); QueueFiller filler = new QueueFiller(r, queue, numThreads); new Thread(filler).start(); rootDir = Paths.get(args[2]); System.out.println("creating thread pool"); ExecutorService executorService = Executors.newFixedThreadPool(numThreads); ExecutorCompletionService<Integer> executorCompletionService = new ExecutorCompletionService<Integer>( executorService); System.out.println("about to start"); for (int i = 0; i < numThreads; i++) { System.out.println("submitted " + i); executorCompletionService.submit(new WGetter(queue)); } int completed = 0; while (completed < numThreads) { try { Future<Integer> future = executorCompletionService.poll(1, TimeUnit.SECONDS); if (future != null) { completed++; } } catch (InterruptedException e) { } } executorService.shutdown(); executorService.shutdownNow(); System.exit(0); }
From source file:com.flipkart.bifrost.CommunicationTest.java
@Test public void testSendReceive() throws Exception { ObjectMapper mapper = new ObjectMapper(); mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); mapper.setSerializationInclusion(JsonInclude.Include.NON_EMPTY); Connection connection = new Connection(Lists.newArrayList("localhost"), "guest", "guest"); connection.start();//from ww w . ja v a2 s .c o m BifrostExecutor<Void> executor = BifrostExecutor.<Void>builder(TestAction.class).connection(connection) .objectMapper(mapper).requestQueue("bifrost-send").responseQueue("bifrost-recv").concurrency(10) .executorService(Executors.newFixedThreadPool(10)).build(); BifrostRemoteCallExecutionServer<Void> executionServer = BifrostRemoteCallExecutionServer .<Void>builder(TestAction.class).objectMapper(mapper).connection(connection).concurrency(10) .requestQueue("bifrost-send").build(); executionServer.start(); long startTime = System.currentTimeMillis(); AtomicInteger counter = new AtomicInteger(0); int requestCount = 100; CompletionService<Void> ecs = new ExecutorCompletionService<>(Executors.newFixedThreadPool(50)); List<Future<Void>> futures = Lists.newArrayListWithCapacity(requestCount); for (int i = 0; i < requestCount; i++) { futures.add(ecs.submit(new ServiceCaller(executor, counter))); } for (int i = 0; i < requestCount; i++) { try { ecs.take().get(); } catch (ExecutionException e) { e.printStackTrace(); } } System.out.println( String.format("Completed: %d in %d ms", counter.get(), (System.currentTimeMillis() - startTime))); executor.shutdown(); executionServer.stop(); connection.stop(); Assert.assertEquals(requestCount, counter.get()); }
From source file:com.tascape.qa.th.SuiteRunner.java
public int startExecution() throws IOException, InterruptedException, SQLException, XMLStreamException { File dir = SYS_CONFIG.getLogPath().resolve(execId).toFile(); LOG.info("Create suite execution log directory {}", dir); if (!dir.exists() && !dir.mkdirs()) { throw new IOException("Cannot create directory " + dir); }/*from ww w. j a va 2 s.c om*/ this.logAppProperties(dir); int threadCount = SYS_CONFIG.getExecutionThreadCount(); LOG.info("Start execution engine with {} thread(s)", threadCount); int len = (threadCount + "").length(); ThreadFactory namedThreadFactory = new ThreadFactoryBuilder().setNameFormat("t%0" + len + "d").build(); ExecutorService executorService = Executors.newFixedThreadPool(threadCount, namedThreadFactory); CompletionService<TestResult> completionService = new ExecutorCompletionService<>(executorService); LOG.info("Start to acquire test cases to execute"); int numberOfFailures = 0; try { List<TestResult> tcrs = this.filter(this.db.getQueuedTestCaseResults(this.execId, 100)); while (!tcrs.isEmpty()) { List<Future<TestResult>> futures = new ArrayList<>(); for (TestResult tcr : tcrs) { LOG.info("Submit test case {}", tcr.getTestCase().format()); futures.add(completionService.submit(new TestRunnerJUnit4(db, tcr))); } LOG.debug("Total {} test cases submitted", futures.size()); for (Future<TestResult> f : futures) { try { Future<TestResult> future = completionService.take(); TestResult tcr = future.get(); if (tcr == null) { continue; } String result = tcr.getResult().result(); LOG.info("Get result of test case {} - {}", tcr.getTestCase().format(), result); if (!ExecutionResult.PASS.name().equals(result) && !result.endsWith("/0")) { numberOfFailures++; } } catch (Throwable ex) { LOG.error("Error executing test thread", ex); numberOfFailures++; } } tcrs = this.filter(this.db.getQueuedTestCaseResults(this.execId, 100)); } } finally { AbstractSuite.getSuites().stream().forEach((suite) -> { try { suite.tearDown(); } catch (Exception ex) { LOG.warn("Error tearing down suite {} - {}", suite.getClass(), ex.getMessage()); } }); } executorService.shutdown(); LOG.info("No more test case to run on this host, updating suite execution result"); this.db.updateSuiteExecutionResult(this.execId); this.db.saveJunitXml(this.execId); return numberOfFailures; }
From source file:com.netflix.curator.framework.recipes.locks.TestInterProcessMutexBase.java
@Test public void testWaitingProcessKilledServer() throws Exception { final Timing timing = new Timing(); final CuratorFramework client = CuratorFrameworkFactory.newClient(server.getConnectString(), new RetryOneTime(1)); try {// w w w . j av a 2 s . c om client.start(); final CountDownLatch latch = new CountDownLatch(1); ConnectionStateListener listener = new ConnectionStateListener() { @Override public void stateChanged(CuratorFramework client, ConnectionState newState) { if (newState == ConnectionState.LOST) { latch.countDown(); } } }; client.getConnectionStateListenable().addListener(listener); final AtomicBoolean isFirst = new AtomicBoolean(true); ExecutorCompletionService<Object> service = new ExecutorCompletionService<Object>( Executors.newFixedThreadPool(2)); for (int i = 0; i < 2; ++i) { service.submit(new Callable<Object>() { @Override public Object call() throws Exception { InterProcessLock lock = makeLock(client); lock.acquire(); try { if (isFirst.compareAndSet(true, false)) { timing.sleepABit(); server.stop(); Assert.assertTrue(timing.awaitLatch(latch)); server = new TestingServer(server.getPort(), server.getTempDirectory()); } } finally { try { lock.release(); } catch (Exception e) { // ignore } } return null; } }); } for (int i = 0; i < 2; ++i) { service.take().get(timing.forWaiting().milliseconds(), TimeUnit.MILLISECONDS); } } finally { IOUtils.closeQuietly(client); } }