List of usage examples for java.util.concurrent ExecutorService invokeAll
<T> List<Future<T>> invokeAll(Collection<? extends Callable<T>> tasks) throws InterruptedException;
From source file:demo.vmware.commands.schemaspecific.CommandCreateData.java
@Override public CommandResult run(ConfigurableApplicationContext mainContext, List<String> parameters) { List<String> messages = new ArrayList<String>(); // lets us track cache growth int attributeCounter = 0; CommandTimer timer = new CommandTimer(); LOG.info("creating test data for " + numCompanies + "," + numRegions + "," + numGroups + "," + numBranches); // use the Java executor service because of it's awesome invokeAll method. // we have 4 cores but gemfire needs some. 2 is probably a more realistic number ExecutorService taskExecutor = Executors.newFixedThreadPool(2); Collection tasks = new ArrayList<CompanyHierarchyPopulator>(); for (int i = 0; i < numCompanies; i++) { // add a task for each company we are creating CompanyHierarchyPopulator poper = new CompanyHierarchyPopulator(); tasks.add(poper);/*from w w w .j a va 2 s. c o m*/ } try { // run all the tasks in execution pool List<Future<?>> futures = taskExecutor.invokeAll(tasks); taskExecutor.shutdown(); // aggregate the results from the call() method for (int i = 0; i < numCompanies; i++) { // should get resulting messages also -- sometime in the future attributeCounter += ((Integer) futures.get(i).get()).intValue(); } timer.stop(); messages.add("Created " + attributeCounter + " attributes and " + " took " + timer.getTimeDiffInSeconds() + " sec"); } catch (ExecutionException e) { // this should never happen LOG.warn("something bad happend", e); messages.add("Something bad happend " + e.getMessage()); } catch (InterruptedException e) { // this should never happen LOG.warn("something bad happend", e); messages.add("Something bad happend " + e.getMessage()); } return new CommandResult(null, messages); }
From source file:com.ibm.bi.dml.runtime.io.ReaderTextCSVParallel.java
/** * /* w w w .j a va 2 s.co m*/ * @param path * @param job * @param hasHeader * @param delim * @return * @throws IOException * @throws DMLRuntimeException */ private MatrixBlock computeCSVSizeAndCreateOutputMatrixBlock(InputSplit[] splits, Path path, JobConf job, boolean hasHeader, String delim, long estnnz) throws IOException, DMLRuntimeException { int nrow = 0; int ncol = 0; FileInputFormat.addInputPath(job, path); TextInputFormat informat = new TextInputFormat(); informat.configure(job); // count no of entities in the first non-header row LongWritable key = new LongWritable(); Text oneLine = new Text(); RecordReader<LongWritable, Text> reader = informat.getRecordReader(splits[0], job, Reporter.NULL); try { if (reader.next(key, oneLine)) { String cellStr = oneLine.toString().trim(); ncol = StringUtils.countMatches(cellStr, delim) + 1; } } finally { IOUtilFunctions.closeSilently(reader); } // count rows in parallel per split try { ExecutorService pool = Executors.newFixedThreadPool(_numThreads); ArrayList<CountRowsTask> tasks = new ArrayList<CountRowsTask>(); for (InputSplit split : splits) { tasks.add(new CountRowsTask(split, informat, job, hasHeader)); hasHeader = false; } pool.invokeAll(tasks); pool.shutdown(); // collect row counts for offset computation // early error notify in case not all tasks successful _offsets = new SplitOffsetInfos(tasks.size()); for (CountRowsTask rt : tasks) { if (!rt.getReturnCode()) throw new IOException("Count task for csv input failed: " + rt.getErrMsg()); _offsets.setOffsetPerSplit(tasks.indexOf(rt), nrow); _offsets.setLenghtPerSplit(tasks.indexOf(rt), rt.getRowCount()); nrow = nrow + rt.getRowCount(); } } catch (Exception e) { throw new IOException("Threadpool Error " + e.getMessage(), e); } // allocate target matrix block based on given size; // need to allocate sparse as well since lock-free insert into target return createOutputMatrixBlock(nrow, ncol, estnnz, true, true); }
From source file:hudson.FilePathTest.java
private List<Future<Integer>> whenFileIsCopied100TimesConcurrently(final File file) throws InterruptedException { List<Callable<Integer>> r = new ArrayList<Callable<Integer>>(); for (int i = 0; i < 100; i++) { r.add(new Callable<Integer>() { public Integer call() throws Exception { class Sink extends OutputStream { private Exception closed; private volatile int count; private void checkNotClosed() throws IOException { if (closed != null) throw new IOException(closed); }/* www . j ava 2 s . co m*/ @Override public void write(int b) throws IOException { count++; checkNotClosed(); } @Override public void write(byte[] b) throws IOException { count += b.length; checkNotClosed(); } @Override public void write(byte[] b, int off, int len) throws IOException { count += len; checkNotClosed(); } @Override public void close() throws IOException { closed = new Exception(); //if (size!=count) // fail(); } } FilePath f = new FilePath(channels.french, file.getPath()); Sink sink = new Sink(); f.copyTo(sink); return sink.count; } }); } ExecutorService es = Executors.newFixedThreadPool(100); try { return es.invokeAll(r); } finally { es.shutdown(); } }
From source file:com.mnxfst.testing.client.TSClient.java
/** * Executes the referenced test plan for all given host names. The result contains a mapping from a host name to the returned result identifier * of that ptest-server instance /*from www.j a v a 2 s .c o m*/ * @param hostNames * @param port * @param threads * @param recurrences * @param recurrenceType * @param testplan * @param additionalParameters * @param urlEncoding * @return * @throws TSClientConfigurationException * @throws TSClientExecutionException */ protected Map<String, String> executeTestPlan(String[] hostNames, int port, long threads, long recurrences, TSPlanRecurrenceType recurrenceType, byte[] testplan, Properties additionalParameters, String urlEncoding) throws TSClientConfigurationException, TSClientExecutionException { // the ptest-server understands http get, thus we use it TODO refactor to post and send testplan as well and do not reference it anymore! StringBuffer buffer = new StringBuffer("/?"); buffer.append(REQUEST_PARAMETER_EXECUTE).append("=1"); buffer.append("&").append(REQUEST_PARAMETER_RECURRENCES).append("=").append(recurrences); buffer.append("&").append(REQUEST_PARAMETER_RECURRENCE_TYPE).append("=").append(recurrenceType.toString()); buffer.append("&").append(REQUEST_PARAMETER_THREADS).append("=").append(threads); try { if (additionalParameters != null && !additionalParameters.isEmpty()) { for (Object key : additionalParameters.keySet()) { String value = (String) additionalParameters.get(key); buffer.append("&").append(key).append("=").append(URLEncoder.encode(value, urlEncoding)); } } } catch (UnsupportedEncodingException e) { throw new TSClientConfigurationException( "Unsupported encoding type: " + urlEncoding + ". Error: " + e.getMessage()); } StringBuffer hn = new StringBuffer(); for (int i = 0; i < hostNames.length; i++) { hn.append(hostNames[i]); if (i < hostNames.length - 1) hn.append(", "); } System.out.println("Execute testplan:"); System.out.println("\thostNames: " + hn.toString()); System.out.println("\tport: " + port); System.out.println("\tthreads: " + threads); System.out.println("\trecurrences: " + recurrences); System.out.println("\trecurrenceType: " + recurrenceType); System.out.println("\turl enc: " + urlEncoding); System.out.println("\n\turi: " + buffer.toString()); TSClientPlanExecCallable[] testplanCallables = new TSClientPlanExecCallable[hostNames.length]; for (int i = 0; i < hostNames.length; i++) { testplanCallables[i] = new TSClientPlanExecCallable(hostNames[i], port, buffer.toString(), testplan); } ExecutorService executorService = Executors.newFixedThreadPool(hostNames.length); List<Future<NameValuePair>> executionResults = new ArrayList<Future<NameValuePair>>(); try { executionResults = executorService.invokeAll(Arrays.asList(testplanCallables)); } catch (InterruptedException e) { System.out.println("Test execution interrupted: " + e.getMessage()); } // collect results from callables Map<String, String> result = new HashMap<String, String>(); for (Future<NameValuePair> r : executionResults) { try { NameValuePair nvp = r.get(); result.put(nvp.getName(), nvp.getValue()); } catch (InterruptedException e) { System.out.println("Interrupted while waiting for results. Error: " + e.getMessage()); } catch (ExecutionException e) { System.out.println("Interrupted while waiting for results. Error: " + e.getMessage()); } } return result; }
From source file:org.apache.camel.processor.aggregator.AggregatorConcurrencyTest.java
public void testAggregateConcurrency() throws Exception { int total = 0; ExecutorService service = Executors.newFixedThreadPool(20); List<Callable<Object>> tasks = new ArrayList<Callable<Object>>(); for (int i = 0; i < size; i++) { final int count = i; total += i;// ww w.j av a 2 s . co m tasks.add(new Callable<Object>() { public Object call() throws Exception { template.sendBodyAndHeader(uri, "Hello World", "index", count); return null; } }); } MockEndpoint mock = getMockEndpoint("mock:result"); mock.expectedMessageCount(1); mock.expectedBodiesReceived(total); mock.expectedHeaderReceived("total", total); mock.expectedPropertyReceived(Exchange.AGGREGATED_SIZE, size); // submit all tasks service.invokeAll(tasks); assertMockEndpointsSatisfied(); assertEquals(100, COUNTER.get()); }
From source file:com.mnxfst.testing.client.TSClient.java
/** * Collects results of previously started tests * @param hostResultIdentifierMapping/*from w ww . j a v a 2s . co m*/ * @param port * @return */ protected Set<TSClientPlanExecutionResult> collectTestplanResults( Map<String, String> hostResultIdentifierMapping, int port) { TSClientPlanResultCollectCallable[] callables = new TSClientPlanResultCollectCallable[hostResultIdentifierMapping .size()]; int count = 0; String uri = null; for (String host : hostResultIdentifierMapping.keySet()) { uri = "/?" + REQUEST_PARAMETER_COLLECT + "=1&" + REQUEST_PARAMETER_RESULT_IDENTIFIER + "=" + hostResultIdentifierMapping.get(host); callables[count] = new TSClientPlanResultCollectCallable(host, port, uri); count = count + 1; } ExecutorService executorService = Executors.newFixedThreadPool(hostResultIdentifierMapping.size()); List<Future<TSClientPlanExecutionResult>> collectedResults = new ArrayList<Future<TSClientPlanExecutionResult>>(); try { collectedResults = executorService.invokeAll(Arrays.asList(callables)); } catch (InterruptedException e) { System.out.println("Test execution result collection interrupted: " + e.getMessage()); } Set<TSClientPlanExecutionResult> result = new HashSet<TSClientPlanExecutionResult>(); for (Future<TSClientPlanExecutionResult> r : collectedResults) { try { result.add(r.get()); } catch (InterruptedException e) { System.out.println("Interrupted while collection results: " + e.getMessage()); } catch (ExecutionException e) { System.out.println("Interrupted while collection results: " + e.getMessage()); } } return result; }
From source file:com.vmware.admiral.adapter.docker.service.SystemImageRetrievalManagerTest.java
@Test public void testGetFromUserResourcesConcurrent() throws Throwable { Path testXenonImagesPath = Files.createTempDirectory("test-xenon-images"); // Set expected configuration ConfigurationState config = new ConfigurationState(); config.documentSelfLink = UriUtils.buildUriPath(ConfigurationFactoryService.SELF_LINK, FileUtil.USER_RESOURCES_PATH_VARIABLE); config.key = FileUtil.USER_RESOURCES_PATH_VARIABLE; config.value = testXenonImagesPath.toAbsolutePath().toString(); MockConfigurationService mockConfigurationService = new MockConfigurationService(config); host.startService(Operation.createPost(UriUtils.buildUri(host, UriUtils .buildUriPath(ConfigurationFactoryService.SELF_LINK, FileUtil.USER_RESOURCES_PATH_VARIABLE))), mockConfigurationService);//from w ww . ja va2s .c o m File imageDir = new File(UriUtils.buildUriPath(testXenonImagesPath.toString(), SystemImageRetrievalManager.SYSTEM_IMAGES_PATH)); imageDir.mkdir(); byte[] content = IOUtils .toByteArray(Thread.currentThread().getContextClassLoader().getResourceAsStream(TEST_IMAGE)); // Basically, rename it so it must be loaded from user resources for sure File tmpFile = new File(UriUtils.buildUriPath(imageDir.getAbsolutePath(), TEST_IMAGE_RES)); tmpFile.createNewFile(); try (OutputStream os = new FileOutputStream(tmpFile)) { os.write(content); } AdapterRequest req = new AdapterRequest(); req.resourceReference = host.getUri(); List<byte[]> retrievedImages = new ArrayList<>(); int numberOfRequests = 4; TestContext ctx = testCreate(numberOfRequests); final ExecutorService threadPool = Executors.newFixedThreadPool(numberOfRequests); List<Callable<Void>> callables = new ArrayList<>(); for (int i = 0; i < numberOfRequests; i++) { callables.add(() -> { host.log("Calling retrieveAgentImage"); retrievalManager.retrieveAgentImage(TEST_IMAGE_RES, req, (image) -> { retrievedImages.add(image); ctx.completeIteration(); }); return null; }); } host.log("Invoke all callables to retrieveAgentImage"); threadPool.invokeAll(callables); ctx.await(); // Assert that all callbacks were called assertEquals(numberOfRequests, retrievedImages.size()); for (int i = 0; i < numberOfRequests; i++) { byte[] image = retrievedImages.get(i); Assert.assertEquals("Unexpected content", new String(content), new String(image)); } // Assert that service was called only once for all concurrent requests assertEquals(1, mockConfigurationService.getNumberOfRequests()); }
From source file:com.thoughtworks.go.server.security.LdapAuthenticationTest.java
@Test public void shouldAuthenticateConcurrently() throws Exception { ldapServer.addUser(employeesOrgUnit, "foleys", "some-password", "Shilpa Foley", "foleys@somecompany.com"); ExecutorService pool = Executors.newFixedThreadPool(100); List<Callable<String>> allCallables = new ArrayList<Callable<String>>(); for (int i = 0; i < 100; i++) { final boolean even = i % 2 == 0; allCallables.add(new Callable<String>() { @Override/* www . ja v a2 s . co m*/ public String call() throws Exception { if (even) { assertAuthenticationOfValidAdminUser("foleys", "some-password"); } else { assertFailedAuthentication("invalid_user", ""); } return ""; } }); } List<Future<String>> futures = pool.invokeAll(allCallables); pool.shutdown(); boolean finishedWithoutTimeout = pool.awaitTermination(10, TimeUnit.SECONDS); assertThat(finishedWithoutTimeout, is(true)); // Assert no exceptions, by getting result. for (Future<String> future : futures) { future.get(); } }
From source file:org.apache.sysml.runtime.io.ReaderTextCSVParallel.java
private void readCSVMatrixFromHDFS(InputSplit[] splits, Path path, JobConf job, MatrixBlock dest, long rlen, long clen, int brlen, int bclen, boolean hasHeader, String delim, boolean fill, double fillValue) throws IOException { FileInputFormat.addInputPath(job, path); TextInputFormat informat = new TextInputFormat(); informat.configure(job);//w w w . ja va2s .c om ExecutorService pool = Executors.newFixedThreadPool(_numThreads); try { // create read tasks for all splits ArrayList<CSVReadTask> tasks = new ArrayList<CSVReadTask>(); int splitCount = 0; for (InputSplit split : splits) { tasks.add(new CSVReadTask(split, _offsets, informat, job, dest, rlen, clen, hasHeader, delim, fill, fillValue, splitCount++)); } pool.invokeAll(tasks); pool.shutdown(); // check return codes and aggregate nnz long lnnz = 0; for (CSVReadTask rt : tasks) { lnnz += rt.getPartialNnz(); if (!rt.getReturnCode()) { Exception err = rt.getException(); throw new IOException("Read task for csv input failed: " + err.toString(), err); } } dest.setNonZeros(lnnz); } catch (Exception e) { throw new IOException("Threadpool issue, while parallel read.", e); } }
From source file:com.saucelabs.sauce_ondemand.driver.SauceOnDemandSPIImpl.java
/** * Creates a list of WebDriver instances based on the contents of a SAUCE_ONDEMAND_BROWSERS environment variable (typically set * by the Sauce Jenkins plugin).//from ww w. jav a 2s. co m * * @param seleniumFactory * @param browserURL * @return */ @Override public List<WebDriver> createWebDrivers(SeleniumFactory seleniumFactory, final String browserURL) { List<WebDriver> webDrivers = new ArrayList<WebDriver>(); String browserJson = readPropertyOrEnv("SAUCE_ONDEMAND_BROWSERS", null); if (browserJson == null) { throw new IllegalArgumentException("Unable to find SAUCE_ONDEMAND_BROWSERS environment variable"); } //parse JSON and extract the browser urls, so that we know how many threads to schedule List<String> browsers = new ArrayList<String>(); try { JSONArray array = new JSONArray(new JSONTokener(browserJson)); for (int i = 0; i < array.length(); i++) { JSONObject object = array.getJSONObject(i); String uri = object.getString("url"); if (!uri.startsWith(SCHEME)) return null; // not ours uri = uri.substring(SCHEME.length()); if (!uri.startsWith("?")) throw new IllegalArgumentException("Missing '?':" + uri); browsers.add(uri); } } catch (JSONException e) { throw new IllegalArgumentException("Error parsing JSON", e); } //create a fixed thread pool for the number of browser ExecutorService service = Executors.newFixedThreadPool(browsers.size()); List<Callable<WebDriver>> callables = new ArrayList<Callable<WebDriver>>(); for (final String browser : browsers) { callables.add(new Callable<WebDriver>() { public WebDriver call() throws Exception { return createWebDriver(browserURL, null, browser); } }); } //invoke all the callables, and wait for each thread to return try { List<Future<WebDriver>> futures = service.invokeAll(callables); for (Future<WebDriver> future : futures) { webDrivers.add(future.get()); } } catch (InterruptedException e) { throw new IllegalArgumentException("Error retrieving webdriver", e); } catch (ExecutionException e) { throw new IllegalArgumentException("Error retrieving webdriver", e); } service.shutdown(); return Collections.unmodifiableList(webDrivers); }