List of usage examples for java.util.concurrent ExecutorService invokeAll
<T> List<Future<T>> invokeAll(Collection<? extends Callable<T>> tasks) throws InterruptedException;
From source file:org.kuali.student.enrollment.class2.courseoffering.service.impl.TestCourseOfferingCodeGeneratorImpl.java
private void test(final String courseOfferingCode, final int threadCount) throws InterruptedException, ExecutionException { Callable<String> task = new Callable<String>() { @Override//from w ww . j a va 2 s .co m public String call() { return offeringCodeGenerator .generateActivityOfferingCode(_makeDefaultMap(courseOfferingCode, new ArrayList<String>())); } }; List<Callable<String>> tasks = Collections.nCopies(threadCount, task); ExecutorService executorService = Executors.newFixedThreadPool(threadCount); List<Future<String>> futures = executorService.invokeAll(tasks); List<String> resultList = new ArrayList<String>(futures.size()); // Check for exceptions for (Future<String> future : futures) { // Throws an exception if an exception was thrown by the task. resultList.add(future.get()); } // Validate the IDs assert (futures.size() == threadCount); List<String> expectedList = new ArrayList<String>(threadCount); String nextCode = ""; for (long i = 1; i <= threadCount; i++) { nextCode = getNextCode(nextCode); expectedList.add(nextCode); } Collections.sort(resultList); System.out.println("Expected/Got: \n" + expectedList + "\n" + resultList); for (int i = 0; i < resultList.size(); i++) { assertTrue("\nWas expecting \n[" + expectedList.get(i) + "] but got \n[" + resultList.get(i) + "]\n", expectedList.get(i).equals(resultList.get(i))); } }
From source file:com.cloudera.oryx.app.speed.als.ALSSpeedModelManager.java
@Override public Iterable<String> buildUpdates(JavaPairRDD<String, String> newData) { if (model == null || model.getFractionLoaded() < minModelLoadFraction) { return Collections.emptyList(); }//from ww w.j a va2 s .com // Order by timestamp and parse as tuples JavaRDD<String> sortedValues = newData.values().sortBy(MLFunctions.TO_TIMESTAMP_FN, true, newData.partitions().size()); JavaPairRDD<Tuple2<String, String>, Double> tuples = sortedValues.mapToPair(TO_TUPLE_FN); JavaPairRDD<Tuple2<String, String>, Double> aggregated; if (model.isImplicit()) { // See comments in ALSUpdate for explanation of how deletes are handled by this. aggregated = tuples.groupByKey().mapValues(MLFunctions.SUM_WITH_NAN); } else { // For non-implicit, last wins. aggregated = tuples.foldByKey(Double.NaN, Functions.<Double>last()); } Collection<UserItemStrength> input = aggregated.filter(MLFunctions.<Tuple2<String, String>>notNaNValue()) .map(TO_UIS_FN).collect(); final Solver XTXsolver; final Solver YTYsolver; try { XTXsolver = model.getXTXSolver(); YTYsolver = model.getYTYSolver(); } catch (SingularMatrixSolverException smse) { return Collections.emptyList(); } final Collection<String> result = new ArrayList<>(); int numThreads = Runtime.getRuntime().availableProcessors(); Collection<Callable<Void>> tasks = new ArrayList<>(numThreads); final Iterator<UserItemStrength> inputIterator = input.iterator(); for (int i = 0; i < numThreads; i++) { tasks.add(new LoggingVoidCallable() { @Override public void doCall() { while (true) { UserItemStrength uis; synchronized (inputIterator) { if (inputIterator.hasNext()) { uis = inputIterator.next(); } else { break; } } String user = uis.getUser(); String item = uis.getItem(); double value = uis.getStrength(); // Xu is the current row u in the X user-feature matrix float[] Xu = model.getUserVector(user); // Yi is the current row i in the Y item-feature matrix float[] Yi = model.getItemVector(item); float[] newXu = ALSUtils.computeUpdatedXu(YTYsolver, value, Xu, Yi, model.isImplicit()); // Similarly for Y vs X float[] newYi = ALSUtils.computeUpdatedXu(XTXsolver, value, Yi, Xu, model.isImplicit()); if (newXu != null) { String update = toUpdateJSON("X", user, newXu, item); synchronized (result) { result.add(update); } } if (newYi != null) { String update = toUpdateJSON("Y", item, newYi, user); synchronized (result) { result.add(update); } } } } }); } ExecutorService executor = Executors.newFixedThreadPool(numThreads); try { executor.invokeAll(tasks); } catch (InterruptedException ie) { throw new IllegalStateException(ie); } finally { executor.shutdown(); } return result; }
From source file:com.mnxfst.testing.handler.exec.client.PTestPlanExecutorClient.java
/** * Executes the referenced test plan on all given hosts * @param threads//from w w w . j a v a 2s . c om * @param recurrences * @param recurrenceType * @param testplan * @param hosts * @param port * @return */ protected Map<String, String> executeTestplan(int threads, int recurrences, String recurrenceType, byte[] testplan, String[] hosts, int port) { StringBuffer buf = new StringBuffer("/planexec?action=execute"); buf.append("&threads=").append(threads); buf.append("&recurrences=").append(recurrences); buf.append("&recurrenceType=").append(recurrenceType); StringBuffer hn = new StringBuffer(); for (int i = 0; i < hosts.length; i++) { hn.append(hosts[i]); if (i < hosts.length - 1) hn.append(", "); } System.out.println("Execute testplan:"); System.out.println("\thostNames: " + hn.toString()); System.out.println("\tport: " + port); System.out.println("\tthreads: " + threads); System.out.println("\trecurrences: " + recurrences); System.out.println("\trecurrenceType: " + recurrenceType); System.out.println("\n\turi: " + buf.toString()); PTestPlanExecutorClientCallable callables[] = new PTestPlanExecutorClientCallable[hosts.length]; for (int i = 0; i < hosts.length; i++) { callables[i] = new PTestPlanExecutorClientCallable(hosts[i], port, buf.toString(), testplan); } ExecutorService executorService = Executors.newFixedThreadPool(hosts.length); List<Future<NameValuePair>> executionResults = new ArrayList<Future<NameValuePair>>(); try { executionResults = executorService.invokeAll(Arrays.asList(callables)); } catch (InterruptedException e) { System.out.println("Test execution interrupted: " + e.getMessage()); } // collect results from callables Map<String, String> result = new HashMap<String, String>(); for (Future<NameValuePair> r : executionResults) { try { NameValuePair nvp = r.get(); result.put(nvp.getName(), nvp.getValue()); } catch (InterruptedException e) { System.out.println("Interrupted while waiting for results. Error: " + e.getMessage()); e.printStackTrace(); } catch (ExecutionException e) { System.out.println("Interrupted while waiting for results. Error: " + e.getMessage()); e.printStackTrace(); } } return result; }
From source file:com.cloudera.oryx.als.computation.LoadRunner.java
public void runLoad() throws InterruptedException { final StorelessUnivariateStatistic recommendedBecause = new Mean(); final StorelessUnivariateStatistic setPreference = new Mean(); final StorelessUnivariateStatistic removePreference = new Mean(); final StorelessUnivariateStatistic ingest = new Mean(); final StorelessUnivariateStatistic refresh = new Mean(); final StorelessUnivariateStatistic estimatePreference = new Mean(); final StorelessUnivariateStatistic mostSimilarItems = new Mean(); final StorelessUnivariateStatistic similarityToItem = new Mean(); final StorelessUnivariateStatistic mostPopularItems = new Mean(); final StorelessUnivariateStatistic recommendToMany = new Mean(); final StorelessUnivariateStatistic recommend = new Mean(); final RandomGenerator random = RandomManager.getRandom(); int numCores = Runtime.getRuntime().availableProcessors(); final int stepsPerWorker = steps / numCores; Collection<Callable<Object>> workers = Lists.newArrayListWithCapacity(numCores); for (int i = 0; i < numCores; i++) { workers.add(new Callable<Object>() { @Override//from www. ja v a 2 s.co m public Void call() throws Exception { for (int i = 0; i < stepsPerWorker; i++) { double r; String userID; String itemID; String itemID2; float value; synchronized (random) { r = random.nextDouble(); userID = uniqueUserIDs[random.nextInt(uniqueUserIDs.length)]; itemID = uniqueItemIDs[random.nextInt(uniqueItemIDs.length)]; itemID2 = uniqueItemIDs[random.nextInt(uniqueItemIDs.length)]; value = random.nextInt(10); } long stepStart = System.currentTimeMillis(); if (r < 0.05) { client.recommendedBecause(userID, itemID, 10); recommendedBecause.increment(System.currentTimeMillis() - stepStart); } else if (r < 0.07) { client.setPreference(userID, itemID); setPreference.increment(System.currentTimeMillis() - stepStart); } else if (r < 0.08) { client.setPreference(userID, itemID, value); setPreference.increment(System.currentTimeMillis() - stepStart); } else if (r < 0.11) { client.removePreference(userID, itemID); removePreference.increment(System.currentTimeMillis() - stepStart); } else if (r < 0.12) { Reader reader = new StringReader( DelimitedDataUtils.encode(userID, itemID, Float.toString(value)) + '\n'); client.ingest(reader); ingest.increment(System.currentTimeMillis() - stepStart); } else if (r < 0.13) { client.refresh(); refresh.increment(System.currentTimeMillis() - stepStart); } else if (r < 0.14) { client.similarityToItem(itemID, itemID2); similarityToItem.increment(System.currentTimeMillis() - stepStart); } else if (r < 0.15) { client.mostPopularItems(10); mostPopularItems.increment(System.currentTimeMillis() - stepStart); } else if (r < 0.19) { client.estimatePreference(userID, itemID); estimatePreference.increment(System.currentTimeMillis() - stepStart); } else if (r < 0.20) { client.estimateForAnonymous(itemID, new String[] { itemID2 }); estimatePreference.increment(System.currentTimeMillis() - stepStart); } else if (r < 0.25) { client.mostSimilarItems(new String[] { itemID }, 10); mostSimilarItems.increment(System.currentTimeMillis() - stepStart); } else if (r < 0.30) { client.recommendToMany(new String[] { userID, userID }, 10, true, null); recommendToMany.increment(System.currentTimeMillis() - stepStart); } else { client.recommend(userID, 10); recommend.increment(System.currentTimeMillis() - stepStart); } } return null; } }); } log.info("Starting load test..."); long start = System.currentTimeMillis(); ExecutorService executor = Executors.newFixedThreadPool(numCores); Iterable<Future<Object>> futures; try { futures = executor.invokeAll(workers); } finally { ExecutorUtils.shutdownNowAndAwait(executor); } long end = System.currentTimeMillis(); ExecutorUtils.checkExceptions(futures); log.info("Finished {} steps in {}ms", steps, end - start); log.info("recommendedBecause: {}", recommendedBecause.getResult()); log.info("setPreference: {}", setPreference.getResult()); log.info("removePreference: {}", removePreference.getResult()); log.info("ingest: {}", ingest.getResult()); log.info("refresh: {}", refresh.getResult()); log.info("estimatePreference: {}", estimatePreference.getResult()); log.info("mostSimilarItems: {}", mostSimilarItems.getResult()); log.info("similarityToItem: {}", similarityToItem.getResult()); log.info("mostPopularItems: {}", mostPopularItems.getResult()); log.info("recommendToMany: {}", recommendToMany.getResult()); log.info("recommend: {}", recommend.getResult()); }
From source file:org.apache.sysml.runtime.matrix.data.LibMatrixDNN.java
private static void runConvTask(TaskType type, ConvolutionParameters params) throws DMLRuntimeException { int constrainedNumThreads = OptimizerUtils.getConstrainedNumThreads(params.numThreads); ConcurrentLinkedQueue<MatrixBlock> im2ColOutBlocks = new ConcurrentLinkedQueue<MatrixBlock>(); ConcurrentLinkedQueue<MatrixBlock> doutReshapedBlocks = new ConcurrentLinkedQueue<MatrixBlock>(); ConcurrentLinkedQueue<MatrixBlock> partialRetBlocks = new ConcurrentLinkedQueue<MatrixBlock>(); if (ALLOW_MULTI_THREADED_OPS && params.isOutputThreadSafe() && constrainedNumThreads > 1) { int poolSize = Math.min(constrainedNumThreads, params.N); addMatrixBlocks(poolSize, type, params, im2ColOutBlocks, doutReshapedBlocks, partialRetBlocks); ArrayList<ConvTask> tasks = new ArrayList<ConvTask>(); int NSize = params.N - poolSize; if (NSize >= constrainedNumThreads) { for (int n = 0; n < params.N; n++) tasks.add(new ConvTask(n, n + 1, type, params, im2ColOutBlocks, doutReshapedBlocks, partialRetBlocks)); } else {//from w w w.j a va 2 s .co m int numNTasks = (int) Math.ceil(((double) NSize) / constrainedNumThreads); for (int n = 0; n < NSize; n += numNTasks) { tasks.add(new ConvTask(n, Math.min(NSize, n + numNTasks), type, params, im2ColOutBlocks, doutReshapedBlocks, partialRetBlocks)); } for (int n = NSize; n < params.N; n++) tasks.add(new ConvTask(n, n + 1, type, params, im2ColOutBlocks, doutReshapedBlocks, partialRetBlocks)); } ExecutorService pool = Executors.newFixedThreadPool(poolSize); List<Future<Object>> taskret; try { taskret = pool.invokeAll(tasks); pool.shutdown(); for (Future<Object> task : taskret) { task.get(); } if (type == TaskType.LoopedIm2ColConv2dBwdFilter) { for (MatrixBlock partialRetBlock : partialRetBlocks) { elementWiseInPlaceAddition(params.output, partialRetBlock); } } } catch (InterruptedException e) { throw new DMLRuntimeException("Error while executing multi-threaded " + type.name(), e); } catch (ExecutionException e) { throw new DMLRuntimeException("Error while executing multi-threaded " + type.name(), e); } } else { addMatrixBlocks(1, type, params, im2ColOutBlocks, doutReshapedBlocks, partialRetBlocks); ConvTask task = new ConvTask(0, 0, type, params, im2ColOutBlocks, doutReshapedBlocks, partialRetBlocks); try { for (int n = 0; n < params.N; n++) { task.n1 = n; task.n2 = n + 1; task.call(); } if (type == TaskType.LoopedIm2ColConv2dBwdFilter) { for (MatrixBlock partialRetBlock : partialRetBlocks) { elementWiseInPlaceAddition(params.output, partialRetBlock); } } } catch (Exception e) { throw new DMLRuntimeException("Error while executing single-threaded " + type.name(), e); } } }
From source file:com.networknt.client.ClientTest.java
private void callApiAsyncMultiThread(final int threadCount) throws InterruptedException, ExecutionException { Callable<String> task = new Callable<String>() { @Override/*from www . ja v a 2s .com*/ public String call() throws Exception { return callApiAsync(); } }; List<Callable<String>> tasks = Collections.nCopies(threadCount, task); ExecutorService executorService = Executors.newFixedThreadPool(threadCount); List<Future<String>> futures = executorService.invokeAll(tasks); List<String> resultList = new ArrayList<String>(futures.size()); for (Future<String> future : futures) { resultList.add(future.get()); } System.out.println("resultList = " + resultList); }
From source file:org.hashes.collision.AbstractMITMGenerator.java
@Override protected List<String> generateNewCollisions(final int numberOfKeys, final ProgressMonitor monitor) { final int hash = this.getHashAlgorithm().hash(this.seed); final Map<Integer, String> lookupMap = this.createLookupMap(hash); final List<Callable<List<String>>> tasks = this.buildTasks(lookupMap, numberOfKeys, monitor); final ExecutorService executor = Executors.newFixedThreadPool(tasks.size()); try {/*from w w w.j av a 2s.c o m*/ List<Future<List<String>>> results = Collections.emptyList(); try { results = executor.invokeAll(tasks); } finally { executor.shutdown(); } final Builder<String> collisions = ImmutableList.builder(); for (final Future<List<String>> future : results) { collisions.addAll(future.get()); } return collisions.build(); } catch (final Exception e) { throw new ComputationException(e); } }
From source file:com.networknt.client.ClientTest.java
private void callApiSyncMultiThread(final int threadCount) throws InterruptedException, ExecutionException { Callable<String> task = new Callable<String>() { @Override// w w w. j a v a 2 s.co m public String call() throws Exception { return callApiSync(); } }; List<Callable<String>> tasks = Collections.nCopies(threadCount, task); long start = System.currentTimeMillis(); ExecutorService executorService = Executors.newFixedThreadPool(threadCount); List<Future<String>> futures = executorService.invokeAll(tasks); List<String> resultList = new ArrayList<String>(futures.size()); // Check for exceptions for (Future<String> future : futures) { // Throws an exception if an exception was thrown by the task. resultList.add(future.get()); } long last = (System.currentTimeMillis() - start); System.out.println("resultList = " + resultList + " response time = " + last); }
From source file:org.apache.sysml.runtime.matrix.data.LibMatrixDatagen.java
/** * Function to generate a matrix of random numbers. This is invoked both * from CP as well as from MR. In case of CP, it generates an entire matrix * block-by-block. A <code>bigrand</code> is passed so that block-level * seeds are generated internally. In case of MR, it generates a single * block for given block-level seed <code>bSeed</code>. * /* w w w . j a va 2s . com*/ * When pdf="uniform", cell values are drawn from uniform distribution in * range <code>[min,max]</code>. * * When pdf="normal", cell values are drawn from standard normal * distribution N(0,1). The range of generated values will always be * (-Inf,+Inf). * * * @param out output matrix block * @param rgen random matrix generator * @param nnzInBlocks number of non-zeros in blocks * @param bigrand Well1024a pseudo-random number generator * @param bSeed seed for random generator * @param k ? * @throws DMLRuntimeException if DMLRuntimeException occurs */ public static void generateRandomMatrix(MatrixBlock out, RandomMatrixGenerator rgen, long[] nnzInBlocks, Well1024a bigrand, long bSeed, int k) throws DMLRuntimeException { int rows = rgen._rows; int cols = rgen._cols; int rpb = rgen._rowsPerBlock; int cpb = rgen._colsPerBlock; double sparsity = rgen._sparsity; //sanity check valid dimensions and sparsity checkMatrixDimensionsAndSparsity(rows, cols, sparsity); /* * Setup min and max for distributions other than "uniform". Min and Max * are set up in such a way that the usual logic of * (max-min)*prng.nextDouble() is still valid. This is done primarily to * share the same code across different distributions. */ double min = rgen._pdf.equalsIgnoreCase(RAND_PDF_UNIFORM) ? rgen._min : 0; double max = rgen._pdf.equalsIgnoreCase(RAND_PDF_UNIFORM) ? rgen._max : 1; //determine the sparsity of output matrix (multi-threaded always invoked from CP): //estimated NNZ is for entire matrix (nnz=0, if 0 initialized) final long estnnz = ((min == 0.0 && max == 0.0) ? 0 : (long) (sparsity * rows * cols)); boolean lsparse = MatrixBlock.evalSparseFormatInMemory(rows, cols, estnnz); //fallback to sequential if single rowblock or too few cells or if MatrixBlock is not thread safe if (k <= 1 || (rows <= rpb && lsparse) || (long) rows * cols < PAR_NUMCELL_THRESHOLD || !MatrixBlock.isThreadSafe(lsparse)) { generateRandomMatrix(out, rgen, nnzInBlocks, bigrand, bSeed); return; } out.reset(rows, cols, lsparse); //special case shortcuts for efficiency if (rgen._pdf.equalsIgnoreCase(RAND_PDF_UNIFORM)) { if (min == 0.0 && max == 0.0) { //all zeros out.nonZeros = 0; return; } else if (!out.sparse && sparsity == 1.0d && min == max) { //equal values out.reset(out.rlen, out.clen, min); return; } } //allocate memory //note: individual sparse rows are allocated on demand, //for consistency with memory estimates and prevent OOMs. if (out.sparse) out.allocateSparseRowsBlock(); else out.allocateDenseBlock(); int nrb = (int) Math.ceil((double) rows / rpb); int ncb = (int) Math.ceil((double) cols / cpb); //default: parallelization over row blocks, fallback to parallelization //over column blocks if possible and necessary (higher degree of par) boolean parcol = (!out.sparse && nrb < k && ncb > nrb); int parnb = parcol ? ncb : nrb; //generate seeds independent of parallelizations long[] seeds = generateSeedsForCP(bigrand, nrb, ncb); try { ExecutorService pool = Executors.newFixedThreadPool(k); ArrayList<RandTask> tasks = new ArrayList<RandTask>(); int blklen = ((int) (Math.ceil((double) parnb / k))); for (int i = 0; i < k & i * blklen < parnb; i++) { int rl = parcol ? 0 : i * blklen; int ru = parcol ? nrb : Math.min((i + 1) * blklen, parnb); int cl = parcol ? i * blklen : 0; int cu = parcol ? Math.min((i + 1) * blklen, parnb) : ncb; long[] lseeds = sliceSeedsForCP(seeds, rl, ru, cl, cu, nrb, ncb); tasks.add(new RandTask(rl, ru, cl, cu, out, rgen, nnzInBlocks, bSeed, lseeds)); } List<Future<Object>> ret = pool.invokeAll(tasks); pool.shutdown(); //exception propagation in case not all tasks successful for (Future<Object> rc : ret) rc.get(); } catch (Exception e) { throw new DMLRuntimeException(e); } out.recomputeNonZeros(); }
From source file:eu.cloud4soa.tests.TestReqSec_Deploy.java
public void deploy() { Collection<CallableNode> children = new ArrayList<CallableNode>(); for (String applicationInstanceName : applicationInstances.keySet()) { String applicationInstanceUriId = applicationInstances.get(applicationInstanceName); //Creazione Thread... CallableNode callableNode = new CallableNode(BASE_URI, applicationInstanceUriId, getPaaSInstanceUriId(selectedPaaS), getPublicKey(), getSecretKey()); children.add(callableNode);/*from ww w .j a v a 2s . c om*/ } ExecutorService executor = Executors.newFixedThreadPool(numberTests); try { List<Future<Boolean>> invokeAll = executor.invokeAll(children); for (Future<Boolean> future : invokeAll) { while (!future.isDone()) ; Boolean get = future.get(); } System.out.print("All " + numberTests + " deploy requests are completed!"); executor.shutdownNow(); } catch (InterruptedException ex) { Logger.getLogger(TestReqSec_Deploy.class.getName()).log(Level.SEVERE, null, ex); } catch (ExecutionException ex) { Logger.getLogger(TestReqSec_Deploy.class.getName()).log(Level.SEVERE, null, ex); } // //Applications // try { // deployApplication(applicationInstanceUriId); // } catch (FileNotFoundException ex) { // Logger.getLogger(TestReqSec_Deploy.class.getName()).log(Level.SEVERE, null, ex); // } }