List of usage examples for java.util.concurrent ExecutorService shutdown
void shutdown();
From source file:com.linkedin.pinot.tools.segment.converter.DictionaryToRawIndexConverter.java
/** * Method to perform the conversion for a set of segments in the {@link #_dataDir} * * @return True if successful, False otherwise * @throws Exception//from w w w . ja v a 2 s .co m */ public boolean convert() throws Exception { if (_help) { printUsage(); return true; } File dataDir = new File(_dataDir); File outputDir = new File(_outputDir); if (!dataDir.exists()) { LOGGER.error("Data directory '{}' does not exist.", _dataDir); return false; } else if (outputDir.exists()) { if (_overwrite) { LOGGER.info("Overwriting existing output directory '{}'", _outputDir); FileUtils.deleteQuietly(outputDir); outputDir = new File(_outputDir); outputDir.mkdir(); } else { LOGGER.error("Output directory '{}' already exists, use -overwrite to overwrite", outputDir); return false; } } File[] segmentFiles = dataDir.listFiles(); if (segmentFiles == null || segmentFiles.length == 0) { LOGGER.error("Empty data directory '{}'.", _dataDir); return false; } boolean ret = true; final File outDir = outputDir; ExecutorService executorService = Executors.newFixedThreadPool(_numThreads); for (final File segmentDir : segmentFiles) { executorService.execute(new Runnable() { @Override public void run() { try { convertSegment(segmentDir, _columns.split("\\s*,\\s*"), outDir, _compressOutput); } catch (Exception e) { LOGGER.error("Exception caught while converting segment {}", segmentDir.getName(), e); e.printStackTrace(); } } }); } executorService.shutdown(); executorService.awaitTermination(1, TimeUnit.HOURS); return ret; }
From source file:com.github.tteofili.looseen.Test20NewsgroupsClassification.java
@Test public void test20Newsgroups() throws Exception { String indexProperty = System.getProperty("index"); if (indexProperty != null) { try {//from w ww .j a va2 s . co m index = Boolean.valueOf(indexProperty); } catch (Exception e) { // ignore } } String splitProperty = System.getProperty("split"); if (splitProperty != null) { try { split = Boolean.valueOf(splitProperty); } catch (Exception e) { // ignore } } Path mainIndexPath = Paths.get(INDEX + "/original"); Directory directory = FSDirectory.open(mainIndexPath); Path trainPath = Paths.get(INDEX + "/train"); Path testPath = Paths.get(INDEX + "/test"); Path cvPath = Paths.get(INDEX + "/cv"); FSDirectory cv = null; FSDirectory test = null; FSDirectory train = null; IndexReader testReader = null; if (split) { cv = FSDirectory.open(cvPath); test = FSDirectory.open(testPath); train = FSDirectory.open(trainPath); } if (index) { delete(mainIndexPath); if (split) { delete(trainPath, testPath, cvPath); } } IndexReader reader = null; List<Classifier<BytesRef>> classifiers = new LinkedList<>(); try { Analyzer analyzer = new StandardAnalyzer(); if (index) { System.out.format("Indexing 20 Newsgroups...%n"); long startIndex = System.currentTimeMillis(); IndexWriter indexWriter = new IndexWriter(directory, new IndexWriterConfig(analyzer)); buildIndex(new File(PREFIX + "/20n/20_newsgroups"), indexWriter); long endIndex = System.currentTimeMillis(); System.out.format("Indexed %d pages in %ds %n", indexWriter.maxDoc(), (endIndex - startIndex) / 1000); indexWriter.close(); } if (split && !index) { reader = DirectoryReader.open(train); } else { reader = DirectoryReader.open(directory); } if (index && split) { // split the index System.out.format("Splitting the index...%n"); long startSplit = System.currentTimeMillis(); DatasetSplitter datasetSplitter = new DatasetSplitter(0.1, 0); datasetSplitter.split(reader, train, test, cv, analyzer, false, CATEGORY_FIELD, BODY_FIELD, SUBJECT_FIELD, CATEGORY_FIELD); reader.close(); reader = DirectoryReader.open(train); // using the train index from now on long endSplit = System.currentTimeMillis(); System.out.format("Splitting done in %ds %n", (endSplit - startSplit) / 1000); } final long startTime = System.currentTimeMillis(); classifiers.add(new KNearestNeighborClassifier(reader, new ClassicSimilarity(), analyzer, null, 1, 0, 0, CATEGORY_FIELD, BODY_FIELD)); classifiers.add(new KNearestNeighborClassifier(reader, null, analyzer, null, 1, 0, 0, CATEGORY_FIELD, BODY_FIELD)); classifiers.add(new KNearestNeighborClassifier(reader, new ClassicSimilarity(), analyzer, null, 3, 0, 0, CATEGORY_FIELD, BODY_FIELD)); classifiers.add(new KNearestNeighborClassifier(reader, new AxiomaticF1EXP(), analyzer, null, 3, 0, 0, CATEGORY_FIELD, BODY_FIELD)); classifiers.add(new KNearestNeighborClassifier(reader, new AxiomaticF1LOG(), analyzer, null, 3, 0, 0, CATEGORY_FIELD, BODY_FIELD)); classifiers.add(new KNearestNeighborClassifier(reader, new LMDirichletSimilarity(), analyzer, null, 3, 1, 1, CATEGORY_FIELD, BODY_FIELD)); classifiers.add(new KNearestNeighborClassifier(reader, new LMJelinekMercerSimilarity(0.3f), analyzer, null, 3, 1, 1, CATEGORY_FIELD, BODY_FIELD)); classifiers.add(new KNearestNeighborClassifier(reader, null, analyzer, null, 3, 1, 1, CATEGORY_FIELD, BODY_FIELD)); classifiers.add(new KNearestNeighborClassifier(reader, new DFRSimilarity(new BasicModelG(), new AfterEffectB(), new NormalizationH1()), analyzer, null, 3, 1, 1, CATEGORY_FIELD, BODY_FIELD)); classifiers.add(new KNearestNeighborClassifier(reader, new DFRSimilarity(new BasicModelP(), new AfterEffectL(), new NormalizationH3()), analyzer, null, 3, 1, 1, CATEGORY_FIELD, BODY_FIELD)); classifiers.add(new KNearestNeighborClassifier(reader, new IBSimilarity(new DistributionSPL(), new LambdaDF(), new Normalization.NoNormalization()), analyzer, null, 3, 1, 1, CATEGORY_FIELD, BODY_FIELD)); classifiers.add(new KNearestNeighborClassifier(reader, new IBSimilarity(new DistributionLL(), new LambdaTTF(), new NormalizationH1()), analyzer, null, 3, 1, 1, CATEGORY_FIELD, BODY_FIELD)); classifiers.add(new MinHashClassifier(reader, BODY_FIELD, CATEGORY_FIELD, 15, 1, 100)); classifiers.add(new MinHashClassifier(reader, BODY_FIELD, CATEGORY_FIELD, 30, 3, 300)); classifiers.add(new MinHashClassifier(reader, BODY_FIELD, CATEGORY_FIELD, 10, 1, 100)); classifiers.add(new KNearestFuzzyClassifier(reader, new LMJelinekMercerSimilarity(0.3f), analyzer, null, 1, CATEGORY_FIELD, BODY_FIELD)); classifiers.add(new KNearestFuzzyClassifier(reader, new IBSimilarity(new DistributionLL(), new LambdaTTF(), new NormalizationH1()), analyzer, null, 1, CATEGORY_FIELD, BODY_FIELD)); classifiers.add(new KNearestFuzzyClassifier(reader, new ClassicSimilarity(), analyzer, null, 1, CATEGORY_FIELD, BODY_FIELD)); classifiers.add(new KNearestFuzzyClassifier(reader, new ClassicSimilarity(), analyzer, null, 3, CATEGORY_FIELD, BODY_FIELD)); classifiers .add(new KNearestFuzzyClassifier(reader, null, analyzer, null, 1, CATEGORY_FIELD, BODY_FIELD)); classifiers .add(new KNearestFuzzyClassifier(reader, null, analyzer, null, 3, CATEGORY_FIELD, BODY_FIELD)); classifiers.add(new KNearestFuzzyClassifier(reader, new AxiomaticF1EXP(), analyzer, null, 3, CATEGORY_FIELD, BODY_FIELD)); classifiers.add(new KNearestFuzzyClassifier(reader, new AxiomaticF1LOG(), analyzer, null, 3, CATEGORY_FIELD, BODY_FIELD)); classifiers.add(new BM25NBClassifier(reader, analyzer, null, CATEGORY_FIELD, BODY_FIELD)); classifiers.add(new CachingNaiveBayesClassifier(reader, analyzer, null, CATEGORY_FIELD, BODY_FIELD)); classifiers.add(new SimpleNaiveBayesClassifier(reader, analyzer, null, CATEGORY_FIELD, BODY_FIELD)); int maxdoc; if (split) { testReader = DirectoryReader.open(test); maxdoc = testReader.maxDoc(); } else { maxdoc = reader.maxDoc(); } System.out.format("Starting evaluation on %d docs...%n", maxdoc); ExecutorService service = Executors.newCachedThreadPool(); List<Future<String>> futures = new LinkedList<>(); for (Classifier<BytesRef> classifier : classifiers) { testClassifier(reader, startTime, testReader, service, futures, classifier); } for (Future<String> f : futures) { System.out.println(f.get()); } Thread.sleep(10000); service.shutdown(); } finally { if (reader != null) { reader.close(); } directory.close(); if (test != null) { test.close(); } if (train != null) { train.close(); } if (cv != null) { cv.close(); } if (testReader != null) { testReader.close(); } for (Classifier c : classifiers) { if (c instanceof Closeable) { ((Closeable) c).close(); } } } }
From source file:com.netflix.curator.framework.recipes.locks.TestInterProcessSemaphore.java
@Test public void testThreads() throws Exception { final int THREAD_QTY = 10; CuratorFramework client = CuratorFrameworkFactory.newClient(server.getConnectString(), new RetryOneTime(1)); client.start();/*from ww w . j a v a2 s . c om*/ try { final InterProcessSemaphoreV2 semaphore = new InterProcessSemaphoreV2(client, "/test", 1); ExecutorService service = Executors.newFixedThreadPool(THREAD_QTY); for (int i = 0; i < THREAD_QTY; ++i) { service.submit(new Callable<Object>() { @Override public Object call() throws Exception { Lease lease = semaphore.acquire(); try { Thread.sleep(1); } finally { lease.close(); } return null; } }); } service.shutdown(); Assert.assertTrue(service.awaitTermination(10, TimeUnit.SECONDS)); } finally { client.close(); } }
From source file:demo.vmware.commands.CommandGetAllRegions.java
/** * Retrieve the contents for all regions, one region per task executor *//* www. j a va 2 s . co m*/ @Override public CommandResult run(ConfigurableApplicationContext mainContext, List<String> parameters) { // Use the template for this meaning we can only fetch the whole contents of any region we have a template for // CommandGetCounts goes right to the cache so it may list more regions Map<String, GemfireTemplate> allRegionTemplates = CommandRegionUtils.getAllGemfireTemplates(mainContext); // use the Java executor service because of it's awesome invokeAll method. ExecutorService taskExecutor = Executors.newFixedThreadPool(allRegionTemplates.size()); Collection tasks = new ArrayList<RegionFetcher>(); CommandTimer timer = new CommandTimer(); for (String key : allRegionTemplates.keySet()) { GemfireTemplate oneTemplate = allRegionTemplates.get(key); if (parallelFetch) { tasks.add(new RegionFetcher(oneTemplate.getRegion().getName(), 0, oneTemplate)); } else { // don't write anything out and don't capture results fetchOneRegion(oneTemplate.getRegion().getName(), 5, oneTemplate); } } if (parallelFetch) { // invokeAll() returns when all tasks are complete try { List<Future<?>> futures = taskExecutor.invokeAll(tasks); taskExecutor.shutdown(); LOG.info("Fetched " + futures.size() + " regions in threads"); // the futures hold the results at this point futures.get(X).get(); } catch (InterruptedException e) { e.printStackTrace(); } } timer.stop(); return new CommandResult(null, "Loading all regions took " + timer.getTimeDiffInSeconds() + " seconds"); }
From source file:com.vmware.bdd.utils.CommonUtil.java
/** * * @param host//from w ww .j av a2 s. c om * @param port * @param waitTime */ public static boolean checkServerConnection(final String host, final int port, int waitTime) { boolean connectResult = false; // validate the server is reachable ExecutorService exec = Executors.newFixedThreadPool(1); try { // if the target ip does not exist or the host is shutdown, it will take about 2 minutes // for the socket connection to time out. // here we fork a child thread to do the actual connecting action, if it does not succeed // within given waitTime, we will consider it to be failure. Future<Boolean> futureResult = exec.submit(new Callable<Boolean>() { @Override public Boolean call() throws Exception { try { new Socket(host, port); return true; } catch (Exception e) { } return false; } }); // wait for the thread to finish within given waitTime Boolean result = (Boolean) waitForThreadResult(futureResult, waitTime); // result==null means the time out occurs if (null != result) { connectResult = result; } } catch (Exception e) { logger.error("Unexpected error occurred with threading."); } finally { if (null != exec) { exec.shutdown(); } } return connectResult; }
From source file:info.magnolia.imaging.caching.CachingImageStreamerRepositoryTest.java
@Test public void testRequestForSimilarUncachedImageOnlyGeneratesItOnce() throws Exception { final HierarchyManager srcHM = MgnlContext.getHierarchyManager("website"); final String srcPath = "/foo/bar"; ContentUtil.createPath(srcHM, srcPath); // ParameterProvider for tests - return a new instance of the same node everytime // if we'd return the same src instance everytime, the purpose of this test would be null final ParameterProviderFactory<Object, Content> ppf = new TestParameterProviderFactory(srcHM, srcPath); final OutputFormat png = new OutputFormat(); png.setFormatName("png"); final BufferedImage dummyImg = ImageIO.read(getClass().getResourceAsStream("/funnel.gif")); assertNotNull("Couldn't load dummy test image", dummyImg); final ImageGenerator<ParameterProvider<Content>> generator = mock(ImageGenerator.class); when(generator.getParameterProviderFactory()).thenReturn(ppf); when(generator.getName()).thenReturn("test"); when(generator.getOutputFormat(isA(ParameterProvider.class))).thenReturn(png); // aaaaand finally, here's the real reason for this test ! when(generator.generate(isA(ParameterProvider.class))).thenReturn(dummyImg); // yeah, we're using a "wrong" workspace for the image cache, to avoid having to setup a custom one in this test final HierarchyManager hm = MgnlContext.getHierarchyManager("config"); final ImageStreamer streamer = new CachingImageStreamer(hm, ppf.getCachingStrategy(), new DefaultImageStreamer()); // Generator instances will always be the same (including paramProvFac) // since they are instantiated with the module config and c2b. // ParamProv is a new instance every time. // streamer can (must) be the same - once single HM, one cache. // thread pool of 10, launching 8 requests, can we hit some concurrency please ? final ExecutorService executor = Executors.newFixedThreadPool(10); final ByteArrayOutputStream[] outs = new ByteArrayOutputStream[8]; final Future[] futures = new Future[8]; for (int i = 0; i < outs.length; i++) { outs[i] = new ByteArrayOutputStream(); futures[i] = executor.submit(new TestJob(generator, streamer, outs[i])); }/*from w w w. j av a2 s . c o m*/ executor.shutdown(); executor.awaitTermination(30, TimeUnit.SECONDS); for (Future<?> future : futures) { assertTrue(future.isDone()); assertFalse(future.isCancelled()); // ignore the results of TestJob - all we care about is if an exception was thrown // and if there was any, it is kept in Future until we call Future.get() future.get(); } final NodeData cachedNodeData = hm.getNodeData("/test/website/foo/bar/generated-image"); // update node meta data Content cachedNode = hm.getContent("/test/website/foo/bar"); cachedNode.getMetaData().setModificationDate(); cachedNode.save(); final InputStream res = cachedNodeData.getStream(); final ByteArrayOutputStream cachedOut = new ByteArrayOutputStream(); IOUtils.copy(res, cachedOut); // assert all outs are the same for (int i = 1; i < outs.length; i++) { // TODO assert they're all equals byte to byte to the source? or in size? can't as-is since we convert... final byte[] a = outs[i - 1].toByteArray(); final byte[] b = outs[i].toByteArray(); assertTrue(a.length > 0); assertEquals("Different sizes (" + Math.abs(a.length - b.length) + " bytes diff.) with i=" + i, a.length, b.length); assertTrue("not equals for outs/" + i, Arrays.equals(a, b)); outs[i - 1] = null; // cleanup all those byte[], or we'll soon run out of memory } assertTrue("failed comparing last thread's result with what we got from hierarchyManager", Arrays.equals(outs[outs.length - 1].toByteArray(), cachedOut.toByteArray())); outs[outs.length - 1] = null; // now start again another bunch of requests... they should ALL get their results from the cache final ExecutorService executor2 = Executors.newFixedThreadPool(10); final ByteArrayOutputStream[] outs2 = new ByteArrayOutputStream[8]; final Future[] futures2 = new Future[8]; for (int i = 0; i < outs2.length; i++) { outs2[i] = new ByteArrayOutputStream(); futures2[i] = executor2.submit(new TestJob(generator, streamer, outs2[i])); } executor2.shutdown(); executor2.awaitTermination(30, TimeUnit.SECONDS); for (Future<?> future : futures2) { assertTrue(future.isDone()); assertFalse(future.isCancelled()); // ignore the results of TestJob - all we care about is if an exception was thrown // and if there was any, it is kept in Future until we call Future.get() future.get(); } final NodeData cachedNodeData2 = hm.getNodeData("/test/website/foo/bar/generated-image"); final InputStream res2 = cachedNodeData2.getStream(); final ByteArrayOutputStream cachedOut2 = new ByteArrayOutputStream(); IOUtils.copy(res2, cachedOut2); // assert all outs are the same for (int i = 1; i < outs2.length; i++) { // TODO assert they're all equals byte to byte to the source? or in size? can't as-is since we re-save.. final byte[] a = outs2[i - 1].toByteArray(); final byte[] b = outs2[i].toByteArray(); assertTrue(a.length > 0); assertEquals("Different sizes (" + Math.abs(a.length - b.length) + " bytes diff.) with i=" + i, a.length, b.length); assertTrue("not equals for outs2/" + i, Arrays.equals(a, b)); outs2[i - 1] = null; } assertTrue("failed comparing last thread's result with what we got from hierarchyManager", Arrays.equals(outs2[outs2.length - 1].toByteArray(), cachedOut2.toByteArray())); outs2[outs2.length - 1] = null; }
From source file:com.ibm.bi.dml.runtime.io.ReaderTextCSVParallel.java
/** * //from w w w. j ava 2s. c o m * @param path * @param job * @param dest * @param rlen * @param clen * @param brlen * @param bclen * @param hasHeader * @param delim * @param fill * @param fillValue * @return * @throws IOException */ private void readCSVMatrixFromHDFS(InputSplit[] splits, Path path, JobConf job, MatrixBlock dest, long rlen, long clen, int brlen, int bclen, boolean hasHeader, String delim, boolean fill, double fillValue) throws IOException { FileInputFormat.addInputPath(job, path); TextInputFormat informat = new TextInputFormat(); informat.configure(job); ExecutorService pool = Executors.newFixedThreadPool(_numThreads); try { // create read tasks for all splits ArrayList<CSVReadTask> tasks = new ArrayList<CSVReadTask>(); int splitCount = 0; for (InputSplit split : splits) { tasks.add(new CSVReadTask(split, _offsets, informat, job, dest, rlen, clen, hasHeader, delim, fill, fillValue, splitCount++)); } pool.invokeAll(tasks); pool.shutdown(); // check return codes and aggregate nnz long lnnz = 0; for (CSVReadTask rt : tasks) { lnnz += rt.getPartialNnz(); if (!rt.getReturnCode()) { Exception err = rt.getException(); throw new IOException("Read task for csv input failed: " + err.toString(), err); } } dest.setNonZeros(lnnz); } catch (Exception e) { throw new IOException("Threadpool issue, while parallel read.", e); } }
From source file:it.wami.map.mongodeploy.OsmSaxHandler.java
@Override public void endElement(String uri, String localName, String qName) throws SAXException { super.endElement(uri, localName, qName); if (qName == NODE) { save(entry, COLL_NODES);/* ww w . j av a2s .co m*/ } if (qName == WAY) { if (!nodesQueue.isEmpty()) { System.out.println("remaining nodes: " + nodesQueue.size()); saveEntry(nodesQueue, COLL_NODES); } if (options.isWayGeometry()) { populateWayGeo((Way) entry); } else { standardWay((Way) entry); } } if (qName == RELATION) { if (!waysRunnables.isEmpty()) { System.out.println("remaining ways: " + waysRunnables.size()); int cores = Runtime.getRuntime().availableProcessors(); ExecutorService executorService = Executors.newFixedThreadPool(cores); for (Runnable currentRunnable : waysRunnables) { executorService.execute(currentRunnable); } executorService.shutdown(); while (!executorService.isTerminated()) { } waysRunnables.clear(); saveEntry(waysQueue, COLL_WAYS); //saveEntry(waysQueue, COLL_WAYS); } if (!waysQueue.isEmpty()) { System.out.println("remaining ways: " + waysRunnables.size()); saveEntry(waysQueue, COLL_WAYS); } if (options.isRelationGeometry()) { populateRelation((Relation) entry); } else { save(entry, COLL_RELATIONS); } } }
From source file:com.yoshio3.modules.AzureADServerAuthModule.java
private AuthenticationResult getAccessToken(AuthorizationCode authorizationCode, String currentUri) throws Throwable { String authCode = authorizationCode.getValue(); ClientCredential credential = new ClientCredential(clientId, secretKey); AuthenticationContext context;// w w w.j ava 2 s.c o m AuthenticationResult result; ExecutorService service = null; try { service = Executors.newFixedThreadPool(1); context = new AuthenticationContext(authority + tenant + "/", true, service); Future<AuthenticationResult> future = context.acquireTokenByAuthorizationCode(authCode, new URI(currentUri), credential, null); result = future.get(); } catch (ExecutionException e) { throw e.getCause(); } finally { if (service != null) { service.shutdown(); } } if (result == null) { throw new ServiceUnavailableException("authentication result was null"); } return result; }
From source file:it.wami.map.mongodeploy.OsmSaxHandler.java
@Override public void endDocument() throws SAXException { super.endDocument(); if (nodesQueue.size() > 0) { System.out.println("remaining nodes: " + nodesQueue.size()); saveEntry(nodesQueue, COLL_NODES); }/*from www .java2 s . c o m*/ if (waysQueue.size() > 0) { System.out.println("remaining ways: " + waysQueue.size()); saveEntry(waysQueue, COLL_WAYS); } if (!relationRunnables.isEmpty()) { if (!relationRunnables.isEmpty()) { int cores = Runtime.getRuntime().availableProcessors(); ExecutorService executorService = Executors.newFixedThreadPool(cores); for (Runnable currentRunnable : relationRunnables) { executorService.execute(currentRunnable); } executorService.shutdown(); while (!executorService.isTerminated()) { } relationRunnables.clear(); System.out.println("remaining relations: " + relationsQueue.size()); saveEntry(relationsQueue, COLL_RELATIONS); } } if (!relationsQueue.isEmpty()) { System.out.println("remaining relations: " + relationsQueue.size()); saveEntry(relationsQueue, COLL_RELATIONS); } if (tagsQueue.size() > 0) { System.out.println("remaining tags: " + tagsQueue.size()); saveEntry(tagsQueue, COLL_TAGS); } end = System.currentTimeMillis(); long time = end - start; System.out.println( "End of document; time - " + (time / (60 * 60 * 1000)) % 60 + "h, " + (time / (60 * 1000)) % 60 + "m, " + (time / 1000) % 60 + "s, " + time % 1000 + "ms (" + (end - start) + ")"); }