List of usage examples for java.util.concurrent ThreadPoolExecutor getCompletedTaskCount
public long getCompletedTaskCount()
From source file:org.apache.hadoop.hbase.regionserver.TestRegionOpen.java
@Test(timeout = 60000) public void testPriorityRegionIsOpenedWithSeparateThreadPool() throws Exception { ThreadPoolExecutor exec = getRS().getExecutorService() .getExecutorThreadPool(ExecutorType.RS_OPEN_PRIORITY_REGION); assertEquals(0, exec.getCompletedTaskCount()); HTableDescriptor htd = new HTableDescriptor(tableName); htd.setPriority(HConstants.HIGH_QOS); htd.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); try (Connection connection = ConnectionFactory.createConnection(HTU.getConfiguration()); Admin admin = connection.getAdmin()) { admin.createTable(htd);//w w w .j a va 2s . c om } assertEquals(1, exec.getCompletedTaskCount()); }
From source file:org.apache.hadoop.hbase.util.FSUtils.java
/** * This function is to scan the root path of the file system to get either the * mapping between the region name and its best locality region server or the * degree of locality of each region on each of the servers having at least * one block of that region. The output map parameters are both optional. * * @param conf//ww w .j av a 2 s . co m * the configuration to use * @param desiredTable * the table you wish to scan locality for * @param threadPoolSize * the thread pool size to use * @param regionToBestLocalityRSMapping * the map into which to put the best locality mapping or null * @param regionDegreeLocalityMapping * the map into which to put the locality degree mapping or null, * must be a thread-safe implementation * @throws IOException * in case of file system errors or interrupts */ private static void getRegionLocalityMappingFromFS(final Configuration conf, final String desiredTable, int threadPoolSize, Map<String, String> regionToBestLocalityRSMapping, Map<String, Map<String, Float>> regionDegreeLocalityMapping) throws IOException { FileSystem fs = FileSystem.get(conf); Path rootPath = FSUtils.getRootDir(conf); long startTime = EnvironmentEdgeManager.currentTimeMillis(); Path queryPath; // The table files are in ${hbase.rootdir}/data/<namespace>/<table>/* if (null == desiredTable) { queryPath = new Path(new Path(rootPath, HConstants.BASE_NAMESPACE_DIR).toString() + "/*/*/*/"); } else { queryPath = new Path(FSUtils.getTableDir(rootPath, TableName.valueOf(desiredTable)).toString() + "/*/"); } // reject all paths that are not appropriate PathFilter pathFilter = new PathFilter() { @Override public boolean accept(Path path) { // this is the region name; it may get some noise data if (null == path) { return false; } // no parent? Path parent = path.getParent(); if (null == parent) { return false; } String regionName = path.getName(); if (null == regionName) { return false; } if (!regionName.toLowerCase().matches("[0-9a-f]+")) { return false; } return true; } }; FileStatus[] statusList = fs.globStatus(queryPath, pathFilter); if (null == statusList) { return; } else { LOG.debug("Query Path: " + queryPath + " ; # list of files: " + statusList.length); } // lower the number of threads in case we have very few expected regions threadPoolSize = Math.min(threadPoolSize, statusList.length); // run in multiple threads ThreadPoolExecutor tpe = new ThreadPoolExecutor(threadPoolSize, threadPoolSize, 60, TimeUnit.SECONDS, new ArrayBlockingQueue<Runnable>(statusList.length)); try { // ignore all file status items that are not of interest for (FileStatus regionStatus : statusList) { if (null == regionStatus) { continue; } if (!regionStatus.isDirectory()) { continue; } Path regionPath = regionStatus.getPath(); if (null == regionPath) { continue; } tpe.execute(new FSRegionScanner(fs, regionPath, regionToBestLocalityRSMapping, regionDegreeLocalityMapping)); } } finally { tpe.shutdown(); int threadWakeFrequency = conf.getInt(HConstants.THREAD_WAKE_FREQUENCY, 60 * 1000); try { // here we wait until TPE terminates, which is either naturally or by // exceptions in the execution of the threads while (!tpe.awaitTermination(threadWakeFrequency, TimeUnit.MILLISECONDS)) { // printing out rough estimate, so as to not introduce // AtomicInteger LOG.info("Locality checking is underway: { Scanned Regions : " + tpe.getCompletedTaskCount() + "/" + tpe.getTaskCount() + " }"); } } catch (InterruptedException e) { throw (InterruptedIOException) new InterruptedIOException().initCause(e); } } long overhead = EnvironmentEdgeManager.currentTimeMillis() - startTime; String overheadMsg = "Scan DFS for locality info takes " + overhead + " ms"; LOG.info(overheadMsg); }
From source file:org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetAsyncDiskService.java
synchronized long countPendingDeletions() { long count = 0; for (ThreadPoolExecutor exec : executors.values()) { count += exec.getTaskCount() - exec.getCompletedTaskCount(); }//from w w w.j ava 2 s .com return count; }
From source file:org.apache.hc.core5.http.benchmark.HttpBenchmark.java
public Results doExecute() throws Exception { final URL url = config.getUrl(); final long endTime = System.currentTimeMillis() + config.getTimeLimit() * 1000; final HttpHost host = new HttpHost(url.getHost(), url.getPort(), url.getProtocol()); final ThreadPoolExecutor workerPool = new ThreadPoolExecutor(config.getThreads(), config.getThreads(), 5, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(), new ThreadFactory() { @Override/*from w w w.j a v a 2s. co m*/ public Thread newThread(final Runnable r) { return new Thread(r, "ClientPool"); } }); workerPool.prestartAllCoreThreads(); SocketFactory socketFactory = null; if ("https".equals(host.getSchemeName())) { final SSLContextBuilder sslContextBuilder = new SSLContextBuilder(); sslContextBuilder.setProtocol("SSL"); if (config.isDisableSSLVerification()) { sslContextBuilder.loadTrustMaterial(null, new TrustStrategy() { @Override public boolean isTrusted(final X509Certificate[] chain, final String authType) throws CertificateException { return true; } }); } else if (config.getTrustStorePath() != null) { sslContextBuilder.loadTrustMaterial(new File(config.getTrustStorePath()), config.getTrustStorePassword() != null ? config.getTrustStorePassword().toCharArray() : null); } if (config.getIdentityStorePath() != null) { sslContextBuilder.loadKeyMaterial(new File(config.getIdentityStorePath()), config.getIdentityStorePassword() != null ? config.getIdentityStorePassword().toCharArray() : null, config.getIdentityStorePassword() != null ? config.getIdentityStorePassword().toCharArray() : null); } final SSLContext sslContext = sslContextBuilder.build(); socketFactory = sslContext.getSocketFactory(); } final BenchmarkWorker[] workers = new BenchmarkWorker[config.getThreads()]; for (int i = 0; i < workers.length; i++) { workers[i] = new BenchmarkWorker(host, createRequest(host), socketFactory, config); workerPool.execute(workers[i]); } while (workerPool.getCompletedTaskCount() < config.getThreads()) { Thread.yield(); try { Thread.sleep(1000); } catch (final InterruptedException ignore) { } if (config.getTimeLimit() != -1 && System.currentTimeMillis() > endTime) { for (int i = 0; i < workers.length; i++) { workers[i].setShutdownSignal(); } } } workerPool.shutdown(); return ResultProcessor.collectResults(workers, host, config.getUrl().toString()); }
From source file:org.apache.http.benchmark.HttpBenchmark.java
public String execute() throws Exception { prepare();/*www .ja va 2 s . c om*/ ThreadPoolExecutor workerPool = new ThreadPoolExecutor(config.getThreads(), config.getThreads(), 5, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(), new ThreadFactory() { public Thread newThread(Runnable r) { return new Thread(r, "ClientPool"); } }); workerPool.prestartAllCoreThreads(); BenchmarkWorker[] workers = new BenchmarkWorker[config.getThreads()]; for (int i = 0; i < workers.length; i++) { workers[i] = new BenchmarkWorker(params, config.getVerbosity(), request[i], host, config.getRequests(), config.isKeepAlive(), config.isDisableSSLVerification(), config.getTrustStorePath(), config.getTrustStorePassword(), config.getIdentityStorePath(), config.getIdentityStorePassword()); workerPool.execute(workers[i]); } while (workerPool.getCompletedTaskCount() < config.getThreads()) { Thread.yield(); try { Thread.sleep(1000); } catch (InterruptedException ignore) { } } workerPool.shutdown(); return ResultProcessor.printResults(workers, host, config.getUrl().toString(), contentLength); }
From source file:org.apache.http.contrib.benchmark.HttpBenchmark.java
private void execute() { prepare();//from www . j a v a2 s . co m ThreadPoolExecutor workerPool = new ThreadPoolExecutor(threads, threads, 5, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(), new ThreadFactory() { public Thread newThread(Runnable r) { return new Thread(r, "ClientPool"); } }); workerPool.prestartAllCoreThreads(); BenchmarkWorker[] workers = new BenchmarkWorker[threads]; for (int i = 0; i < threads; i++) { workers[i] = new BenchmarkWorker(params, verbosity, request[i], host, requests, keepAlive); workerPool.execute(workers[i]); } while (workerPool.getCompletedTaskCount() < threads) { Thread.yield(); try { Thread.sleep(1000); } catch (InterruptedException ignore) { } } workerPool.shutdown(); ResultProcessor.printResults(workers, host, url.toString(), contentLength); }
From source file:org.cesecore.audit.log.LogPerformanceTest.java
@Test public void testSecureLog() throws Exception { log.trace(">testSecureLog"); // Use 1024 SHA512withRSA logManagement.changeLogManagement(roleMgmgToken, createDigSignLogManagementData()); // Export (delete) any old log first final CryptoToken cryptoToken = createTokenWithKeyPair(); for (final String logDeviceId : securityEventsAuditor.getQuerySupportingLogDevices()) { final String exportFilename = securityEventsAuditor.exportAuditLogs(roleMgmgToken, cryptoToken, new Date(), true, keyAlias, keyPairSignAlgorithm, logDeviceId).getExportedFile(); assertExportAndSignatureExists(exportFilename); }/*from w w w . j a v a 2 s . c o m*/ // Log some and measure time final ThreadPoolExecutor workers = (ThreadPoolExecutor) Executors.newFixedThreadPool(THREADS); final long startTimeLog = System.currentTimeMillis(); for (int i = 0; i < WORKERS; i++) { workers.execute(new Runnable() { @Override public void run() { try { securityEventsLogger.log(roleMgmgToken, EventTypes.AUTHENTICATION, EventStatus.SUCCESS, ModuleTypes.AUTHENTICATION, ServiceTypes.CORE); } catch (Exception e) { fail("Logging should work"); } } }); } while (workers.getCompletedTaskCount() < WORKERS && (System.currentTimeMillis() - startTimeLog) < TIMEOUT_MS) { Thread.sleep(250); } final int completedTaskCount = Long.valueOf(workers.getCompletedTaskCount()).intValue(); log.info("securityEventsLogger.log: " + completedTaskCount + " completed in " + (System.currentTimeMillis() - startTimeLog) + " ms using " + THREADS + " threads."); workers.shutdown(); for (final String logDeviceId : securityEventsAuditor.getQuerySupportingLogDevices()) { log.info("using device: " + logDeviceId); // Validate final QueryCriteria criteria = QueryCriteria.create() .add(Criteria.orderDesc(AuditLogEntry.FIELD_TIMESTAMP)); final List<? extends AuditLogEntry> list = securityEventsAuditor.selectAuditLogs(roleMgmgToken, 1, completedTaskCount, criteria, logDeviceId); assertEquals(list.size(), completedTaskCount); final long startTimeVerify = System.currentTimeMillis(); securityEventsAuditor.verifyLogsIntegrity(roleMgmgToken, new Date(), logDeviceId); log.info("securityEventsLogger.verify: " + (System.currentTimeMillis() - startTimeVerify)); // Export final long startTimeExport = System.currentTimeMillis(); final String exportFilename = securityEventsAuditor.exportAuditLogs(roleMgmgToken, cryptoToken, new Date(), true, keyAlias, keyPairSignAlgorithm, logDeviceId).getExportedFile(); log.info("securityEventsLogger.export: " + (System.currentTimeMillis() - startTimeExport)); assertExportAndSignatureExists(exportFilename); } log.trace("<testSecureLog"); }
From source file:org.paxle.filter.robots.impl.RobotsTxtManager.java
/** * {@inheritDoc}// ww w. j a v a2 s . c o m * @see org.osgi.service.monitor.Monitorable#getStatusVariable(String) */ public StatusVariable getStatusVariable(String id) throws IllegalArgumentException { if (!VAR_NAMES.contains(id)) { throw new IllegalArgumentException("Invalid Status Variable name " + id); } int val = 0; int type = StatusVariable.CM_GAUGE; if (id.equals(MONITOR_STORE_SIZE)) { val = this.loader.size(); } else if (id.startsWith(MONITOR_JOBS_PREFIX)) { ThreadPoolExecutor execService = this.execService; if (id.equals(MONITOR_JOBS_ACTIVE)) { val = execService.getActiveCount(); } else if (id.equals(MONITOR_JOBS_IDLE)) { long max = execService.getMaximumPoolSize(); long active = execService.getActiveCount(); val = (int) (max - active); } else if (id.equals(MONITOR_JOBS_MAX)) { val = execService.getMaximumPoolSize(); } else if (id.equals(MONITOR_JOBS_PENDING)) { long enqued = execService.getTaskCount(); long total = execService.getCompletedTaskCount(); long active = execService.getActiveCount(); val = (int) (enqued - total - active); } else if (id.equals(MONITOR_JOBS_TOTAL)) { val = (int) execService.getCompletedTaskCount(); type = StatusVariable.CM_CC; } } return new StatusVariable(id, type, val); }