List of usage examples for java.util.concurrent BlockingQueue add
boolean add(E e);
From source file:org.apache.hive.ptest.execution.HostExecutor.java
/** * Executes parallel test until the parallel work queue is empty. Then * executes the isolated tests on the host. During each phase if a * AbortDroneException is thrown the drone is removed possibly * leaving this host with zero functioning drones. If all drones * are removed the host will be replaced before the next run. *//*from www .j a va 2s .co m*/ private void executeTests(final BlockingQueue<TestBatch> parallelWorkQueue, final BlockingQueue<TestBatch> isolatedWorkQueue, final Set<TestBatch> failedTestResults) throws Exception { if (mShutdown) { mLogger.warn("Shutting down host " + mHost.getName()); return; } mLogger.info("Starting parallel execution on " + mHost.getName()); List<ListenableFuture<Void>> droneResults = Lists.newArrayList(); for (final Drone drone : ImmutableList.copyOf(mDrones)) { droneResults.add(mExecutor.submit(new Callable<Void>() { @Override public Void call() throws Exception { TestBatch batch = null; Stopwatch sw = Stopwatch.createUnstarted(); try { do { batch = parallelWorkQueue.poll(mNumPollSeconds, TimeUnit.SECONDS); if (mShutdown) { mLogger.warn("Shutting down host " + mHost.getName()); return null; } if (batch != null) { numParallelBatchesProcessed++; sw.reset().start(); try { if (!executeTestBatch(drone, batch, failedTestResults)) { failedTestResults.add(batch); } } finally { sw.stop(); mLogger.info( "Finished processing parallel batch [{}] on host {}. ElapsedTime(ms)={}", new Object[] { batch.getName(), getHost().toShortString(), sw.elapsed(TimeUnit.MILLISECONDS) }); } } } while (!mShutdown && !parallelWorkQueue.isEmpty()); } catch (AbortDroneException ex) { mDrones.remove(drone); // return value not checked due to concurrent access mLogger.error("Aborting drone during parallel execution", ex); if (batch != null) { Preconditions.checkState(parallelWorkQueue.add(batch), "Could not add batch to parallel queue " + batch); } } return null; } })); } if (mShutdown) { mLogger.warn("Shutting down host " + mHost.getName()); return; } Futures.allAsList(droneResults).get(); mLogger.info("Starting isolated execution on " + mHost.getName()); for (Drone drone : ImmutableList.copyOf(mDrones)) { TestBatch batch = null; Stopwatch sw = Stopwatch.createUnstarted(); try { do { batch = isolatedWorkQueue.poll(mNumPollSeconds, TimeUnit.SECONDS); if (batch != null) { numIsolatedBatchesProcessed++; sw.reset().start(); try { if (!executeTestBatch(drone, batch, failedTestResults)) { failedTestResults.add(batch); } } finally { sw.stop(); mLogger.info("Finished processing isolated batch [{}] on host {}. ElapsedTime(ms)={}", new Object[] { batch.getName(), getHost().toShortString(), sw.elapsed(TimeUnit.MILLISECONDS) }); } } } while (!mShutdown && !isolatedWorkQueue.isEmpty()); } catch (AbortDroneException ex) { mDrones.remove(drone); // return value not checked due to concurrent access mLogger.error("Aborting drone during isolated execution", ex); if (batch != null) { Preconditions.checkState(isolatedWorkQueue.add(batch), "Could not add batch to isolated queue " + batch); } } } }
From source file:alluxio.master.file.DefaultFileSystemMaster.java
/** * Checks the consistency of the root in a multi-threaded and incremental fashion. This method * will only READ lock the directories and files actively being checked and release them after the * check on the file / directory is complete. * * @return a list of paths in Alluxio which are not consistent with the under storage * @throws InterruptedException if the thread is interrupted during execution * @throws IOException if an error occurs interacting with the under storage *///from w w w . j a v a 2 s . c om private List<AlluxioURI> startupCheckConsistency(final ExecutorService service) throws InterruptedException, IOException { /** A marker {@link StartupConsistencyChecker}s add to the queue to signal completion */ final long completionMarker = -1; /** A shared queue of directories which have yet to be checked */ final BlockingQueue<Long> dirsToCheck = new LinkedBlockingQueue<>(); /** * A {@link Callable} which checks the consistency of a directory. */ final class StartupConsistencyChecker implements Callable<List<AlluxioURI>> { /** The path to check, guaranteed to be a directory in Alluxio. */ private final Long mFileId; /** * Creates a new callable which checks the consistency of a directory. * @param fileId the path to check */ private StartupConsistencyChecker(Long fileId) { mFileId = fileId; } /** * Checks the consistency of the directory and all immediate children which are files. All * immediate children which are directories are added to the shared queue of directories to * check. The parent directory is READ locked during the entire call while the children are * READ locked only during the consistency check of the children files. * * @return a list of inconsistent uris * @throws IOException if an error occurs interacting with the under storage */ @Override public List<AlluxioURI> call() throws IOException { List<AlluxioURI> inconsistentUris = new ArrayList<>(); try (LockedInodePath dir = mInodeTree.lockFullInodePath(mFileId, InodeTree.LockMode.READ)) { Inode parentInode = dir.getInode(); AlluxioURI parentUri = dir.getUri(); if (!checkConsistencyInternal(parentInode, parentUri)) { inconsistentUris.add(parentUri); } for (Inode childInode : ((InodeDirectory) parentInode).getChildren()) { try { childInode.lockReadAndCheckParent(parentInode); } catch (InvalidPathException e) { // This should be safe, continue. LOG.debug("Error during startup check consistency, ignoring and continuing.", e); continue; } try { AlluxioURI childUri = parentUri.join(childInode.getName()); if (childInode.isDirectory()) { dirsToCheck.add(childInode.getId()); } else { if (!checkConsistencyInternal(childInode, childUri)) { inconsistentUris.add(childUri); } } } finally { childInode.unlockRead(); } } } catch (FileDoesNotExistException e) { // This should be safe, continue. LOG.debug("A file scheduled for consistency check was deleted before the check."); } catch (InvalidPathException e) { // This should not happen. LOG.error("An invalid path was discovered during the consistency check, skipping.", e); } dirsToCheck.add(completionMarker); return inconsistentUris; } } // Add the root to the directories to check. dirsToCheck.add(mInodeTree.getRoot().getId()); List<Future<List<AlluxioURI>>> results = new ArrayList<>(); // Tracks how many checkers have been started. long started = 0; // Tracks how many checkers have completed. long completed = 0; do { Long fileId = dirsToCheck.take(); if (fileId == completionMarker) { // A thread signaled completion. completed++; } else { // A new directory needs to be checked. StartupConsistencyChecker checker = new StartupConsistencyChecker(fileId); results.add(service.submit(checker)); started++; } } while (started != completed); // Return the total set of inconsistent paths discovered. List<AlluxioURI> inconsistentUris = new ArrayList<>(); for (Future<List<AlluxioURI>> result : results) { try { inconsistentUris.addAll(result.get()); } catch (Exception e) { // This shouldn't happen, all futures should be complete. Throwables.propagate(e); } } service.shutdown(); return inconsistentUris; }
From source file:org.apache.reef.io.network.NetworkServiceTest.java
/** * NetworkService messaging rate benchmark. *//*from ww w .j a v a 2s .com*/ @Test public void testMessagingNetworkServiceRateDisjoint() throws Exception { Assume.assumeFalse("Use log level INFO to run benchmarking", LOG.isLoggable(Level.FINEST)); LOG.log(Level.FINEST, name.getMethodName()); final IdentifierFactory factory = new StringIdentifierFactory(); final Injector injector = Tang.Factory.getTang().newInjector(); injector.bindVolatileParameter(NameServerParameters.NameServerIdentifierFactory.class, factory); injector.bindVolatileInstance(LocalAddressProvider.class, this.localAddressProvider); try (final NameServer server = injector.getInstance(NameServer.class)) { final int nameServerPort = server.getPort(); final BlockingQueue<Object> barrier = new LinkedBlockingQueue<>(); final int numThreads = 4; final int size = 2000; final int numMessages = 300000 / (Math.max(1, size / 512)); final int totalNumMessages = numMessages * numThreads; final ExecutorService e = Executors.newCachedThreadPool(); for (int t = 0; t < numThreads; t++) { final int tt = t; e.submit(new Runnable() { @Override public void run() { try { final Monitor monitor = new Monitor(); // network service final String name2 = "task2-" + tt; final String name1 = "task1-" + tt; final Configuration nameResolverConf = Tang.Factory.getTang() .newConfigurationBuilder(NameResolverConfiguration.CONF .set(NameResolverConfiguration.NAME_SERVER_HOSTNAME, localAddress) .set(NameResolverConfiguration.NAME_SERVICE_PORT, nameServerPort) .build()) .build(); final Injector injector = Tang.Factory.getTang().newInjector(nameResolverConf); LOG.log(Level.FINEST, "=== Test network service receiver start"); LOG.log(Level.FINEST, "=== Test network service sender start"); try (final NameResolver nameResolver = injector.getInstance(NameResolver.class)) { injector.bindVolatileParameter( NetworkServiceParameters.NetworkServiceIdentifierFactory.class, factory); injector.bindVolatileInstance(NameResolver.class, nameResolver); injector.bindVolatileParameter(NetworkServiceParameters.NetworkServiceCodec.class, new StringCodec()); injector.bindVolatileParameter( NetworkServiceParameters.NetworkServiceTransportFactory.class, injector.getInstance(MessagingTransportFactory.class)); injector.bindVolatileParameter( NetworkServiceParameters.NetworkServiceExceptionHandler.class, new ExceptionHandler()); final Injector injectorNs2 = injector.forkInjector(); injectorNs2.bindVolatileParameter( NetworkServiceParameters.NetworkServiceHandler.class, new MessageHandler<String>(name2, monitor, numMessages)); final NetworkService<String> ns2 = injectorNs2.getInstance(NetworkService.class); final Injector injectorNs1 = injector.forkInjector(); injectorNs1.bindVolatileParameter( NetworkServiceParameters.NetworkServiceHandler.class, new MessageHandler<String>(name1, null, 0)); final NetworkService<String> ns1 = injectorNs1.getInstance(NetworkService.class); ns2.registerId(factory.getNewInstance(name2)); final int port2 = ns2.getTransport().getListeningPort(); server.register(factory.getNewInstance(name2), new InetSocketAddress(localAddress, port2)); ns1.registerId(factory.getNewInstance(name1)); final int port1 = ns1.getTransport().getListeningPort(); server.register(factory.getNewInstance(name1), new InetSocketAddress(localAddress, port1)); final Identifier destId = factory.getNewInstance(name2); final String message = StringUtils.repeat('1', size); try (Connection<String> conn = ns1.newConnection(destId)) { conn.open(); for (int i = 0; i < numMessages; i++) { conn.write(message); } monitor.mwait(); } catch (final NetworkException e) { e.printStackTrace(); throw new RuntimeException(e); } } } catch (final Exception e) { e.printStackTrace(); throw new RuntimeException(e); } } }); } // start and time final long start = System.currentTimeMillis(); final Object ignore = new Object(); for (int i = 0; i < numThreads; i++) { barrier.add(ignore); } e.shutdown(); e.awaitTermination(100, TimeUnit.SECONDS); final long end = System.currentTimeMillis(); final double runtime = ((double) end - start) / 1000; LOG.log(Level.FINEST, "size: " + size + "; messages/s: " + totalNumMessages / runtime + " bandwidth(bytes/s): " + ((double) totalNumMessages * 2 * size) / runtime); // x2 for unicode chars } }
From source file:any.servable.LsServable.java
public void process(String cmd, String content, BlockingQueue<Message> outQueue) { logger.info("start LS SERVABLE"); String filename = content.substring(0, content.length()); File file = new File(filename); try {/*from ww w. j av a 2 s . c o m*/ File[] list = file.listFiles(new FilenameFilter() { @Override public boolean accept(File arg0, String name) { return !(name.startsWith(".") && !(name.endsWith("~"))); } }); StringBuilder sb = new StringBuilder(); // parent File fp = file.getParentFile(); logger.debug("do file: " + file + " - list: " + list + " fp:" + fp); if (fp != null) { TblUtil tbl = new TblUtil(); // DISPLAYTEXT tbl.set(TblUtil.DISPLAYTEXT, "[..] " + fp.getName()); // DETAILTEXT tbl.set(TblUtil.DETAILTEXT, "UP"); // // ICONRES // try { // JFileChooser ch = new JFileChooser(); // Icon icon = ch.getIcon(fp); // // BufferedImage offscreen = new BufferedImage( // icon.getIconHeight(), icon.getIconWidth(), // BufferedImage.TYPE_4BYTE_ABGR); // icon.paintIcon(null, offscreen.getGraphics(), 0, 0); // // ByteArrayOutputStream baos = new ByteArrayOutputStream(); // ImageIO.write(offscreen, "png", baos); // baos.flush(); // byte[] imageInByte = baos.toByteArray(); // baos.close(); // // String strrep = new String(imageInByte); // if (!resSet.containsKey(strrep)) { // // System.out.println(fp.getName() + ": " // + icon.toString() + " " + icon.hashCode()); // // outQueue.put(new Message("res", icon.toString(), // "image/png", "NO", imageInByte)); // resSet.put(strrep, icon.toString()); // tbl.set(TblUtil.ICONRES, icon.toString()); // } else { // tbl.set(TblUtil.ICONRES, resSet.get(strrep)); // } // } catch (Error e) { // // TODO // System.err.println("Exception due to icon caught"); // // e.printStackTrace(); // } // TABCMD String tcmd = ((fp.isDirectory() ? "ls://host" : "get://host") + fp.getAbsolutePath()); tbl.set(TblUtil.TABCMD, tcmd); // DETAILCMD tbl.set(TblUtil.DETAILCMD, "tmpl:" + tcmd); // DELETECMD sb.append(tbl.makeCell()); } logger.debug("do list: " + list); if (list != null) { for (File f : list) { logger.debug("do file: " + f); TblUtil tbl = new TblUtil(); // DISPLAYTEXT tbl.set(TblUtil.DISPLAYTEXT, f.getName()); // DETAILTEXT tbl.set(TblUtil.DETAILTEXT, (f.isDirectory() ? " -- " : humanReadableByteCount(f.length(), true)) + " - " + df.format(f.lastModified())); // // ICONRES // try { // JFileChooser ch = new JFileChooser(); // Icon icon = ch.getIcon(f); // // BufferedImage offscreen = new BufferedImage( // icon.getIconHeight(), icon.getIconWidth(), // BufferedImage.TYPE_4BYTE_ABGR); // icon.paintIcon(null, offscreen.getGraphics(), 0, 0); // // ByteArrayOutputStream baos = new ByteArrayOutputStream(); // ImageIO.write(offscreen, "png", baos); // baos.flush(); // byte[] imageInByte = baos.toByteArray(); // baos.close(); // // String strrep = new String(imageInByte); // if (!resSet.containsKey(strrep)) { // // System.out.println(f.getName() + ": " // + icon.toString() + " " + icon.hashCode()); // // outQueue.put(new Message("res", icon.toString(), // "image/png", "NO", imageInByte)); // resSet.put(strrep, icon.toString()); // tbl.set(TblUtil.ICONRES, icon.toString()); // } else { // tbl.set(TblUtil.ICONRES, resSet.get(strrep)); // } // // } catch (Error e) { // // TODO // System.err.println("Exception due to icon caught"); // // e.printStackTrace(); // } String fullpath = f.getAbsolutePath(); if (!fullpath.startsWith("/")) { fullpath = "/" + fullpath; } // TABCMD String tcmd = ((f.isDirectory() ? "ls://host" : "get://host") + fullpath); tbl.set(TblUtil.TABCMD, tcmd); // DETAILCMD tbl.set(TblUtil.DETAILCMD, "tmpl:" + tcmd); // DELETECMD sb.append(tbl.makeCell()); } outQueue.put( new Message("vset", file.getName(), TblUtil.TYPE, "YES", "1", sb.toString().getBytes())); } } catch (InterruptedException e) { e.printStackTrace(); final Writer result = new StringWriter(); final PrintWriter printWriter = new PrintWriter(result); e.printStackTrace(printWriter); outQueue.add(new Message("vset", "_error", "text/plain", "YES", result.toString().getBytes())); // } catch (IOException e) { // e.printStackTrace(); // final Writer result = new StringWriter(); // final PrintWriter printWriter = new PrintWriter(result); // e.printStackTrace(printWriter); // outQueue.add(new Message("vset", "_error", "text/plain", "YES", // result.toString().getBytes())); } catch (NullPointerException e) { e.printStackTrace(); final Writer result = new StringWriter(); final PrintWriter printWriter = new PrintWriter(result); e.printStackTrace(printWriter); outQueue.add(new Message("vset", "_error", "text/plain", "YES", result.toString().getBytes())); } logger.info("finished LS SERVABLE"); }