List of usage examples for java.util.concurrent ConcurrentLinkedQueue poll
public E poll()
From source file:com.ibm.crail.tools.CrailBenchmark.java
void readSequentialAsync(String filename, int size, int loop, int batch) throws Exception { System.out.println("readSequentialAsync, filename " + filename + ", size " + size + ", loop " + loop + ", batch " + batch); ConcurrentLinkedQueue<CrailBuffer> bufferQueue = new ConcurrentLinkedQueue<CrailBuffer>(); for (int i = 0; i < batch; i++) { CrailBuffer buf = null;/* ww w . j av a 2 s . c om*/ if (size == CrailConstants.BUFFER_SIZE) { buf = fs.allocateBuffer(); } else if (size < CrailConstants.BUFFER_SIZE) { CrailBuffer _buf = fs.allocateBuffer(); _buf.clear().limit(size); buf = _buf.slice(); } else { buf = OffHeapBuffer.wrap(ByteBuffer.allocateDirect(size)); } bufferQueue.add(buf); } //warmup warmUp(filename, warmup, bufferQueue); //benchmark System.out.println("starting benchmark..."); double sumbytes = 0; double ops = 0; fs.getStatistics().reset(); CrailFile file = fs.lookup(filename).get().asFile(); CrailInputStream directStream = file.getDirectInputStream(file.getCapacity()); HashMap<Integer, CrailBuffer> futureMap = new HashMap<Integer, CrailBuffer>(); LinkedBlockingQueue<Future<CrailResult>> futureQueue = new LinkedBlockingQueue<Future<CrailResult>>(); long start = System.currentTimeMillis(); for (int i = 0; i < batch - 1 && ops < loop; i++) { CrailBuffer buf = bufferQueue.poll(); buf.clear(); Future<CrailResult> future = directStream.read(buf); futureQueue.add(future); futureMap.put(future.hashCode(), buf); ops = ops + 1.0; } while (ops < loop) { CrailBuffer buf = bufferQueue.poll(); buf.clear(); Future<CrailResult> future = directStream.read(buf); futureQueue.add(future); futureMap.put(future.hashCode(), buf); future = futureQueue.poll(); CrailResult result = future.get(); buf = futureMap.get(future.hashCode()); bufferQueue.add(buf); sumbytes = sumbytes + result.getLen(); ops = ops + 1.0; } while (!futureQueue.isEmpty()) { Future<CrailResult> future = futureQueue.poll(); CrailResult result = future.get(); futureMap.get(future.hashCode()); sumbytes = sumbytes + result.getLen(); ops = ops + 1.0; } long end = System.currentTimeMillis(); double executionTime = ((double) (end - start)) / 1000.0; double throughput = 0.0; double latency = 0.0; double sumbits = sumbytes * 8.0; if (executionTime > 0) { throughput = sumbits / executionTime / 1000.0 / 1000.0; latency = 1000000.0 * executionTime / ops; } directStream.close(); System.out.println("execution time " + executionTime); System.out.println("ops " + ops); System.out.println("sumbytes " + sumbytes); System.out.println("throughput " + throughput); System.out.println("latency " + latency); fs.getStatistics().print("close"); }
From source file:me.schiz.jmeter.ring.udp.sampler.UDPRingSampler.java
@Override public SampleResult sample(Entry entry) { boolean idling = false; SampleResult newSampleResult = new SampleResult(); newSampleResult.setSampleLabel(getName()); ConcurrentLinkedQueue<SampleResult> queue = tlQueue.get(); if (queue == null) { queue = new ConcurrentLinkedQueue<SampleResult>(); tlQueue.set(queue);/*from ww w . j a v a 2s . c om*/ } Ring ring = UDPRingSourceElement.get(getSource()); Token t; int tid = -1; byte[] request_in_bytes = new byte[0]; ByteBuffer request = tlRequest.get(); if (request == null) { request = tlBuffer.get(); if (request == null) { request = ByteBuffer.allocateDirect(8 * 1024 * 1024); tlBuffer.set(request); } request.clear(); if (isHex()) { try { request_in_bytes = Hex.decodeHex(getRequest().toCharArray()); } catch (DecoderException e) { log.error("can't decode request", e); idling = true; } } else { request_in_bytes = getRequest().getBytes(); } request.put(request_in_bytes); } if (!idling) { try { request.flip(); while (tid == -1) { tid = ring.acquire(); } t = ring.get(tid); t.lock.lock(); if (isHex()) t.ishex = true; newSampleResult.sampleStart(); try { //t.socketChannel.write(request); t.sampleResult = newSampleResult; t.queue = queue; ring.write(tid, request); request.clear(); newSampleResult.setSuccessful(true); } catch (IOException e) { newSampleResult.setSuccessful(false); ring.reset(tid); log.warn("IOException", e); } finally { t.lock.unlock(); } } catch (Exception e) { log.error("Exception", e); newSampleResult.setSuccessful(false); newSampleResult.setResponseCode(e.getClass().getName()); while (!queue.offer(newSampleResult)) { } if (tid != -1) ring.reset(tid); } finally { newSampleResult.setRequestHeaders(getRequest()); } } SampleResult sampleResult = queue.poll(); return sampleResult; }
From source file:com.ibm.crail.tools.CrailBenchmark.java
void writeAsync(String filename, int size, int loop, int batch, int storageClass, int locationClass) throws Exception { System.out.println("writeAsync, filename " + filename + ", size " + size + ", loop " + loop + ", batch " + batch + ", storageClass " + storageClass + ", locationClass " + locationClass); ConcurrentLinkedQueue<CrailBuffer> bufferQueue = new ConcurrentLinkedQueue<CrailBuffer>(); for (int i = 0; i < batch; i++) { CrailBuffer buf = null;// www . ja v a 2 s .co m if (size == CrailConstants.BUFFER_SIZE) { buf = fs.allocateBuffer(); } else if (size < CrailConstants.BUFFER_SIZE) { CrailBuffer _buf = fs.allocateBuffer(); _buf.clear().limit(size); buf = _buf.slice(); } else { buf = OffHeapBuffer.wrap(ByteBuffer.allocateDirect(size)); } bufferQueue.add(buf); } //warmup warmUp(filename, warmup, bufferQueue); //benchmark System.out.println("starting benchmark..."); LinkedBlockingQueue<Future<CrailResult>> futureQueue = new LinkedBlockingQueue<Future<CrailResult>>(); HashMap<Integer, CrailBuffer> futureMap = new HashMap<Integer, CrailBuffer>(); fs.getStatistics().reset(); long _loop = (long) loop; long _bufsize = (long) CrailConstants.BUFFER_SIZE; long _capacity = _loop * _bufsize; double sumbytes = 0; double ops = 0; CrailFile file = fs.create(filename, CrailNodeType.DATAFILE, CrailStorageClass.get(storageClass), CrailLocationClass.get(locationClass)).get().asFile(); CrailOutputStream directStream = file.getDirectOutputStream(_capacity); long start = System.currentTimeMillis(); for (int i = 0; i < batch - 1 && ops < loop; i++) { CrailBuffer buf = bufferQueue.poll(); buf.clear(); Future<CrailResult> future = directStream.write(buf); futureQueue.add(future); futureMap.put(future.hashCode(), buf); ops = ops + 1.0; } while (ops < loop) { CrailBuffer buf = bufferQueue.poll(); buf.clear(); Future<CrailResult> future = directStream.write(buf); futureQueue.add(future); futureMap.put(future.hashCode(), buf); future = futureQueue.poll(); future.get(); buf = futureMap.get(future.hashCode()); bufferQueue.add(buf); sumbytes = sumbytes + buf.capacity(); ops = ops + 1.0; } while (!futureQueue.isEmpty()) { Future<CrailResult> future = futureQueue.poll(); future.get(); CrailBuffer buf = futureMap.get(future.hashCode()); sumbytes = sumbytes + buf.capacity(); ops = ops + 1.0; } long end = System.currentTimeMillis(); double executionTime = ((double) (end - start)) / 1000.0; double throughput = 0.0; double latency = 0.0; double sumbits = sumbytes * 8.0; if (executionTime > 0) { throughput = sumbits / executionTime / 1000.0 / 1000.0; latency = 1000000.0 * executionTime / ops; } directStream.close(); System.out.println("execution time " + executionTime); System.out.println("ops " + ops); System.out.println("sumbytes " + sumbytes); System.out.println("throughput " + throughput); System.out.println("latency " + latency); fs.getStatistics().print("close"); }
From source file:org.wso2.developerstudio.eclipse.greg.manager.remote.views.RegistryBrowserView.java
private String searchRegistryNodeForResource(RegistryNode node, String caption) throws InvalidRegistryURLException, UnknownRegistryException { ConcurrentLinkedQueue<RegistryResourceNode> queue = new ConcurrentLinkedQueue<RegistryResourceNode>(); queue.addAll(node.getRegistryContainer().getRegistryContent()); while (queue.peek() != null) { RegistryResourceNode registryResourceNode = queue.poll(); if (caption.equalsIgnoreCase(registryResourceNode.getCaption())) { return registryResourceNode.getRegistryResourcePath(); } else {//from w ww .j ava2 s . c om queue.addAll(registryResourceNode.getResourceNodeList()); } } // for (RegistryResourceNode registryResourceNode : queue) { // if(caption.equalsIgnoreCase(registryResourceNode.getCaption())){ // return registryResourceNode.getRegistryResourcePath(); // }else{ // queue.addAll(registryResourceNode.getResourceNodeList()); // } // } return null; }
From source file:org.wso2.developerstudio.eclipse.greg.manager.remote.views.RegistryBrowserView.java
private RegistryResourceNode searchRegistryNodeForResourceNode(RegistryNode node, String caption) throws InvalidRegistryURLException, UnknownRegistryException { ConcurrentLinkedQueue<RegistryResourceNode> queue = new ConcurrentLinkedQueue<RegistryResourceNode>(); queue.addAll(node.getRegistryContainer().getRegistryContent()); while (queue.peek() != null) { RegistryResourceNode registryResourceNode = queue.poll(); if (caption.equalsIgnoreCase(registryResourceNode.getCaption())) { return registryResourceNode; } else {/* w ww. j a v a 2s .co m*/ queue.addAll(registryResourceNode.getResourceNodeList()); } } // for (RegistryResourceNode registryResourceNode : queue) { // if(caption.equalsIgnoreCase(registryResourceNode.getCaption())){ // return registryResourceNode.getRegistryResourcePath(); // }else{ // queue.addAll(registryResourceNode.getResourceNodeList()); // } // } return null; }
From source file:dendroscope.autumn.hybridnumber.ComputeHybridNumber.java
/** * recursively compute the hybrid number * * @param root1//from www . ja v a2 s . c om * @param root2 * @param isReduced @return hybrid number * @param retry * @param topLevel * @param scoreAbove * @param additionalAbove */ private int computeHybridNumberRec(final Root root1, final Root root2, boolean isReduced, Integer previousHybrid, BitSet retry, final boolean topLevel, final int scoreAbove, final ValuesList additionalAbove) throws IOException, CanceledException { if (System.currentTimeMillis() > nextTime) { synchronized (progressListener) { nextTime += waitTime; waitTime *= 1.5; progressListener.incrementProgress(); } } else progressListener.checkForCancel(); // System.err.println("computeHybridNumberRec: tree1=" + Basic.toString(root1.getTaxa()) + " tree2=" + Basic.toString(root2.getTaxa())); // root1.reorderSubTree(); // root2.reorderSubTree(); if (checking) { root1.checkTree(); root2.checkTree(); } BitSet taxa = root1.getTaxa(); String key = root1.toStringTreeSparse() + root2.toStringTreeSparse(); // System.err.println("Key: "+key); Integer value; synchronized (lookupTable) { value = (Integer) lookupTable.get(key); if (value != null) return value; } if (!root2.getTaxa().equals(taxa)) throw new RuntimeException("Unequal taxon sets: X=" + Basic.toString(root1.getTaxa()) + " vs " + Basic.toString(root2.getTaxa())); if (!isReduced) { switch (SubtreeReduction.apply(root1, root2, null)) { case ISOMORPHIC: synchronized (lookupTable) { lookupTable.put(key, 0); } if (topLevel) { bestScore.lowerTo(0); progressListener.setSubtask("Best score: " + bestScore); } return 0; // two trees are isomorphic, no hybrid node needed case REDUCED: // a reduction was performed, cannot maintain lexicographical ordering in removal loop below previousHybrid = null; break; case IRREDUCIBLE: break; } Single<Integer> placeHolderTaxa = new Single<Integer>(); final Pair<Root, Root> clusterTrees = ClusterReduction.apply(root1, root2, placeHolderTaxa); final boolean retryTop = false && (previousHybrid != null && placeHolderTaxa.get() < previousHybrid); // if the taxa involved in the cluster reduction come before the previously removed hybrid, do full retry // retryTop doesn't work final BitSet fRetry = retry; if (clusterTrees != null) // will perform cluster-reduction { final Value score1 = new Value(0); final Value score2 = new Value(1); // because the cluster could not be reduced using an subtree reduction, can assume that we will need one reticulation for this final boolean verbose = ProgramProperties.get("verbose-HL-parallel", false); if (verbose) System.err.println("Starting parallel loop"); final CountDownLatch countDownLatch = new CountDownLatch(2); final Integer fPrevious = previousHybrid; // setup task: final Task task1 = new Task(); // first of two cluster-reduction tasks task1.setRunnable(new Runnable() { public void run() { try { if (verbose) { System.err.println("Launching thread on cluster-reduction"); System.err .println("Active threads " + scheduledThreadPoolExecutor.getActiveCount()); } final ValuesList additionalAbove1 = additionalAbove.copyWithAdditionalElement(score2); if (scoreAbove + additionalAbove1.sum() < bestScore.get()) { int h = computeHybridNumberRec(root1, root2, false, fPrevious, fRetry, false, scoreAbove, additionalAbove1); score1.set(h); } else { score1.set(LARGE); } additionalAbove1.clear(); } catch (Exception ex) { while (countDownLatch.getCount() > 0) countDownLatch.countDown(); } countDownLatch.countDown(); } }); final Task task2 = new Task(); // second of two cluster-reduction tasks task2.setRunnable(new Runnable() { public void run() { try { if (verbose) { System.err.println("Launching thread on cluster-reduction"); System.err .println("Active threads " + scheduledThreadPoolExecutor.getActiveCount()); } final ValuesList additionalAbove2 = additionalAbove.copyWithAdditionalElement(score1); if (scoreAbove + additionalAbove2.sum() < bestScore.get()) { int h = computeHybridNumberRec(clusterTrees.getFirst(), clusterTrees.getSecond(), true, fPrevious, fRetry, false, scoreAbove, additionalAbove2); score2.set(h); } else { score2.set(LARGE); } additionalAbove2.clear(); } catch (Exception ex) { while (countDownLatch.getCount() > 0) countDownLatch.countDown(); } countDownLatch.countDown(); } }); // start a task in this thread scheduledThreadPoolExecutor.execute(task1); task2.run(); task1.run(); // try to run task1 in current thread if it hasn't yet started execution. If the task is already running or has completed, will simply return try { if (verbose) System.err.println("waiting..."); // wait until all tasks have completed countDownLatch.await(); if (verbose) System.err.println("done"); } catch (InterruptedException e) { Basic.caught(e); } clusterTrees.getFirst().deleteSubTree(); clusterTrees.getSecond().deleteSubTree(); int total = scoreAbove + additionalAbove.sum() + score1.get() + score2.get(); if (topLevel && (total < bestScore.get())) // score above will be zero, but put this here anyway to avoid confusion { bestScore.lowerTo(total); progressListener.setSubtask("Current best score: " + bestScore); } synchronized (lookupTable) { Integer old = (Integer) lookupTable.get(key); if (old == null || total < old) lookupTable.put(key, total); } return score1.get() + score2.get(); } } List<Root> leaves1 = root1.getAllLeaves(); if (leaves1.size() <= 2) // try 2 rather than one... { return 0; } final boolean verbose = ProgramProperties.get("verbose-HL-parallel", false); if (verbose) System.err.println("Starting parallel loop"); final CountDownLatch countDownLatch = new CountDownLatch(leaves1.size()); final Value bestSubH = new Value(LARGE); // schedule all tasks to be performed final ConcurrentLinkedQueue<Task> queue = new ConcurrentLinkedQueue<Task>(); for (Node leaf2remove : leaves1) { final BitSet taxa2remove = ((Root) leaf2remove).getTaxa(); if (previousHybrid == null || previousHybrid < taxa2remove.nextSetBit(0)) { if (scoreAbove + additionalAbove.sum() + 1 >= bestScore.get()) return LARGE; // other thread has found a better result, abort // setup task: final Task task = new Task(); task.setRunnable(new Runnable() { public void run() { try { if (verbose) { System.err.println("Launching thread on " + Basic.toString(taxa2remove)); System.err .println("Active threads " + scheduledThreadPoolExecutor.getActiveCount()); } queue.remove(task); if (scoreAbove + additionalAbove.sum() + 1 < bestScore.get()) { Root tree1X = CopyWithTaxaRemoved.apply(root1, taxa2remove); Root tree2X = CopyWithTaxaRemoved.apply(root2, taxa2remove); Refine.apply(tree1X, tree2X); int scoreBelow = computeHybridNumberRec(tree1X, tree2X, false, taxa2remove.nextSetBit(0), null, false, scoreAbove + 1, additionalAbove) + 1; if (topLevel && scoreBelow < bestScore.get()) { bestScore.lowerTo(scoreBelow); progressListener.setSubtask("Current best score: " + bestScore); } synchronized (bestSubH) { if (scoreBelow < bestSubH.get()) bestSubH.set(scoreBelow); } tree1X.deleteSubTree(); tree2X.deleteSubTree(); } } catch (Exception ex) { while (countDownLatch.getCount() > 0) countDownLatch.countDown(); } countDownLatch.countDown(); } }); queue.add(task); } else // no task for this item, count down { countDownLatch.countDown(); progressListener.checkForCancel(); } } // grab one task for the current thread: Task taskForCurrentThread = queue.size() > 0 ? queue.poll() : null; // launch all others in the executor for (Task task : queue) scheduledThreadPoolExecutor.execute(task); // start a task in this thread if (taskForCurrentThread != null) taskForCurrentThread.run(); // try to run other tasks from the queue. Note that any task that is already running will return immediately while (queue.size() > 0) { Task task = queue.poll(); if (task != null) task.run(); } try { if (verbose) System.err.println("waiting..."); // wait until all tasks have completed countDownLatch.await(); if (verbose) System.err.println("done"); } catch (InterruptedException e) { Basic.caught(e); return LARGE; } // return the best value synchronized (lookupTable) { Integer old = (Integer) lookupTable.get(key); if (old == null || old > bestSubH.get()) lookupTable.put(key, bestSubH.get()); } return bestSubH.get(); }
From source file:com.datatorrent.stram.StreamingContainerManager.java
/** * process the heartbeat from each container. * called by the RPC thread for each container. (i.e. called by multiple threads) * * @param heartbeat//from w w w .ja v a 2s . co m * @return heartbeat response */ @SuppressWarnings("StatementWithEmptyBody") public ContainerHeartbeatResponse processHeartbeat(ContainerHeartbeat heartbeat) { long currentTimeMillis = clock.getTime(); final StreamingContainerAgent sca = this.containers.get(heartbeat.getContainerId()); if (sca == null || sca.container.getState() == PTContainer.State.KILLED) { // could be orphaned container that was replaced and needs to terminate LOG.error("Unknown container {}", heartbeat.getContainerId()); ContainerHeartbeatResponse response = new ContainerHeartbeatResponse(); response.shutdown = true; return response; } //LOG.debug("{} {} {}", new Object[]{sca.container.containerId, sca.container.bufferServerAddress, sca.container.getState()}); if (sca.container.getState() == PTContainer.State.ALLOCATED) { // capture dynamically assigned address from container if (sca.container.bufferServerAddress == null && heartbeat.bufferServerHost != null) { sca.container.bufferServerAddress = InetSocketAddress.createUnresolved(heartbeat.bufferServerHost, heartbeat.bufferServerPort); LOG.info("Container {} buffer server: {}", sca.container.getExternalId(), sca.container.bufferServerAddress); } final long containerStartTime = System.currentTimeMillis(); sca.container.setState(PTContainer.State.ACTIVE); sca.container.setStartedTime(containerStartTime); sca.container.setFinishedTime(-1); sca.jvmName = heartbeat.jvmName; poolExecutor.submit(new Runnable() { @Override public void run() { try { containerFile.append(sca.getContainerInfo()); } catch (IOException ex) { LOG.warn("Cannot write to container file"); } for (PTOperator ptOp : sca.container.getOperators()) { try { JSONObject operatorInfo = new JSONObject(); operatorInfo.put("name", ptOp.getName()); operatorInfo.put("id", ptOp.getId()); operatorInfo.put("container", sca.container.getExternalId()); operatorInfo.put("startTime", containerStartTime); operatorFile.append(operatorInfo); } catch (IOException | JSONException ex) { LOG.warn("Cannot write to operator file: ", ex); } } } }); } if (heartbeat.restartRequested) { LOG.error("Container {} restart request", sca.container.getExternalId()); containerStopRequests.put(sca.container.getExternalId(), sca.container.getExternalId()); } sca.memoryMBFree = heartbeat.memoryMBFree; sca.gcCollectionCount = heartbeat.gcCollectionCount; sca.gcCollectionTime = heartbeat.gcCollectionTime; sca.undeployOpers.clear(); sca.deployOpers.clear(); if (!this.deployChangeInProgress.get()) { sca.deployCnt = this.deployChangeCnt; } Set<Integer> reportedOperators = Sets.newHashSetWithExpectedSize(sca.container.getOperators().size()); for (OperatorHeartbeat shb : heartbeat.getContainerStats().operators) { long maxEndWindowTimestamp = 0; reportedOperators.add(shb.nodeId); PTOperator oper = this.plan.getAllOperators().get(shb.getNodeId()); if (oper == null) { LOG.info("Heartbeat for unknown operator {} (container {})", shb.getNodeId(), heartbeat.getContainerId()); sca.undeployOpers.add(shb.nodeId); continue; } if (shb.requestResponse != null) { for (StatsListener.OperatorResponse obj : shb.requestResponse) { if (obj instanceof OperatorResponse) { // This is to identify platform requests commandResponse.put((Long) obj.getResponseId(), obj.getResponse()); LOG.debug(" Got back the response {} for the request {}", obj, obj.getResponseId()); } else { // This is to identify user requests oper.stats.responses.add(obj); } } } //LOG.debug("heartbeat {} {}/{} {}", oper, oper.getState(), shb.getState(), oper.getContainer().getExternalId()); if (!(oper.getState() == PTOperator.State.ACTIVE && shb.getState() == OperatorHeartbeat.DeployState.ACTIVE)) { // deploy state may require synchronization processOperatorDeployStatus(oper, shb, sca); } oper.stats.lastHeartbeat = shb; List<ContainerStats.OperatorStats> statsList = shb.getOperatorStatsContainer(); if (!statsList.isEmpty()) { long tuplesProcessed = 0; long tuplesEmitted = 0; long totalCpuTimeUsed = 0; int statCount = 0; long maxDequeueTimestamp = -1; oper.stats.recordingId = null; final OperatorStatus status = oper.stats; status.statsRevs.checkout(); for (Map.Entry<String, PortStatus> entry : status.inputPortStatusList.entrySet()) { entry.getValue().recordingId = null; } for (Map.Entry<String, PortStatus> entry : status.outputPortStatusList.entrySet()) { entry.getValue().recordingId = null; } for (ContainerStats.OperatorStats stats : statsList) { if (stats == null) { LOG.warn("Operator {} statistics list contains null element", shb.getNodeId()); continue; } /* report checkpoint-ed WindowId status of the operator */ if (stats.checkpoint instanceof Checkpoint) { if (oper.getRecentCheckpoint() == null || oper.getRecentCheckpoint().windowId < stats.checkpoint.getWindowId()) { addCheckpoint(oper, (Checkpoint) stats.checkpoint); if (stats.checkpointStats != null) { status.checkpointStats = stats.checkpointStats; status.checkpointTimeMA.add(stats.checkpointStats.checkpointTime); } oper.failureCount = 0; } } oper.stats.recordingId = stats.recordingId; /* report all the other stuff */ // calculate the stats related to end window EndWindowStats endWindowStats = new EndWindowStats(); // end window stats for a particular window id for a particular node Collection<ContainerStats.OperatorStats.PortStats> ports = stats.inputPorts; if (ports != null) { Set<String> currentInputPortSet = Sets.newHashSetWithExpectedSize(ports.size()); for (ContainerStats.OperatorStats.PortStats s : ports) { currentInputPortSet.add(s.id); PortStatus ps = status.inputPortStatusList.get(s.id); if (ps == null) { ps = status.new PortStatus(); ps.portName = s.id; status.inputPortStatusList.put(s.id, ps); } ps.totalTuples += s.tupleCount; ps.recordingId = s.recordingId; tuplesProcessed += s.tupleCount; endWindowStats.dequeueTimestamps.put(s.id, s.endWindowTimestamp); Pair<Integer, String> operatorPortName = new Pair<>(oper.getId(), s.id); Long lastEndWindowTimestamp = operatorPortLastEndWindowTimestamps.get(operatorPortName); if (lastEndWindowTimestamp == null) { lastEndWindowTimestamp = lastStatsTimestamp; } long portElapsedMillis = Math.max(s.endWindowTimestamp - lastEndWindowTimestamp, 0); //LOG.debug("=== PROCESSED TUPLE COUNT for {}: {}, {}, {}, {}", operatorPortName, s.tupleCount, portElapsedMillis, operatorPortLastEndWindowTimestamps.get(operatorPortName), lastStatsTimestamp); ps.tuplesPMSMA.add(s.tupleCount, portElapsedMillis); ps.bufferServerBytesPMSMA.add(s.bufferServerBytes, portElapsedMillis); ps.queueSizeMA.add(s.queueSize); operatorPortLastEndWindowTimestamps.put(operatorPortName, s.endWindowTimestamp); if (maxEndWindowTimestamp < s.endWindowTimestamp) { maxEndWindowTimestamp = s.endWindowTimestamp; } if (s.endWindowTimestamp > maxDequeueTimestamp) { maxDequeueTimestamp = s.endWindowTimestamp; } } // need to remove dead ports, for unifiers Iterator<Map.Entry<String, PortStatus>> it = status.inputPortStatusList.entrySet() .iterator(); while (it.hasNext()) { Map.Entry<String, PortStatus> entry = it.next(); if (!currentInputPortSet.contains(entry.getKey())) { it.remove(); } } } ports = stats.outputPorts; if (ports != null) { Set<String> currentOutputPortSet = Sets.newHashSetWithExpectedSize(ports.size()); for (ContainerStats.OperatorStats.PortStats s : ports) { currentOutputPortSet.add(s.id); PortStatus ps = status.outputPortStatusList.get(s.id); if (ps == null) { ps = status.new PortStatus(); ps.portName = s.id; status.outputPortStatusList.put(s.id, ps); } ps.totalTuples += s.tupleCount; ps.recordingId = s.recordingId; tuplesEmitted += s.tupleCount; Pair<Integer, String> operatorPortName = new Pair<>(oper.getId(), s.id); Long lastEndWindowTimestamp = operatorPortLastEndWindowTimestamps.get(operatorPortName); if (lastEndWindowTimestamp == null) { lastEndWindowTimestamp = lastStatsTimestamp; } long portElapsedMillis = Math.max(s.endWindowTimestamp - lastEndWindowTimestamp, 0); //LOG.debug("=== EMITTED TUPLE COUNT for {}: {}, {}, {}, {}", operatorPortName, s.tupleCount, portElapsedMillis, operatorPortLastEndWindowTimestamps.get(operatorPortName), lastStatsTimestamp); ps.tuplesPMSMA.add(s.tupleCount, portElapsedMillis); ps.bufferServerBytesPMSMA.add(s.bufferServerBytes, portElapsedMillis); operatorPortLastEndWindowTimestamps.put(operatorPortName, s.endWindowTimestamp); if (maxEndWindowTimestamp < s.endWindowTimestamp) { maxEndWindowTimestamp = s.endWindowTimestamp; } } if (ports.size() > 0) { endWindowStats.emitTimestamp = ports.iterator().next().endWindowTimestamp; } // need to remove dead ports, for unifiers Iterator<Map.Entry<String, PortStatus>> it = status.outputPortStatusList.entrySet() .iterator(); while (it.hasNext()) { Map.Entry<String, PortStatus> entry = it.next(); if (!currentOutputPortSet.contains(entry.getKey())) { it.remove(); } } } // for output operator, just take the maximum dequeue time for emit timestamp. // (we don't know the latency for output operators because they don't emit tuples) if (endWindowStats.emitTimestamp < 0) { endWindowStats.emitTimestamp = maxDequeueTimestamp; } if (status.currentWindowId.get() != stats.windowId) { status.lastWindowIdChangeTms = currentTimeMillis; status.currentWindowId.set(stats.windowId); } totalCpuTimeUsed += stats.cpuTimeUsed; statCount++; if (oper.getOperatorMeta().getValue(OperatorContext.COUNTERS_AGGREGATOR) != null) { endWindowStats.counters = stats.counters; } if (oper.getOperatorMeta().getMetricAggregatorMeta() != null && oper.getOperatorMeta().getMetricAggregatorMeta().getAggregator() != null) { endWindowStats.metrics = stats.metrics; } if (stats.windowId > currentEndWindowStatsWindowId) { Map<Integer, EndWindowStats> endWindowStatsMap = endWindowStatsOperatorMap .get(stats.windowId); if (endWindowStatsMap == null) { endWindowStatsMap = new ConcurrentSkipListMap<Integer, EndWindowStats>(); Map<Integer, EndWindowStats> endWindowStatsMapPrevious = endWindowStatsOperatorMap .putIfAbsent(stats.windowId, endWindowStatsMap); if (endWindowStatsMapPrevious != null) { endWindowStatsMap = endWindowStatsMapPrevious; } } endWindowStatsMap.put(shb.getNodeId(), endWindowStats); if (!oper.getInputs().isEmpty()) { long latency = Long.MAX_VALUE; long adjustedEndWindowEmitTimestamp = endWindowStats.emitTimestamp; MovingAverageLong rpcLatency = rpcLatencies.get(oper.getContainer().getExternalId()); if (rpcLatency != null) { adjustedEndWindowEmitTimestamp += rpcLatency.getAvg(); } PTOperator slowestUpstream = null; for (PTInput input : oper.getInputs()) { PTOperator upstreamOp = input.source.source; if (upstreamOp.getOperatorMeta().getOperator() instanceof Operator.DelayOperator) { continue; } EndWindowStats ews = endWindowStatsMap.get(upstreamOp.getId()); long portLatency; if (ews == null) { // This is when the operator is likely to be behind too many windows. We need to give an estimate for // latency at this point, by looking at the number of windows behind int widthMillis = plan.getLogicalPlan() .getValue(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS); portLatency = (upstreamOp.stats.currentWindowId.get() - oper.stats.currentWindowId.get()) * widthMillis; } else { MovingAverageLong upstreamRPCLatency = rpcLatencies .get(upstreamOp.getContainer().getExternalId()); portLatency = adjustedEndWindowEmitTimestamp - ews.emitTimestamp; if (upstreamRPCLatency != null) { portLatency -= upstreamRPCLatency.getAvg(); } } if (portLatency < 0) { portLatency = 0; } if (latency > portLatency) { latency = portLatency; slowestUpstream = upstreamOp; } } status.latencyMA.add(latency); slowestUpstreamOp.put(oper, slowestUpstream); } Set<Integer> allCurrentOperators = plan.getAllOperators().keySet(); int numOperators = plan.getAllOperators().size(); if (allCurrentOperators.containsAll(endWindowStatsMap.keySet()) && endWindowStatsMap.size() == numOperators) { completeEndWindowStatsWindowId = stats.windowId; } } } status.totalTuplesProcessed.add(tuplesProcessed); status.totalTuplesEmitted.add(tuplesEmitted); OperatorMeta logicalOperator = oper.getOperatorMeta(); LogicalOperatorStatus logicalStatus = logicalOperator.getStatus(); if (!oper.isUnifier()) { logicalStatus.totalTuplesProcessed += tuplesProcessed; logicalStatus.totalTuplesEmitted += tuplesEmitted; } long lastMaxEndWindowTimestamp = operatorLastEndWindowTimestamps.containsKey(oper.getId()) ? operatorLastEndWindowTimestamps.get(oper.getId()) : lastStatsTimestamp; if (maxEndWindowTimestamp >= lastMaxEndWindowTimestamp) { double tuplesProcessedPMSMA = 0.0; double tuplesEmittedPMSMA = 0.0; if (statCount != 0) { //LOG.debug("CPU for {}: {} / {} - {}", oper.getId(), totalCpuTimeUsed, maxEndWindowTimestamp, lastMaxEndWindowTimestamp); status.cpuNanosPMSMA.add(totalCpuTimeUsed, maxEndWindowTimestamp - lastMaxEndWindowTimestamp); } for (PortStatus ps : status.inputPortStatusList.values()) { tuplesProcessedPMSMA += ps.tuplesPMSMA.getAvg(); } for (PortStatus ps : status.outputPortStatusList.values()) { tuplesEmittedPMSMA += ps.tuplesPMSMA.getAvg(); } status.tuplesProcessedPSMA.set(Math.round(tuplesProcessedPMSMA * 1000)); status.tuplesEmittedPSMA.set(Math.round(tuplesEmittedPMSMA * 1000)); } else { //LOG.warn("This timestamp for {} is lower than the previous!! {} < {}", oper.getId(), maxEndWindowTimestamp, lastMaxEndWindowTimestamp); } operatorLastEndWindowTimestamps.put(oper.getId(), maxEndWindowTimestamp); status.listenerStats.add(statsList); this.reportStats.put(oper, oper); status.statsRevs.commit(); } if (lastStatsTimestamp < maxEndWindowTimestamp) { lastStatsTimestamp = maxEndWindowTimestamp; } } sca.lastHeartbeatMillis = currentTimeMillis; for (PTOperator oper : sca.container.getOperators()) { if (!reportedOperators.contains(oper.getId())) { processOperatorDeployStatus(oper, null, sca); } } ContainerHeartbeatResponse rsp = getHeartbeatResponse(sca); if (heartbeat.getContainerStats().operators.isEmpty() && isApplicationIdle()) { LOG.info("requesting idle shutdown for container {}", heartbeat.getContainerId()); rsp.shutdown = true; } else { if (sca.shutdownRequested) { LOG.info("requesting shutdown for container {}", heartbeat.getContainerId()); rsp.shutdown = true; } } List<StramToNodeRequest> requests = rsp.nodeRequests != null ? rsp.nodeRequests : new ArrayList<StramToNodeRequest>(); ConcurrentLinkedQueue<StramToNodeRequest> operatorRequests = sca.getOperatorRequests(); while (true) { StramToNodeRequest r = operatorRequests.poll(); if (r == null) { break; } requests.add(r); } rsp.nodeRequests = requests; rsp.committedWindowId = committedWindowId; return rsp; }