Example usage for java.util.concurrent ConcurrentLinkedQueue poll

List of usage examples for java.util.concurrent ConcurrentLinkedQueue poll

Introduction

In this page you can find the example usage for java.util.concurrent ConcurrentLinkedQueue poll.

Prototype

public E poll() 

Source Link

Usage

From source file:com.ibm.crail.tools.CrailBenchmark.java

void readSequentialAsync(String filename, int size, int loop, int batch) throws Exception {
    System.out.println("readSequentialAsync, filename " + filename + ", size " + size + ", loop " + loop
            + ", batch " + batch);

    ConcurrentLinkedQueue<CrailBuffer> bufferQueue = new ConcurrentLinkedQueue<CrailBuffer>();
    for (int i = 0; i < batch; i++) {
        CrailBuffer buf = null;/* ww  w .  j  av  a 2  s  .  c om*/
        if (size == CrailConstants.BUFFER_SIZE) {
            buf = fs.allocateBuffer();
        } else if (size < CrailConstants.BUFFER_SIZE) {
            CrailBuffer _buf = fs.allocateBuffer();
            _buf.clear().limit(size);
            buf = _buf.slice();
        } else {
            buf = OffHeapBuffer.wrap(ByteBuffer.allocateDirect(size));
        }
        bufferQueue.add(buf);
    }

    //warmup
    warmUp(filename, warmup, bufferQueue);

    //benchmark
    System.out.println("starting benchmark...");
    double sumbytes = 0;
    double ops = 0;
    fs.getStatistics().reset();
    CrailFile file = fs.lookup(filename).get().asFile();
    CrailInputStream directStream = file.getDirectInputStream(file.getCapacity());
    HashMap<Integer, CrailBuffer> futureMap = new HashMap<Integer, CrailBuffer>();
    LinkedBlockingQueue<Future<CrailResult>> futureQueue = new LinkedBlockingQueue<Future<CrailResult>>();
    long start = System.currentTimeMillis();
    for (int i = 0; i < batch - 1 && ops < loop; i++) {
        CrailBuffer buf = bufferQueue.poll();
        buf.clear();
        Future<CrailResult> future = directStream.read(buf);
        futureQueue.add(future);
        futureMap.put(future.hashCode(), buf);
        ops = ops + 1.0;
    }
    while (ops < loop) {
        CrailBuffer buf = bufferQueue.poll();
        buf.clear();
        Future<CrailResult> future = directStream.read(buf);
        futureQueue.add(future);
        futureMap.put(future.hashCode(), buf);

        future = futureQueue.poll();
        CrailResult result = future.get();
        buf = futureMap.get(future.hashCode());
        bufferQueue.add(buf);

        sumbytes = sumbytes + result.getLen();
        ops = ops + 1.0;
    }
    while (!futureQueue.isEmpty()) {
        Future<CrailResult> future = futureQueue.poll();
        CrailResult result = future.get();
        futureMap.get(future.hashCode());
        sumbytes = sumbytes + result.getLen();
        ops = ops + 1.0;
    }
    long end = System.currentTimeMillis();
    double executionTime = ((double) (end - start)) / 1000.0;
    double throughput = 0.0;
    double latency = 0.0;
    double sumbits = sumbytes * 8.0;
    if (executionTime > 0) {
        throughput = sumbits / executionTime / 1000.0 / 1000.0;
        latency = 1000000.0 * executionTime / ops;
    }
    directStream.close();

    System.out.println("execution time " + executionTime);
    System.out.println("ops " + ops);
    System.out.println("sumbytes " + sumbytes);
    System.out.println("throughput " + throughput);
    System.out.println("latency " + latency);

    fs.getStatistics().print("close");
}

From source file:me.schiz.jmeter.ring.udp.sampler.UDPRingSampler.java

@Override
public SampleResult sample(Entry entry) {
    boolean idling = false;
    SampleResult newSampleResult = new SampleResult();
    newSampleResult.setSampleLabel(getName());

    ConcurrentLinkedQueue<SampleResult> queue = tlQueue.get();
    if (queue == null) {
        queue = new ConcurrentLinkedQueue<SampleResult>();
        tlQueue.set(queue);/*from ww  w . j  a v  a  2s  .  c  om*/
    }

    Ring ring = UDPRingSourceElement.get(getSource());
    Token t;
    int tid = -1;
    byte[] request_in_bytes = new byte[0];

    ByteBuffer request = tlRequest.get();
    if (request == null) {
        request = tlBuffer.get();
        if (request == null) {
            request = ByteBuffer.allocateDirect(8 * 1024 * 1024);
            tlBuffer.set(request);
        }
        request.clear();

        if (isHex()) {
            try {
                request_in_bytes = Hex.decodeHex(getRequest().toCharArray());
            } catch (DecoderException e) {
                log.error("can't decode request", e);
                idling = true;
            }
        } else {
            request_in_bytes = getRequest().getBytes();
        }
        request.put(request_in_bytes);
    }
    if (!idling) {
        try {
            request.flip();
            while (tid == -1) {
                tid = ring.acquire();
            }
            t = ring.get(tid);
            t.lock.lock();
            if (isHex())
                t.ishex = true;
            newSampleResult.sampleStart();
            try {
                //t.socketChannel.write(request);
                t.sampleResult = newSampleResult;
                t.queue = queue;
                ring.write(tid, request);
                request.clear();
                newSampleResult.setSuccessful(true);
            } catch (IOException e) {
                newSampleResult.setSuccessful(false);
                ring.reset(tid);
                log.warn("IOException", e);
            } finally {
                t.lock.unlock();
            }

        } catch (Exception e) {
            log.error("Exception", e);
            newSampleResult.setSuccessful(false);
            newSampleResult.setResponseCode(e.getClass().getName());
            while (!queue.offer(newSampleResult)) {
            }
            if (tid != -1)
                ring.reset(tid);
        } finally {
            newSampleResult.setRequestHeaders(getRequest());
        }
    }
    SampleResult sampleResult = queue.poll();
    return sampleResult;
}

From source file:com.ibm.crail.tools.CrailBenchmark.java

void writeAsync(String filename, int size, int loop, int batch, int storageClass, int locationClass)
        throws Exception {
    System.out.println("writeAsync, filename " + filename + ", size " + size + ", loop " + loop + ", batch "
            + batch + ", storageClass " + storageClass + ", locationClass " + locationClass);

    ConcurrentLinkedQueue<CrailBuffer> bufferQueue = new ConcurrentLinkedQueue<CrailBuffer>();
    for (int i = 0; i < batch; i++) {
        CrailBuffer buf = null;// www  .  ja v  a 2 s .co m
        if (size == CrailConstants.BUFFER_SIZE) {
            buf = fs.allocateBuffer();
        } else if (size < CrailConstants.BUFFER_SIZE) {
            CrailBuffer _buf = fs.allocateBuffer();
            _buf.clear().limit(size);
            buf = _buf.slice();
        } else {
            buf = OffHeapBuffer.wrap(ByteBuffer.allocateDirect(size));
        }
        bufferQueue.add(buf);
    }

    //warmup
    warmUp(filename, warmup, bufferQueue);

    //benchmark
    System.out.println("starting benchmark...");
    LinkedBlockingQueue<Future<CrailResult>> futureQueue = new LinkedBlockingQueue<Future<CrailResult>>();
    HashMap<Integer, CrailBuffer> futureMap = new HashMap<Integer, CrailBuffer>();
    fs.getStatistics().reset();
    long _loop = (long) loop;
    long _bufsize = (long) CrailConstants.BUFFER_SIZE;
    long _capacity = _loop * _bufsize;
    double sumbytes = 0;
    double ops = 0;
    CrailFile file = fs.create(filename, CrailNodeType.DATAFILE, CrailStorageClass.get(storageClass),
            CrailLocationClass.get(locationClass)).get().asFile();
    CrailOutputStream directStream = file.getDirectOutputStream(_capacity);
    long start = System.currentTimeMillis();
    for (int i = 0; i < batch - 1 && ops < loop; i++) {
        CrailBuffer buf = bufferQueue.poll();
        buf.clear();
        Future<CrailResult> future = directStream.write(buf);
        futureQueue.add(future);
        futureMap.put(future.hashCode(), buf);
        ops = ops + 1.0;
    }
    while (ops < loop) {
        CrailBuffer buf = bufferQueue.poll();
        buf.clear();
        Future<CrailResult> future = directStream.write(buf);
        futureQueue.add(future);
        futureMap.put(future.hashCode(), buf);

        future = futureQueue.poll();
        future.get();
        buf = futureMap.get(future.hashCode());
        bufferQueue.add(buf);

        sumbytes = sumbytes + buf.capacity();
        ops = ops + 1.0;
    }
    while (!futureQueue.isEmpty()) {
        Future<CrailResult> future = futureQueue.poll();
        future.get();
        CrailBuffer buf = futureMap.get(future.hashCode());
        sumbytes = sumbytes + buf.capacity();
        ops = ops + 1.0;
    }
    long end = System.currentTimeMillis();
    double executionTime = ((double) (end - start)) / 1000.0;
    double throughput = 0.0;
    double latency = 0.0;
    double sumbits = sumbytes * 8.0;
    if (executionTime > 0) {
        throughput = sumbits / executionTime / 1000.0 / 1000.0;
        latency = 1000000.0 * executionTime / ops;
    }
    directStream.close();

    System.out.println("execution time " + executionTime);
    System.out.println("ops " + ops);
    System.out.println("sumbytes " + sumbytes);
    System.out.println("throughput " + throughput);
    System.out.println("latency " + latency);

    fs.getStatistics().print("close");
}

From source file:org.wso2.developerstudio.eclipse.greg.manager.remote.views.RegistryBrowserView.java

private String searchRegistryNodeForResource(RegistryNode node, String caption)
        throws InvalidRegistryURLException, UnknownRegistryException {
    ConcurrentLinkedQueue<RegistryResourceNode> queue = new ConcurrentLinkedQueue<RegistryResourceNode>();
    queue.addAll(node.getRegistryContainer().getRegistryContent());
    while (queue.peek() != null) {
        RegistryResourceNode registryResourceNode = queue.poll();
        if (caption.equalsIgnoreCase(registryResourceNode.getCaption())) {
            return registryResourceNode.getRegistryResourcePath();
        } else {//from   w  ww .j ava2 s  .  c  om
            queue.addAll(registryResourceNode.getResourceNodeList());
        }
    }

    //      for (RegistryResourceNode registryResourceNode : queue) {
    //         if(caption.equalsIgnoreCase(registryResourceNode.getCaption())){
    //              return registryResourceNode.getRegistryResourcePath();
    //           }else{
    //              queue.addAll(registryResourceNode.getResourceNodeList());
    //           }   
    //        }
    return null;
}

From source file:org.wso2.developerstudio.eclipse.greg.manager.remote.views.RegistryBrowserView.java

private RegistryResourceNode searchRegistryNodeForResourceNode(RegistryNode node, String caption)
        throws InvalidRegistryURLException, UnknownRegistryException {
    ConcurrentLinkedQueue<RegistryResourceNode> queue = new ConcurrentLinkedQueue<RegistryResourceNode>();
    queue.addAll(node.getRegistryContainer().getRegistryContent());
    while (queue.peek() != null) {
        RegistryResourceNode registryResourceNode = queue.poll();
        if (caption.equalsIgnoreCase(registryResourceNode.getCaption())) {
            return registryResourceNode;
        } else {/* w ww.  j  a v a  2s .co  m*/
            queue.addAll(registryResourceNode.getResourceNodeList());
        }
    }

    //      for (RegistryResourceNode registryResourceNode : queue) {
    //         if(caption.equalsIgnoreCase(registryResourceNode.getCaption())){
    //              return registryResourceNode.getRegistryResourcePath();
    //           }else{
    //              queue.addAll(registryResourceNode.getResourceNodeList());
    //           }   
    //        }
    return null;
}

From source file:dendroscope.autumn.hybridnumber.ComputeHybridNumber.java

/**
 * recursively compute the hybrid number
 *
 * @param root1//from www .  ja  v  a2 s  .  c  om
 * @param root2
 * @param isReduced       @return hybrid number
 * @param retry
 * @param topLevel
 * @param scoreAbove
 * @param additionalAbove
 */
private int computeHybridNumberRec(final Root root1, final Root root2, boolean isReduced,
        Integer previousHybrid, BitSet retry, final boolean topLevel, final int scoreAbove,
        final ValuesList additionalAbove) throws IOException, CanceledException {
    if (System.currentTimeMillis() > nextTime) {
        synchronized (progressListener) {
            nextTime += waitTime;
            waitTime *= 1.5;
            progressListener.incrementProgress();
        }
    } else
        progressListener.checkForCancel();

    // System.err.println("computeHybridNumberRec: tree1=" + Basic.toString(root1.getTaxa()) + " tree2=" + Basic.toString(root2.getTaxa()));
    // root1.reorderSubTree();
    //  root2.reorderSubTree();
    if (checking) {
        root1.checkTree();
        root2.checkTree();
    }

    BitSet taxa = root1.getTaxa();

    String key = root1.toStringTreeSparse() + root2.toStringTreeSparse();
    // System.err.println("Key: "+key);
    Integer value;
    synchronized (lookupTable) {
        value = (Integer) lookupTable.get(key);
        if (value != null)
            return value;
    }

    if (!root2.getTaxa().equals(taxa))
        throw new RuntimeException("Unequal taxon sets: X=" + Basic.toString(root1.getTaxa()) + " vs "
                + Basic.toString(root2.getTaxa()));
    if (!isReduced) {
        switch (SubtreeReduction.apply(root1, root2, null)) {
        case ISOMORPHIC:
            synchronized (lookupTable) {
                lookupTable.put(key, 0);
            }
            if (topLevel) {
                bestScore.lowerTo(0);
                progressListener.setSubtask("Best score: " + bestScore);
            }
            return 0; // two trees are isomorphic, no hybrid node needed
        case REDUCED: // a reduction was performed, cannot maintain lexicographical ordering in removal loop below
            previousHybrid = null;
            break;
        case IRREDUCIBLE:
            break;
        }

        Single<Integer> placeHolderTaxa = new Single<Integer>();
        final Pair<Root, Root> clusterTrees = ClusterReduction.apply(root1, root2, placeHolderTaxa);
        final boolean retryTop = false && (previousHybrid != null && placeHolderTaxa.get() < previousHybrid);
        // if the taxa involved in the cluster reduction come before the previously removed hybrid, do full retry
        // retryTop doesn't work
        final BitSet fRetry = retry;

        if (clusterTrees != null) // will perform cluster-reduction
        {
            final Value score1 = new Value(0);
            final Value score2 = new Value(1); // because the cluster could not be reduced using an subtree reduction, can assume that we will need one reticulation for this

            final boolean verbose = ProgramProperties.get("verbose-HL-parallel", false);
            if (verbose)
                System.err.println("Starting parallel loop");

            final CountDownLatch countDownLatch = new CountDownLatch(2);
            final Integer fPrevious = previousHybrid;

            // setup task:
            final Task task1 = new Task(); // first of two cluster-reduction tasks
            task1.setRunnable(new Runnable() {
                public void run() {
                    try {
                        if (verbose) {
                            System.err.println("Launching thread on cluster-reduction");
                            System.err
                                    .println("Active threads " + scheduledThreadPoolExecutor.getActiveCount());
                        }
                        final ValuesList additionalAbove1 = additionalAbove.copyWithAdditionalElement(score2);
                        if (scoreAbove + additionalAbove1.sum() < bestScore.get()) {
                            int h = computeHybridNumberRec(root1, root2, false, fPrevious, fRetry, false,
                                    scoreAbove, additionalAbove1);
                            score1.set(h);
                        } else {
                            score1.set(LARGE);
                        }
                        additionalAbove1.clear();
                    } catch (Exception ex) {
                        while (countDownLatch.getCount() > 0)
                            countDownLatch.countDown();
                    }
                    countDownLatch.countDown();
                }
            });

            final Task task2 = new Task(); // second of two cluster-reduction tasks
            task2.setRunnable(new Runnable() {
                public void run() {
                    try {
                        if (verbose) {
                            System.err.println("Launching thread on cluster-reduction");
                            System.err
                                    .println("Active threads " + scheduledThreadPoolExecutor.getActiveCount());
                        }
                        final ValuesList additionalAbove2 = additionalAbove.copyWithAdditionalElement(score1);
                        if (scoreAbove + additionalAbove2.sum() < bestScore.get()) {
                            int h = computeHybridNumberRec(clusterTrees.getFirst(), clusterTrees.getSecond(),
                                    true, fPrevious, fRetry, false, scoreAbove, additionalAbove2);
                            score2.set(h);
                        } else {
                            score2.set(LARGE);
                        }
                        additionalAbove2.clear();
                    } catch (Exception ex) {
                        while (countDownLatch.getCount() > 0)
                            countDownLatch.countDown();
                    }
                    countDownLatch.countDown();
                }
            });

            // start a task in this thread
            scheduledThreadPoolExecutor.execute(task1);
            task2.run();
            task1.run(); // try to run task1 in current thread if it hasn't yet started execution. If the task is already running or has completed, will simply return

            try {
                if (verbose)
                    System.err.println("waiting...");
                // wait until all tasks have completed
                countDownLatch.await();
                if (verbose)
                    System.err.println("done");
            } catch (InterruptedException e) {
                Basic.caught(e);
            }

            clusterTrees.getFirst().deleteSubTree();
            clusterTrees.getSecond().deleteSubTree();

            int total = scoreAbove + additionalAbove.sum() + score1.get() + score2.get();

            if (topLevel && (total < bestScore.get())) // score above will be zero, but put this here anyway to avoid confusion
            {
                bestScore.lowerTo(total);
                progressListener.setSubtask("Current best score: " + bestScore);
            }

            synchronized (lookupTable) {
                Integer old = (Integer) lookupTable.get(key);
                if (old == null || total < old)
                    lookupTable.put(key, total);
            }
            return score1.get() + score2.get();
        }
    }

    List<Root> leaves1 = root1.getAllLeaves();

    if (leaves1.size() <= 2) // try 2 rather than one...
    {
        return 0;
    }

    final boolean verbose = ProgramProperties.get("verbose-HL-parallel", false);
    if (verbose)
        System.err.println("Starting parallel loop");

    final CountDownLatch countDownLatch = new CountDownLatch(leaves1.size());

    final Value bestSubH = new Value(LARGE);

    // schedule all tasks to be performed
    final ConcurrentLinkedQueue<Task> queue = new ConcurrentLinkedQueue<Task>();

    for (Node leaf2remove : leaves1) {
        final BitSet taxa2remove = ((Root) leaf2remove).getTaxa();

        if (previousHybrid == null || previousHybrid < taxa2remove.nextSetBit(0)) {

            if (scoreAbove + additionalAbove.sum() + 1 >= bestScore.get())
                return LARGE; // other thread has found a better result, abort

            // setup task:
            final Task task = new Task();
            task.setRunnable(new Runnable() {
                public void run() {
                    try {
                        if (verbose) {
                            System.err.println("Launching thread on " + Basic.toString(taxa2remove));
                            System.err
                                    .println("Active threads " + scheduledThreadPoolExecutor.getActiveCount());
                        }
                        queue.remove(task);
                        if (scoreAbove + additionalAbove.sum() + 1 < bestScore.get()) {
                            Root tree1X = CopyWithTaxaRemoved.apply(root1, taxa2remove);
                            Root tree2X = CopyWithTaxaRemoved.apply(root2, taxa2remove);

                            Refine.apply(tree1X, tree2X);

                            int scoreBelow = computeHybridNumberRec(tree1X, tree2X, false,
                                    taxa2remove.nextSetBit(0), null, false, scoreAbove + 1, additionalAbove)
                                    + 1;

                            if (topLevel && scoreBelow < bestScore.get()) {
                                bestScore.lowerTo(scoreBelow);
                                progressListener.setSubtask("Current best score: " + bestScore);
                            }

                            synchronized (bestSubH) {
                                if (scoreBelow < bestSubH.get())
                                    bestSubH.set(scoreBelow);
                            }

                            tree1X.deleteSubTree();
                            tree2X.deleteSubTree();
                        }
                    } catch (Exception ex) {
                        while (countDownLatch.getCount() > 0)
                            countDownLatch.countDown();
                    }
                    countDownLatch.countDown();
                }
            });
            queue.add(task);
        } else // no task for this item, count down
        {
            countDownLatch.countDown();
            progressListener.checkForCancel();
        }
    }
    // grab one task for the current thread:
    Task taskForCurrentThread = queue.size() > 0 ? queue.poll() : null;
    // launch all others in the executor
    for (Task task : queue)
        scheduledThreadPoolExecutor.execute(task);

    // start a task in this thread
    if (taskForCurrentThread != null)
        taskForCurrentThread.run();

    // try to run other tasks from the queue. Note that any task that is already running will return immediately
    while (queue.size() > 0) {
        Task task = queue.poll();
        if (task != null)
            task.run();
    }
    try {
        if (verbose)
            System.err.println("waiting...");
        // wait until all tasks have completed
        countDownLatch.await();

        if (verbose)
            System.err.println("done");
    } catch (InterruptedException e) {
        Basic.caught(e);
        return LARGE;
    }
    // return the best value
    synchronized (lookupTable) {
        Integer old = (Integer) lookupTable.get(key);
        if (old == null || old > bestSubH.get())
            lookupTable.put(key, bestSubH.get());
    }
    return bestSubH.get();
}

From source file:com.datatorrent.stram.StreamingContainerManager.java

/**
 * process the heartbeat from each container.
 * called by the RPC thread for each container. (i.e. called by multiple threads)
 *
 * @param heartbeat//from   w w w .ja v  a 2s .  co m
 * @return heartbeat response
 */
@SuppressWarnings("StatementWithEmptyBody")
public ContainerHeartbeatResponse processHeartbeat(ContainerHeartbeat heartbeat) {
    long currentTimeMillis = clock.getTime();

    final StreamingContainerAgent sca = this.containers.get(heartbeat.getContainerId());
    if (sca == null || sca.container.getState() == PTContainer.State.KILLED) {
        // could be orphaned container that was replaced and needs to terminate
        LOG.error("Unknown container {}", heartbeat.getContainerId());
        ContainerHeartbeatResponse response = new ContainerHeartbeatResponse();
        response.shutdown = true;
        return response;
    }

    //LOG.debug("{} {} {}", new Object[]{sca.container.containerId, sca.container.bufferServerAddress, sca.container.getState()});
    if (sca.container.getState() == PTContainer.State.ALLOCATED) {
        // capture dynamically assigned address from container
        if (sca.container.bufferServerAddress == null && heartbeat.bufferServerHost != null) {
            sca.container.bufferServerAddress = InetSocketAddress.createUnresolved(heartbeat.bufferServerHost,
                    heartbeat.bufferServerPort);
            LOG.info("Container {} buffer server: {}", sca.container.getExternalId(),
                    sca.container.bufferServerAddress);
        }
        final long containerStartTime = System.currentTimeMillis();
        sca.container.setState(PTContainer.State.ACTIVE);
        sca.container.setStartedTime(containerStartTime);
        sca.container.setFinishedTime(-1);
        sca.jvmName = heartbeat.jvmName;
        poolExecutor.submit(new Runnable() {
            @Override
            public void run() {
                try {
                    containerFile.append(sca.getContainerInfo());
                } catch (IOException ex) {
                    LOG.warn("Cannot write to container file");
                }
                for (PTOperator ptOp : sca.container.getOperators()) {
                    try {
                        JSONObject operatorInfo = new JSONObject();
                        operatorInfo.put("name", ptOp.getName());
                        operatorInfo.put("id", ptOp.getId());
                        operatorInfo.put("container", sca.container.getExternalId());
                        operatorInfo.put("startTime", containerStartTime);
                        operatorFile.append(operatorInfo);
                    } catch (IOException | JSONException ex) {
                        LOG.warn("Cannot write to operator file: ", ex);
                    }
                }
            }
        });
    }

    if (heartbeat.restartRequested) {
        LOG.error("Container {} restart request", sca.container.getExternalId());
        containerStopRequests.put(sca.container.getExternalId(), sca.container.getExternalId());
    }

    sca.memoryMBFree = heartbeat.memoryMBFree;
    sca.gcCollectionCount = heartbeat.gcCollectionCount;
    sca.gcCollectionTime = heartbeat.gcCollectionTime;

    sca.undeployOpers.clear();
    sca.deployOpers.clear();
    if (!this.deployChangeInProgress.get()) {
        sca.deployCnt = this.deployChangeCnt;
    }
    Set<Integer> reportedOperators = Sets.newHashSetWithExpectedSize(sca.container.getOperators().size());

    for (OperatorHeartbeat shb : heartbeat.getContainerStats().operators) {

        long maxEndWindowTimestamp = 0;

        reportedOperators.add(shb.nodeId);
        PTOperator oper = this.plan.getAllOperators().get(shb.getNodeId());

        if (oper == null) {
            LOG.info("Heartbeat for unknown operator {} (container {})", shb.getNodeId(),
                    heartbeat.getContainerId());
            sca.undeployOpers.add(shb.nodeId);
            continue;
        }

        if (shb.requestResponse != null) {
            for (StatsListener.OperatorResponse obj : shb.requestResponse) {
                if (obj instanceof OperatorResponse) { // This is to identify platform requests
                    commandResponse.put((Long) obj.getResponseId(), obj.getResponse());
                    LOG.debug(" Got back the response {} for the request {}", obj, obj.getResponseId());
                } else { // This is to identify user requests
                    oper.stats.responses.add(obj);
                }
            }
        }

        //LOG.debug("heartbeat {} {}/{} {}", oper, oper.getState(), shb.getState(), oper.getContainer().getExternalId());
        if (!(oper.getState() == PTOperator.State.ACTIVE
                && shb.getState() == OperatorHeartbeat.DeployState.ACTIVE)) {
            // deploy state may require synchronization
            processOperatorDeployStatus(oper, shb, sca);
        }

        oper.stats.lastHeartbeat = shb;
        List<ContainerStats.OperatorStats> statsList = shb.getOperatorStatsContainer();

        if (!statsList.isEmpty()) {
            long tuplesProcessed = 0;
            long tuplesEmitted = 0;
            long totalCpuTimeUsed = 0;
            int statCount = 0;
            long maxDequeueTimestamp = -1;
            oper.stats.recordingId = null;

            final OperatorStatus status = oper.stats;
            status.statsRevs.checkout();

            for (Map.Entry<String, PortStatus> entry : status.inputPortStatusList.entrySet()) {
                entry.getValue().recordingId = null;
            }
            for (Map.Entry<String, PortStatus> entry : status.outputPortStatusList.entrySet()) {
                entry.getValue().recordingId = null;
            }
            for (ContainerStats.OperatorStats stats : statsList) {
                if (stats == null) {
                    LOG.warn("Operator {} statistics list contains null element", shb.getNodeId());
                    continue;
                }

                /* report checkpoint-ed WindowId status of the operator */
                if (stats.checkpoint instanceof Checkpoint) {
                    if (oper.getRecentCheckpoint() == null
                            || oper.getRecentCheckpoint().windowId < stats.checkpoint.getWindowId()) {
                        addCheckpoint(oper, (Checkpoint) stats.checkpoint);
                        if (stats.checkpointStats != null) {
                            status.checkpointStats = stats.checkpointStats;
                            status.checkpointTimeMA.add(stats.checkpointStats.checkpointTime);
                        }
                        oper.failureCount = 0;
                    }
                }

                oper.stats.recordingId = stats.recordingId;

                /* report all the other stuff */

                // calculate the stats related to end window
                EndWindowStats endWindowStats = new EndWindowStats(); // end window stats for a particular window id for a particular node
                Collection<ContainerStats.OperatorStats.PortStats> ports = stats.inputPorts;
                if (ports != null) {
                    Set<String> currentInputPortSet = Sets.newHashSetWithExpectedSize(ports.size());
                    for (ContainerStats.OperatorStats.PortStats s : ports) {
                        currentInputPortSet.add(s.id);
                        PortStatus ps = status.inputPortStatusList.get(s.id);
                        if (ps == null) {
                            ps = status.new PortStatus();
                            ps.portName = s.id;
                            status.inputPortStatusList.put(s.id, ps);
                        }
                        ps.totalTuples += s.tupleCount;
                        ps.recordingId = s.recordingId;

                        tuplesProcessed += s.tupleCount;
                        endWindowStats.dequeueTimestamps.put(s.id, s.endWindowTimestamp);

                        Pair<Integer, String> operatorPortName = new Pair<>(oper.getId(), s.id);
                        Long lastEndWindowTimestamp = operatorPortLastEndWindowTimestamps.get(operatorPortName);
                        if (lastEndWindowTimestamp == null) {
                            lastEndWindowTimestamp = lastStatsTimestamp;
                        }
                        long portElapsedMillis = Math.max(s.endWindowTimestamp - lastEndWindowTimestamp, 0);
                        //LOG.debug("=== PROCESSED TUPLE COUNT for {}: {}, {}, {}, {}", operatorPortName, s.tupleCount, portElapsedMillis, operatorPortLastEndWindowTimestamps.get(operatorPortName), lastStatsTimestamp);
                        ps.tuplesPMSMA.add(s.tupleCount, portElapsedMillis);
                        ps.bufferServerBytesPMSMA.add(s.bufferServerBytes, portElapsedMillis);
                        ps.queueSizeMA.add(s.queueSize);

                        operatorPortLastEndWindowTimestamps.put(operatorPortName, s.endWindowTimestamp);
                        if (maxEndWindowTimestamp < s.endWindowTimestamp) {
                            maxEndWindowTimestamp = s.endWindowTimestamp;
                        }
                        if (s.endWindowTimestamp > maxDequeueTimestamp) {
                            maxDequeueTimestamp = s.endWindowTimestamp;
                        }
                    }
                    // need to remove dead ports, for unifiers
                    Iterator<Map.Entry<String, PortStatus>> it = status.inputPortStatusList.entrySet()
                            .iterator();
                    while (it.hasNext()) {
                        Map.Entry<String, PortStatus> entry = it.next();
                        if (!currentInputPortSet.contains(entry.getKey())) {
                            it.remove();
                        }
                    }
                }

                ports = stats.outputPorts;
                if (ports != null) {
                    Set<String> currentOutputPortSet = Sets.newHashSetWithExpectedSize(ports.size());
                    for (ContainerStats.OperatorStats.PortStats s : ports) {
                        currentOutputPortSet.add(s.id);
                        PortStatus ps = status.outputPortStatusList.get(s.id);
                        if (ps == null) {
                            ps = status.new PortStatus();
                            ps.portName = s.id;
                            status.outputPortStatusList.put(s.id, ps);
                        }
                        ps.totalTuples += s.tupleCount;
                        ps.recordingId = s.recordingId;

                        tuplesEmitted += s.tupleCount;
                        Pair<Integer, String> operatorPortName = new Pair<>(oper.getId(), s.id);
                        Long lastEndWindowTimestamp = operatorPortLastEndWindowTimestamps.get(operatorPortName);
                        if (lastEndWindowTimestamp == null) {
                            lastEndWindowTimestamp = lastStatsTimestamp;
                        }
                        long portElapsedMillis = Math.max(s.endWindowTimestamp - lastEndWindowTimestamp, 0);
                        //LOG.debug("=== EMITTED TUPLE COUNT for {}: {}, {}, {}, {}", operatorPortName, s.tupleCount, portElapsedMillis, operatorPortLastEndWindowTimestamps.get(operatorPortName), lastStatsTimestamp);
                        ps.tuplesPMSMA.add(s.tupleCount, portElapsedMillis);
                        ps.bufferServerBytesPMSMA.add(s.bufferServerBytes, portElapsedMillis);

                        operatorPortLastEndWindowTimestamps.put(operatorPortName, s.endWindowTimestamp);
                        if (maxEndWindowTimestamp < s.endWindowTimestamp) {
                            maxEndWindowTimestamp = s.endWindowTimestamp;
                        }
                    }
                    if (ports.size() > 0) {
                        endWindowStats.emitTimestamp = ports.iterator().next().endWindowTimestamp;
                    }
                    // need to remove dead ports, for unifiers
                    Iterator<Map.Entry<String, PortStatus>> it = status.outputPortStatusList.entrySet()
                            .iterator();
                    while (it.hasNext()) {
                        Map.Entry<String, PortStatus> entry = it.next();
                        if (!currentOutputPortSet.contains(entry.getKey())) {
                            it.remove();
                        }
                    }
                }

                // for output operator, just take the maximum dequeue time for emit timestamp.
                // (we don't know the latency for output operators because they don't emit tuples)
                if (endWindowStats.emitTimestamp < 0) {
                    endWindowStats.emitTimestamp = maxDequeueTimestamp;
                }

                if (status.currentWindowId.get() != stats.windowId) {
                    status.lastWindowIdChangeTms = currentTimeMillis;
                    status.currentWindowId.set(stats.windowId);
                }
                totalCpuTimeUsed += stats.cpuTimeUsed;
                statCount++;

                if (oper.getOperatorMeta().getValue(OperatorContext.COUNTERS_AGGREGATOR) != null) {
                    endWindowStats.counters = stats.counters;
                }
                if (oper.getOperatorMeta().getMetricAggregatorMeta() != null
                        && oper.getOperatorMeta().getMetricAggregatorMeta().getAggregator() != null) {
                    endWindowStats.metrics = stats.metrics;
                }

                if (stats.windowId > currentEndWindowStatsWindowId) {
                    Map<Integer, EndWindowStats> endWindowStatsMap = endWindowStatsOperatorMap
                            .get(stats.windowId);
                    if (endWindowStatsMap == null) {
                        endWindowStatsMap = new ConcurrentSkipListMap<Integer, EndWindowStats>();
                        Map<Integer, EndWindowStats> endWindowStatsMapPrevious = endWindowStatsOperatorMap
                                .putIfAbsent(stats.windowId, endWindowStatsMap);
                        if (endWindowStatsMapPrevious != null) {
                            endWindowStatsMap = endWindowStatsMapPrevious;
                        }
                    }
                    endWindowStatsMap.put(shb.getNodeId(), endWindowStats);

                    if (!oper.getInputs().isEmpty()) {
                        long latency = Long.MAX_VALUE;
                        long adjustedEndWindowEmitTimestamp = endWindowStats.emitTimestamp;
                        MovingAverageLong rpcLatency = rpcLatencies.get(oper.getContainer().getExternalId());
                        if (rpcLatency != null) {
                            adjustedEndWindowEmitTimestamp += rpcLatency.getAvg();
                        }
                        PTOperator slowestUpstream = null;
                        for (PTInput input : oper.getInputs()) {
                            PTOperator upstreamOp = input.source.source;
                            if (upstreamOp.getOperatorMeta().getOperator() instanceof Operator.DelayOperator) {
                                continue;
                            }
                            EndWindowStats ews = endWindowStatsMap.get(upstreamOp.getId());
                            long portLatency;
                            if (ews == null) {
                                // This is when the operator is likely to be behind too many windows. We need to give an estimate for
                                // latency at this point, by looking at the number of windows behind
                                int widthMillis = plan.getLogicalPlan()
                                        .getValue(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS);
                                portLatency = (upstreamOp.stats.currentWindowId.get()
                                        - oper.stats.currentWindowId.get()) * widthMillis;
                            } else {
                                MovingAverageLong upstreamRPCLatency = rpcLatencies
                                        .get(upstreamOp.getContainer().getExternalId());
                                portLatency = adjustedEndWindowEmitTimestamp - ews.emitTimestamp;
                                if (upstreamRPCLatency != null) {
                                    portLatency -= upstreamRPCLatency.getAvg();
                                }
                            }
                            if (portLatency < 0) {
                                portLatency = 0;
                            }
                            if (latency > portLatency) {
                                latency = portLatency;
                                slowestUpstream = upstreamOp;
                            }
                        }
                        status.latencyMA.add(latency);
                        slowestUpstreamOp.put(oper, slowestUpstream);
                    }

                    Set<Integer> allCurrentOperators = plan.getAllOperators().keySet();
                    int numOperators = plan.getAllOperators().size();
                    if (allCurrentOperators.containsAll(endWindowStatsMap.keySet())
                            && endWindowStatsMap.size() == numOperators) {
                        completeEndWindowStatsWindowId = stats.windowId;
                    }
                }
            }

            status.totalTuplesProcessed.add(tuplesProcessed);
            status.totalTuplesEmitted.add(tuplesEmitted);
            OperatorMeta logicalOperator = oper.getOperatorMeta();
            LogicalOperatorStatus logicalStatus = logicalOperator.getStatus();
            if (!oper.isUnifier()) {
                logicalStatus.totalTuplesProcessed += tuplesProcessed;
                logicalStatus.totalTuplesEmitted += tuplesEmitted;
            }
            long lastMaxEndWindowTimestamp = operatorLastEndWindowTimestamps.containsKey(oper.getId())
                    ? operatorLastEndWindowTimestamps.get(oper.getId())
                    : lastStatsTimestamp;
            if (maxEndWindowTimestamp >= lastMaxEndWindowTimestamp) {
                double tuplesProcessedPMSMA = 0.0;
                double tuplesEmittedPMSMA = 0.0;
                if (statCount != 0) {
                    //LOG.debug("CPU for {}: {} / {} - {}", oper.getId(), totalCpuTimeUsed, maxEndWindowTimestamp, lastMaxEndWindowTimestamp);
                    status.cpuNanosPMSMA.add(totalCpuTimeUsed,
                            maxEndWindowTimestamp - lastMaxEndWindowTimestamp);
                }

                for (PortStatus ps : status.inputPortStatusList.values()) {
                    tuplesProcessedPMSMA += ps.tuplesPMSMA.getAvg();
                }
                for (PortStatus ps : status.outputPortStatusList.values()) {
                    tuplesEmittedPMSMA += ps.tuplesPMSMA.getAvg();
                }
                status.tuplesProcessedPSMA.set(Math.round(tuplesProcessedPMSMA * 1000));
                status.tuplesEmittedPSMA.set(Math.round(tuplesEmittedPMSMA * 1000));
            } else {
                //LOG.warn("This timestamp for {} is lower than the previous!! {} < {}", oper.getId(), maxEndWindowTimestamp, lastMaxEndWindowTimestamp);
            }
            operatorLastEndWindowTimestamps.put(oper.getId(), maxEndWindowTimestamp);
            status.listenerStats.add(statsList);
            this.reportStats.put(oper, oper);

            status.statsRevs.commit();
        }
        if (lastStatsTimestamp < maxEndWindowTimestamp) {
            lastStatsTimestamp = maxEndWindowTimestamp;
        }
    }

    sca.lastHeartbeatMillis = currentTimeMillis;

    for (PTOperator oper : sca.container.getOperators()) {
        if (!reportedOperators.contains(oper.getId())) {
            processOperatorDeployStatus(oper, null, sca);
        }
    }

    ContainerHeartbeatResponse rsp = getHeartbeatResponse(sca);

    if (heartbeat.getContainerStats().operators.isEmpty() && isApplicationIdle()) {
        LOG.info("requesting idle shutdown for container {}", heartbeat.getContainerId());
        rsp.shutdown = true;
    } else {
        if (sca.shutdownRequested) {
            LOG.info("requesting shutdown for container {}", heartbeat.getContainerId());
            rsp.shutdown = true;
        }
    }

    List<StramToNodeRequest> requests = rsp.nodeRequests != null ? rsp.nodeRequests
            : new ArrayList<StramToNodeRequest>();
    ConcurrentLinkedQueue<StramToNodeRequest> operatorRequests = sca.getOperatorRequests();
    while (true) {
        StramToNodeRequest r = operatorRequests.poll();
        if (r == null) {
            break;
        }
        requests.add(r);
    }
    rsp.nodeRequests = requests;
    rsp.committedWindowId = committedWindowId;
    return rsp;
}