List of usage examples for java.util.concurrent ConcurrentSkipListMap ConcurrentSkipListMap
public ConcurrentSkipListMap()
From source file:org.apache.hadoop.hbase.coprocessor.client.TimeseriesAggregationClient.java
/** * This is the client side interface/handle for calling the average method for a given cf-cq * combination. It was necessary to add one more call stack as its return type should be a decimal * value, irrespective of what columninterpreter says. So, this methods collects the necessary * parameters to compute the average and returs the double value. * @param table/*from w w w . j a va2 s . co m*/ * @param ci * @param scan * @return <R, S> * @throws Throwable */ public <R, S, P extends Message, Q extends Message, T extends Message> ConcurrentSkipListMap<Long, Double> avg( final Table table, final ColumnInterpreter<R, S, P, Q, T> ci, Scan scan) throws Throwable { ConcurrentSkipListMap<Long, Pair<S, Long>> p = getAvgArgs(table, ci, scan); ConcurrentSkipListMap<Long, Double> avg = new ConcurrentSkipListMap<Long, Double>(); for (Map.Entry<Long, Pair<S, Long>> entry : p.entrySet()) { avg.put(entry.getKey(), ci.divideForAvg(entry.getValue().getFirst(), entry.getValue().getSecond())); } return avg; }
From source file:com.palantir.atlasdb.transaction.impl.SnapshotTransaction.java
private ConcurrentNavigableMap<Cell, byte[]> getLocalWrites(String tableName) { ConcurrentNavigableMap<Cell, byte[]> writes = writesByTable.get(tableName); if (writes == null) { writes = new ConcurrentSkipListMap<Cell, byte[]>(); ConcurrentNavigableMap<Cell, byte[]> previous = writesByTable.putIfAbsent(tableName, writes); if (previous != null) { writes = previous;//from w ww.ja v a 2s . c om } } return writes; }
From source file:com.jivesoftware.os.upena.deployable.UpenaMain.java
private RowsStorageProvider rowsStorageProvider(final OrderIdProvider orderIdProvider) { return (workingDirectory, tableDomain, tableName) -> { final File directory = new File(workingDirectory, tableDomain); directory.mkdirs();//from w w w .j av a 2 s .c o m File file = new File(directory, tableName.getTableName() + ".kvt"); BinaryRowMarshaller rowMarshaller = new BinaryRowMarshaller(); RowsIndexProvider tableIndexProvider = (tableName1) -> { NavigableMap<RowIndexKey, RowIndexValue> navigableMap = new ConcurrentSkipListMap<>(); return new MemoryRowsIndex(navigableMap); }; return new RowTable(tableName, orderIdProvider, rowMarshaller, new BinaryRowsTx(file, rowMarshaller, tableIndexProvider, 1000)); }; }
From source file:com.datatorrent.stram.StreamingContainerManager.java
/** * process the heartbeat from each container. * called by the RPC thread for each container. (i.e. called by multiple threads) * * @param heartbeat//from ww w . j a v a 2 s . com * @return heartbeat response */ @SuppressWarnings("StatementWithEmptyBody") public ContainerHeartbeatResponse processHeartbeat(ContainerHeartbeat heartbeat) { long currentTimeMillis = clock.getTime(); final StreamingContainerAgent sca = this.containers.get(heartbeat.getContainerId()); if (sca == null || sca.container.getState() == PTContainer.State.KILLED) { // could be orphaned container that was replaced and needs to terminate LOG.error("Unknown container {}", heartbeat.getContainerId()); ContainerHeartbeatResponse response = new ContainerHeartbeatResponse(); response.shutdown = true; return response; } //LOG.debug("{} {} {}", new Object[]{sca.container.containerId, sca.container.bufferServerAddress, sca.container.getState()}); if (sca.container.getState() == PTContainer.State.ALLOCATED) { // capture dynamically assigned address from container if (sca.container.bufferServerAddress == null && heartbeat.bufferServerHost != null) { sca.container.bufferServerAddress = InetSocketAddress.createUnresolved(heartbeat.bufferServerHost, heartbeat.bufferServerPort); LOG.info("Container {} buffer server: {}", sca.container.getExternalId(), sca.container.bufferServerAddress); } final long containerStartTime = System.currentTimeMillis(); sca.container.setState(PTContainer.State.ACTIVE); sca.container.setStartedTime(containerStartTime); sca.container.setFinishedTime(-1); sca.jvmName = heartbeat.jvmName; poolExecutor.submit(new Runnable() { @Override public void run() { try { containerFile.append(sca.getContainerInfo()); } catch (IOException ex) { LOG.warn("Cannot write to container file"); } for (PTOperator ptOp : sca.container.getOperators()) { try { JSONObject operatorInfo = new JSONObject(); operatorInfo.put("name", ptOp.getName()); operatorInfo.put("id", ptOp.getId()); operatorInfo.put("container", sca.container.getExternalId()); operatorInfo.put("startTime", containerStartTime); operatorFile.append(operatorInfo); } catch (IOException | JSONException ex) { LOG.warn("Cannot write to operator file: ", ex); } } } }); } if (heartbeat.restartRequested) { LOG.error("Container {} restart request", sca.container.getExternalId()); containerStopRequests.put(sca.container.getExternalId(), sca.container.getExternalId()); } sca.memoryMBFree = heartbeat.memoryMBFree; sca.gcCollectionCount = heartbeat.gcCollectionCount; sca.gcCollectionTime = heartbeat.gcCollectionTime; sca.undeployOpers.clear(); sca.deployOpers.clear(); if (!this.deployChangeInProgress.get()) { sca.deployCnt = this.deployChangeCnt; } Set<Integer> reportedOperators = Sets.newHashSetWithExpectedSize(sca.container.getOperators().size()); for (OperatorHeartbeat shb : heartbeat.getContainerStats().operators) { long maxEndWindowTimestamp = 0; reportedOperators.add(shb.nodeId); PTOperator oper = this.plan.getAllOperators().get(shb.getNodeId()); if (oper == null) { LOG.info("Heartbeat for unknown operator {} (container {})", shb.getNodeId(), heartbeat.getContainerId()); sca.undeployOpers.add(shb.nodeId); continue; } if (shb.requestResponse != null) { for (StatsListener.OperatorResponse obj : shb.requestResponse) { if (obj instanceof OperatorResponse) { // This is to identify platform requests commandResponse.put((Long) obj.getResponseId(), obj.getResponse()); LOG.debug(" Got back the response {} for the request {}", obj, obj.getResponseId()); } else { // This is to identify user requests oper.stats.responses.add(obj); } } } //LOG.debug("heartbeat {} {}/{} {}", oper, oper.getState(), shb.getState(), oper.getContainer().getExternalId()); if (!(oper.getState() == PTOperator.State.ACTIVE && shb.getState() == OperatorHeartbeat.DeployState.ACTIVE)) { // deploy state may require synchronization processOperatorDeployStatus(oper, shb, sca); } oper.stats.lastHeartbeat = shb; List<ContainerStats.OperatorStats> statsList = shb.getOperatorStatsContainer(); if (!statsList.isEmpty()) { long tuplesProcessed = 0; long tuplesEmitted = 0; long totalCpuTimeUsed = 0; int statCount = 0; long maxDequeueTimestamp = -1; oper.stats.recordingId = null; final OperatorStatus status = oper.stats; status.statsRevs.checkout(); for (Map.Entry<String, PortStatus> entry : status.inputPortStatusList.entrySet()) { entry.getValue().recordingId = null; } for (Map.Entry<String, PortStatus> entry : status.outputPortStatusList.entrySet()) { entry.getValue().recordingId = null; } for (ContainerStats.OperatorStats stats : statsList) { if (stats == null) { LOG.warn("Operator {} statistics list contains null element", shb.getNodeId()); continue; } /* report checkpoint-ed WindowId status of the operator */ if (stats.checkpoint instanceof Checkpoint) { if (oper.getRecentCheckpoint() == null || oper.getRecentCheckpoint().windowId < stats.checkpoint.getWindowId()) { addCheckpoint(oper, (Checkpoint) stats.checkpoint); if (stats.checkpointStats != null) { status.checkpointStats = stats.checkpointStats; status.checkpointTimeMA.add(stats.checkpointStats.checkpointTime); } oper.failureCount = 0; } } oper.stats.recordingId = stats.recordingId; /* report all the other stuff */ // calculate the stats related to end window EndWindowStats endWindowStats = new EndWindowStats(); // end window stats for a particular window id for a particular node Collection<ContainerStats.OperatorStats.PortStats> ports = stats.inputPorts; if (ports != null) { Set<String> currentInputPortSet = Sets.newHashSetWithExpectedSize(ports.size()); for (ContainerStats.OperatorStats.PortStats s : ports) { currentInputPortSet.add(s.id); PortStatus ps = status.inputPortStatusList.get(s.id); if (ps == null) { ps = status.new PortStatus(); ps.portName = s.id; status.inputPortStatusList.put(s.id, ps); } ps.totalTuples += s.tupleCount; ps.recordingId = s.recordingId; tuplesProcessed += s.tupleCount; endWindowStats.dequeueTimestamps.put(s.id, s.endWindowTimestamp); Pair<Integer, String> operatorPortName = new Pair<>(oper.getId(), s.id); Long lastEndWindowTimestamp = operatorPortLastEndWindowTimestamps.get(operatorPortName); if (lastEndWindowTimestamp == null) { lastEndWindowTimestamp = lastStatsTimestamp; } long portElapsedMillis = Math.max(s.endWindowTimestamp - lastEndWindowTimestamp, 0); //LOG.debug("=== PROCESSED TUPLE COUNT for {}: {}, {}, {}, {}", operatorPortName, s.tupleCount, portElapsedMillis, operatorPortLastEndWindowTimestamps.get(operatorPortName), lastStatsTimestamp); ps.tuplesPMSMA.add(s.tupleCount, portElapsedMillis); ps.bufferServerBytesPMSMA.add(s.bufferServerBytes, portElapsedMillis); ps.queueSizeMA.add(s.queueSize); operatorPortLastEndWindowTimestamps.put(operatorPortName, s.endWindowTimestamp); if (maxEndWindowTimestamp < s.endWindowTimestamp) { maxEndWindowTimestamp = s.endWindowTimestamp; } if (s.endWindowTimestamp > maxDequeueTimestamp) { maxDequeueTimestamp = s.endWindowTimestamp; } } // need to remove dead ports, for unifiers Iterator<Map.Entry<String, PortStatus>> it = status.inputPortStatusList.entrySet() .iterator(); while (it.hasNext()) { Map.Entry<String, PortStatus> entry = it.next(); if (!currentInputPortSet.contains(entry.getKey())) { it.remove(); } } } ports = stats.outputPorts; if (ports != null) { Set<String> currentOutputPortSet = Sets.newHashSetWithExpectedSize(ports.size()); for (ContainerStats.OperatorStats.PortStats s : ports) { currentOutputPortSet.add(s.id); PortStatus ps = status.outputPortStatusList.get(s.id); if (ps == null) { ps = status.new PortStatus(); ps.portName = s.id; status.outputPortStatusList.put(s.id, ps); } ps.totalTuples += s.tupleCount; ps.recordingId = s.recordingId; tuplesEmitted += s.tupleCount; Pair<Integer, String> operatorPortName = new Pair<>(oper.getId(), s.id); Long lastEndWindowTimestamp = operatorPortLastEndWindowTimestamps.get(operatorPortName); if (lastEndWindowTimestamp == null) { lastEndWindowTimestamp = lastStatsTimestamp; } long portElapsedMillis = Math.max(s.endWindowTimestamp - lastEndWindowTimestamp, 0); //LOG.debug("=== EMITTED TUPLE COUNT for {}: {}, {}, {}, {}", operatorPortName, s.tupleCount, portElapsedMillis, operatorPortLastEndWindowTimestamps.get(operatorPortName), lastStatsTimestamp); ps.tuplesPMSMA.add(s.tupleCount, portElapsedMillis); ps.bufferServerBytesPMSMA.add(s.bufferServerBytes, portElapsedMillis); operatorPortLastEndWindowTimestamps.put(operatorPortName, s.endWindowTimestamp); if (maxEndWindowTimestamp < s.endWindowTimestamp) { maxEndWindowTimestamp = s.endWindowTimestamp; } } if (ports.size() > 0) { endWindowStats.emitTimestamp = ports.iterator().next().endWindowTimestamp; } // need to remove dead ports, for unifiers Iterator<Map.Entry<String, PortStatus>> it = status.outputPortStatusList.entrySet() .iterator(); while (it.hasNext()) { Map.Entry<String, PortStatus> entry = it.next(); if (!currentOutputPortSet.contains(entry.getKey())) { it.remove(); } } } // for output operator, just take the maximum dequeue time for emit timestamp. // (we don't know the latency for output operators because they don't emit tuples) if (endWindowStats.emitTimestamp < 0) { endWindowStats.emitTimestamp = maxDequeueTimestamp; } if (status.currentWindowId.get() != stats.windowId) { status.lastWindowIdChangeTms = currentTimeMillis; status.currentWindowId.set(stats.windowId); } totalCpuTimeUsed += stats.cpuTimeUsed; statCount++; if (oper.getOperatorMeta().getValue(OperatorContext.COUNTERS_AGGREGATOR) != null) { endWindowStats.counters = stats.counters; } if (oper.getOperatorMeta().getMetricAggregatorMeta() != null && oper.getOperatorMeta().getMetricAggregatorMeta().getAggregator() != null) { endWindowStats.metrics = stats.metrics; } if (stats.windowId > currentEndWindowStatsWindowId) { Map<Integer, EndWindowStats> endWindowStatsMap = endWindowStatsOperatorMap .get(stats.windowId); if (endWindowStatsMap == null) { endWindowStatsMap = new ConcurrentSkipListMap<Integer, EndWindowStats>(); Map<Integer, EndWindowStats> endWindowStatsMapPrevious = endWindowStatsOperatorMap .putIfAbsent(stats.windowId, endWindowStatsMap); if (endWindowStatsMapPrevious != null) { endWindowStatsMap = endWindowStatsMapPrevious; } } endWindowStatsMap.put(shb.getNodeId(), endWindowStats); if (!oper.getInputs().isEmpty()) { long latency = Long.MAX_VALUE; long adjustedEndWindowEmitTimestamp = endWindowStats.emitTimestamp; MovingAverageLong rpcLatency = rpcLatencies.get(oper.getContainer().getExternalId()); if (rpcLatency != null) { adjustedEndWindowEmitTimestamp += rpcLatency.getAvg(); } PTOperator slowestUpstream = null; for (PTInput input : oper.getInputs()) { PTOperator upstreamOp = input.source.source; if (upstreamOp.getOperatorMeta().getOperator() instanceof Operator.DelayOperator) { continue; } EndWindowStats ews = endWindowStatsMap.get(upstreamOp.getId()); long portLatency; if (ews == null) { // This is when the operator is likely to be behind too many windows. We need to give an estimate for // latency at this point, by looking at the number of windows behind int widthMillis = plan.getLogicalPlan() .getValue(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS); portLatency = (upstreamOp.stats.currentWindowId.get() - oper.stats.currentWindowId.get()) * widthMillis; } else { MovingAverageLong upstreamRPCLatency = rpcLatencies .get(upstreamOp.getContainer().getExternalId()); portLatency = adjustedEndWindowEmitTimestamp - ews.emitTimestamp; if (upstreamRPCLatency != null) { portLatency -= upstreamRPCLatency.getAvg(); } } if (portLatency < 0) { portLatency = 0; } if (latency > portLatency) { latency = portLatency; slowestUpstream = upstreamOp; } } status.latencyMA.add(latency); slowestUpstreamOp.put(oper, slowestUpstream); } Set<Integer> allCurrentOperators = plan.getAllOperators().keySet(); int numOperators = plan.getAllOperators().size(); if (allCurrentOperators.containsAll(endWindowStatsMap.keySet()) && endWindowStatsMap.size() == numOperators) { completeEndWindowStatsWindowId = stats.windowId; } } } status.totalTuplesProcessed.add(tuplesProcessed); status.totalTuplesEmitted.add(tuplesEmitted); OperatorMeta logicalOperator = oper.getOperatorMeta(); LogicalOperatorStatus logicalStatus = logicalOperator.getStatus(); if (!oper.isUnifier()) { logicalStatus.totalTuplesProcessed += tuplesProcessed; logicalStatus.totalTuplesEmitted += tuplesEmitted; } long lastMaxEndWindowTimestamp = operatorLastEndWindowTimestamps.containsKey(oper.getId()) ? operatorLastEndWindowTimestamps.get(oper.getId()) : lastStatsTimestamp; if (maxEndWindowTimestamp >= lastMaxEndWindowTimestamp) { double tuplesProcessedPMSMA = 0.0; double tuplesEmittedPMSMA = 0.0; if (statCount != 0) { //LOG.debug("CPU for {}: {} / {} - {}", oper.getId(), totalCpuTimeUsed, maxEndWindowTimestamp, lastMaxEndWindowTimestamp); status.cpuNanosPMSMA.add(totalCpuTimeUsed, maxEndWindowTimestamp - lastMaxEndWindowTimestamp); } for (PortStatus ps : status.inputPortStatusList.values()) { tuplesProcessedPMSMA += ps.tuplesPMSMA.getAvg(); } for (PortStatus ps : status.outputPortStatusList.values()) { tuplesEmittedPMSMA += ps.tuplesPMSMA.getAvg(); } status.tuplesProcessedPSMA.set(Math.round(tuplesProcessedPMSMA * 1000)); status.tuplesEmittedPSMA.set(Math.round(tuplesEmittedPMSMA * 1000)); } else { //LOG.warn("This timestamp for {} is lower than the previous!! {} < {}", oper.getId(), maxEndWindowTimestamp, lastMaxEndWindowTimestamp); } operatorLastEndWindowTimestamps.put(oper.getId(), maxEndWindowTimestamp); status.listenerStats.add(statsList); this.reportStats.put(oper, oper); status.statsRevs.commit(); } if (lastStatsTimestamp < maxEndWindowTimestamp) { lastStatsTimestamp = maxEndWindowTimestamp; } } sca.lastHeartbeatMillis = currentTimeMillis; for (PTOperator oper : sca.container.getOperators()) { if (!reportedOperators.contains(oper.getId())) { processOperatorDeployStatus(oper, null, sca); } } ContainerHeartbeatResponse rsp = getHeartbeatResponse(sca); if (heartbeat.getContainerStats().operators.isEmpty() && isApplicationIdle()) { LOG.info("requesting idle shutdown for container {}", heartbeat.getContainerId()); rsp.shutdown = true; } else { if (sca.shutdownRequested) { LOG.info("requesting shutdown for container {}", heartbeat.getContainerId()); rsp.shutdown = true; } } List<StramToNodeRequest> requests = rsp.nodeRequests != null ? rsp.nodeRequests : new ArrayList<StramToNodeRequest>(); ConcurrentLinkedQueue<StramToNodeRequest> operatorRequests = sca.getOperatorRequests(); while (true) { StramToNodeRequest r = operatorRequests.poll(); if (r == null) { break; } requests.add(r); } rsp.nodeRequests = requests; rsp.committedWindowId = committedWindowId; return rsp; }