Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.spark.network.yarn; import com.codahale.metrics.*; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; import org.apache.hadoop.metrics2.MetricsCollector; import org.apache.hadoop.metrics2.MetricsInfo; import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.metrics2.MetricsSource; import java.util.Map; import java.util.concurrent.TimeUnit; /** * Modeled off of YARN's NodeManagerMetrics. */ public class YarnShuffleServiceMetrics implements MetricsSource { // Converting from the dropwizard-metrics default of nanoseconds into milliseconds to match how // MetricsServlet serializes times (to milliseconds) configured via the MetricsModule passed into // its Jackson ObjectMapper. Without this rate factor applied, the Timer metrics from // ExternalShuffleBlockManager#ShuffleMetrics with "Millis" suffixes are misleading, as they // would otherwise contain values in nanoseconds units private static final double rateFactor = (double) TimeUnit.MILLISECONDS.toNanos(1L); private final MetricSet metricSet; public YarnShuffleServiceMetrics(MetricSet metricSet) { this.metricSet = metricSet; } /** * Get metrics from the source * * @param collector to contain the resulting metrics snapshot * @param all if true, return all metrics even if unchanged. */ @Override public void getMetrics(MetricsCollector collector, boolean all) { MetricsRecordBuilder metricsRecordBuilder = collector.addRecord("shuffleService"); for (Map.Entry<String, Metric> entry : metricSet.getMetrics().entrySet()) { collectMetric(metricsRecordBuilder, entry.getKey(), entry.getValue()); } } private static void addSnapshotToMetricRecordBuilder(Snapshot snapshot, MetricsRecordBuilder builder, String name, String metricType) { ImmutableMap<String, Double> doubleValues = ImmutableMap.<String, Double>builder() .put("median", snapshot.getMedian()).put("mean", snapshot.getMean()) .put("75th", snapshot.get75thPercentile()).put("95th", snapshot.get95thPercentile()) .put("98th", snapshot.get98thPercentile()).put("99th", snapshot.get99thPercentile()) .put("999th", snapshot.get999thPercentile()).build(); ImmutableMap<String, Long> longValues = ImmutableMap.<String, Long>builder().put("min", snapshot.getMin()) .put("max", snapshot.getMax()).build(); for (Map.Entry<String, Double> entry : doubleValues.entrySet()) { builder.addGauge(new ShuffleServiceMetricsInfo(name + "_" + entry.getKey(), entry.getKey() + " of " + metricType + " " + name), entry.getValue() / rateFactor); } for (Map.Entry<String, Long> entry : longValues.entrySet()) { builder.addGauge(new ShuffleServiceMetricsInfo(name + "_" + entry.getKey(), entry.getKey() + " of " + metricType + " " + name), entry.getValue() / rateFactor); } } @VisibleForTesting public static void collectMetric(MetricsRecordBuilder metricsRecordBuilder, String name, Metric metric) { // The metric types used in ExternalShuffleBlockHandler.ShuffleMetrics if (metric instanceof Timer) { Timer t = (Timer) metric; Snapshot snapshot = t.getSnapshot(); metricsRecordBuilder .addCounter(new ShuffleServiceMetricsInfo(name + "_count", "Count of timer " + name), t.getCount()) .addGauge(new ShuffleServiceMetricsInfo(name + "_rate15", "15 minute rate of timer " + name), t.getFifteenMinuteRate()) .addGauge(new ShuffleServiceMetricsInfo(name + "_rate5", "5 minute rate of timer " + name), t.getFiveMinuteRate()) .addGauge(new ShuffleServiceMetricsInfo(name + "_rate1", "1 minute rate of timer " + name), t.getOneMinuteRate()) .addGauge(new ShuffleServiceMetricsInfo(name + "_rateMean", "Mean rate of timer " + name), t.getMeanRate()); addSnapshotToMetricRecordBuilder(snapshot, metricsRecordBuilder, name, "timer"); } else if (metric instanceof Meter) { Meter m = (Meter) metric; metricsRecordBuilder .addCounter(new ShuffleServiceMetricsInfo(name + "_count", "Count of meter " + name), m.getCount()) .addGauge(new ShuffleServiceMetricsInfo(name + "_rate15", "15 minute rate of meter " + name), m.getFifteenMinuteRate()) .addGauge(new ShuffleServiceMetricsInfo(name + "_rate5", "5 minute rate of meter " + name), m.getFiveMinuteRate()) .addGauge(new ShuffleServiceMetricsInfo(name + "_rate1", "1 minute rate of meter " + name), m.getOneMinuteRate()) .addGauge(new ShuffleServiceMetricsInfo(name + "_rateMean", "Mean rate of meter " + name), m.getMeanRate()); } else if (metric instanceof Gauge) { Gauge m = (Gauge) metric; Object gaugeValue = m.getValue(); if (gaugeValue instanceof Integer) { Integer intValue = (Integer) gaugeValue; metricsRecordBuilder.addGauge( new ShuffleServiceMetricsInfo(name, "Integer value of " + "gauge " + name), intValue.intValue()); } } } private static class ShuffleServiceMetricsInfo implements MetricsInfo { private final String name; private final String description; ShuffleServiceMetricsInfo(String name, String description) { this.name = name; this.description = description; } @Override public String name() { return name; } @Override public String description() { return description; } } }