com.google.pubsub.clients.common.MetricsHandler.java Source code

Java tutorial

Introduction

Here is the source code for com.google.pubsub.clients.common.MetricsHandler.java

Source

// Copyright 2016 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
////////////////////////////////////////////////////////////////////////////////
package com.google.pubsub.clients.common;

import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.jackson2.JacksonFactory;
import com.google.api.services.monitoring.v3.Monitoring;
import com.google.api.services.monitoring.v3.model.BucketOptions;
import com.google.api.services.monitoring.v3.model.CreateTimeSeriesRequest;
import com.google.api.services.monitoring.v3.model.Distribution;
import com.google.api.services.monitoring.v3.model.Explicit;
import com.google.api.services.monitoring.v3.model.LabelDescriptor;
import com.google.api.services.monitoring.v3.model.Metric;
import com.google.api.services.monitoring.v3.model.MetricDescriptor;
import com.google.api.services.monitoring.v3.model.MonitoredResource;
import com.google.api.services.monitoring.v3.model.Point;
import com.google.api.services.monitoring.v3.model.TimeInterval;
import com.google.api.services.monitoring.v3.model.TimeSeries;
import com.google.api.services.monitoring.v3.model.TypedValue;
import com.google.common.collect.ImmutableMap;
import com.google.pubsub.flic.common.LatencyDistribution;
import java.io.IOException;
import java.security.GeneralSecurityException;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.TimeZone;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.http.HttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.protocol.RequestAcceptEncoding;
import org.apache.http.client.protocol.ResponseContentEncoding;
import org.apache.http.conn.scheme.PlainSocketFactory;
import org.apache.http.conn.scheme.Scheme;
import org.apache.http.conn.scheme.SchemeRegistry;
import org.apache.http.conn.ssl.SSLSocketFactory;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.params.HttpConnectionParams;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * A class that is used to record metrics related to the execution of the load tests, such metrics
 * are recorded using Google's Cloud Monitoring API.
 */
public class MetricsHandler {
    private static final Logger log = LoggerFactory.getLogger(MetricsHandler.class);
    private final String project;
    private final SimpleDateFormat dateFormatter;
    private final LatencyDistribution distribution;
    private final ScheduledExecutorService executor;
    private final String clientType;
    private final MonitoredResource monitoredResource;
    private Monitoring monitoring;
    private MetricName metricName;

    MetricsHandler(String project, String clientType, MetricName metricName) {
        this.project = project;
        this.clientType = clientType;
        this.metricName = metricName;
        dateFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
        dateFormatter.setTimeZone(TimeZone.getTimeZone("GMT"));
        distribution = new LatencyDistribution();
        monitoredResource = new MonitoredResource().setType("gce_instance");
        executor = Executors.newSingleThreadScheduledExecutor();
        executor.execute(this::initialize);
    }

    private void initialize() {
        synchronized (this) {
            try {
                HttpTransport transport = GoogleNetHttpTransport.newTrustedTransport();
                JsonFactory jsonFactory = new JacksonFactory();
                GoogleCredential credential = GoogleCredential.getApplicationDefault(transport, jsonFactory);
                if (credential.createScopedRequired()) {
                    credential = credential.createScoped(
                            Collections.singletonList("https://www.googleapis.com/auth/cloud-platform"));
                }
                monitoring = new Monitoring.Builder(transport, jsonFactory, credential)
                        .setApplicationName("Cloud Pub/Sub Loadtest Framework").build();
                String zoneId;
                String instanceId;
                try {
                    DefaultHttpClient httpClient = new DefaultHttpClient();
                    httpClient.addRequestInterceptor(new RequestAcceptEncoding());
                    httpClient.addResponseInterceptor(new ResponseContentEncoding());

                    HttpConnectionParams.setConnectionTimeout(httpClient.getParams(), 30000);
                    HttpConnectionParams.setSoTimeout(httpClient.getParams(), 30000);
                    HttpConnectionParams.setSoKeepalive(httpClient.getParams(), true);
                    HttpConnectionParams.setStaleCheckingEnabled(httpClient.getParams(), false);
                    HttpConnectionParams.setTcpNoDelay(httpClient.getParams(), true);

                    SchemeRegistry schemeRegistry = httpClient.getConnectionManager().getSchemeRegistry();
                    schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));
                    schemeRegistry.register(new Scheme("https", 443, SSLSocketFactory.getSocketFactory()));
                    httpClient.setKeepAliveStrategy((response, ctx) -> 30);
                    HttpGet zoneIdRequest = new HttpGet(
                            "http://metadata.google.internal/computeMetadata/v1/instance/zone");
                    zoneIdRequest.setHeader("Metadata-Flavor", "Google");
                    HttpResponse zoneIdResponse = httpClient.execute(zoneIdRequest);
                    String tempZoneId = EntityUtils.toString(zoneIdResponse.getEntity());
                    if (tempZoneId.lastIndexOf("/") >= 0) {
                        zoneId = tempZoneId.substring(tempZoneId.lastIndexOf("/") + 1);
                    } else {
                        zoneId = tempZoneId;
                    }
                    HttpGet instanceIdRequest = new HttpGet(
                            "http://metadata.google.internal/computeMetadata/v1/instance/id");
                    instanceIdRequest.setHeader("Metadata-Flavor", "Google");
                    HttpResponse instanceIdResponse = httpClient.execute(instanceIdRequest);
                    instanceId = EntityUtils.toString(instanceIdResponse.getEntity());
                } catch (IOException e) {
                    log.info("Unable to connect to metadata server, assuming not on GCE, setting "
                            + "defaults for instance and zone.");
                    instanceId = "local";
                    zoneId = "us-east1-b"; // Must use a valid cloud zone even if running local.
                }

                monitoredResource.setLabels(
                        ImmutableMap.of("project_id", project, "instance_id", instanceId, "zone", zoneId));
                createMetrics();
            } catch (IOException e) {
                log.error("Unable to initialize MetricsHandler, trying again.", e);
                executor.execute(this::initialize);
            } catch (GeneralSecurityException e) {
                log.error("Unable to initialize MetricsHandler permanently, credentials error.", e);
            }
        }
    }

    private void createMetrics() {
        try {
            MetricDescriptor metricDescriptor = new MetricDescriptor()
                    .setType("custom.googleapis.com/cloud-pubsub/loadclient/" + metricName);
            metricDescriptor.setDisplayName(metricName.toPrettyString()).setDescription(metricName.toPrettyString())
                    .setName(metricDescriptor.getType())
                    .setLabels(Collections.singletonList(new LabelDescriptor().setKey("client_type")
                            .setDescription("The type of client reporting latency.").setValueType("STRING")))
                    .setMetricKind("GAUGE").setValueType("DISTRIBUTION").setUnit("ms");
            monitoring.projects().metricDescriptors().create("projects/" + project, metricDescriptor).execute();
        } catch (Exception e) {
            log.info("Metrics already exist.");
        }
    }

    public synchronized void recordLatency(long latencyMs) {
        distribution.recordLatency(latencyMs);
    }

    public synchronized void recordLatencyBatch(long latencyMs, int batchSize) {
        distribution.recordLatencyBatch(latencyMs, batchSize);
    }

    private void reportMetrics(LatencyDistribution distribution) {
        CreateTimeSeriesRequest request;
        synchronized (this) {
            String now = dateFormatter.format(new Date());
            request = new CreateTimeSeriesRequest().setTimeSeries(Collections.singletonList(new TimeSeries()
                    .setMetric(new Metric().setType("custom.googleapis.com/cloud-pubsub/loadclient/" + metricName)
                            .setLabels(ImmutableMap.of("client_type", clientType)))
                    .setMetricKind("GAUGE").setValueType("DISTRIBUTION")
                    .setPoints(Collections.singletonList(new Point()
                            .setValue(new TypedValue().setDistributionValue(new Distribution()
                                    .setBucketCounts(distribution.getBucketValuesAsList())
                                    .setCount(distribution.getCount()).setMean(distribution.getMean())
                                    .setSumOfSquaredDeviation(distribution.getSumOfSquareDeviations())
                                    .setBucketOptions(new BucketOptions()
                                            .setExplicitBuckets(new Explicit().setBounds(Arrays.asList(
                                                    ArrayUtils.toObject(LatencyDistribution.LATENCY_BUCKETS)))))))
                            .setInterval(new TimeInterval().setStartTime(now).setEndTime(now))))
                    .setResource(monitoredResource)));
        }
        try {
            monitoring.projects().timeSeries().create("projects/" + project, request).execute();
        } catch (IOException e) {
            log.error("Error reporting latency.", e);
        }
    }

    // Flushes current bucket to Stackdriver and returns the bucket values.
    synchronized List<Long> flushBucketValues() {
        LatencyDistribution latencyDistribution = distribution.copy();
        executor.submit(() -> reportMetrics(latencyDistribution));
        List<Long> bucketValues = distribution.getBucketValuesAsList();
        distribution.reset();
        return bucketValues;
    }

    /**
     * The possible metrics to report to Stackdriver.
     */
    public enum MetricName {
        END_TO_END_LATENCY, PUBLISH_ACK_LATENCY;

        @Override
        public String toString() {
            return name().toLowerCase();
        }

        public String toPrettyString() {
            return name().toLowerCase().replace('-', ' ');
        }
    }
}