Java tutorial
/* * Copyright 2015 Cyanogen Inc. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.cyngn.vertx.bosun; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import com.google.common.net.MediaType; import io.netty.handler.codec.http.HttpResponseStatus; import io.vertx.core.AbstractVerticle; import io.vertx.core.Future; import io.vertx.core.Handler; import io.vertx.core.buffer.Buffer; import io.vertx.core.eventbus.EventBus; import io.vertx.core.eventbus.Message; import io.vertx.core.http.HttpClient; import io.vertx.core.http.HttpClientOptions; import io.vertx.core.http.HttpHeaders; import io.vertx.core.json.JsonArray; import io.vertx.core.json.JsonObject; import io.vertx.core.logging.Logger; import io.vertx.core.logging.LoggerFactory; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; /** * Handles consuming metrics over the message bus and sending them on to bosun. * * @author truelove@cyngn.com (Jeremy Truelove) 07/22/15 */ public class BosunReporter extends AbstractVerticle implements Handler<Message<JsonObject>> { public final static String DEFAULT_ADDRESS = "vertx.bosun-reporter"; private Logger logger = LoggerFactory.getLogger(BosunReporter.class); public static final String PUT_COMMAND = "put"; public static final String INDEX_COMMAND = "index"; public static final int DEFAULT_MSG_ERROR_CODE = -1; public static final String ACTION_FIELD = "action"; public static final String PUT_API = "/api/put"; public static final String INDEX_API = "/api/index"; public static final int OPENTSDB_DEFAULT_MAX_TAGS = 8; private final int DEFAULT_TIMEOUT_MS = 3000; private final int DEFAULT_UNIQUE_METRICS_INDEXED = 1000000; private final int DEFAULT_INDEX_EXPIRY_MINUTES = 10; private static int FIVE_MINUTES_MILLI = 1000 * 60 * 5; public final static String RESULT_FIELD = "result"; private JsonArray hosts; private int maxTags; private int maxIndexCacheSize; private int indexExpiryInMinutes; private int timeout; private Map<String, Consumer<Message<JsonObject>>> handlers; private List<HttpClient> connections; private String address; private EventBus eventBus; private AtomicInteger currentConnectionIndex = new AtomicInteger(0); private LoadingCache<String, Boolean> distinctMetrics; private long reportingTimerId = -1; private AtomicInteger metricsIndexed; private AtomicInteger metricsPut; private AtomicInteger metricsErrors; @Override public void start(final Future<Void> startedResult) { // setup the default config values JsonObject config = context.config(); hosts = config.getJsonArray("hosts", new JsonArray("[{ \"host\" : \"localhost\", \"port\" : 8070}]")); address = config.getString("address", DEFAULT_ADDRESS); maxTags = config.getInteger("max_tags", OPENTSDB_DEFAULT_MAX_TAGS); maxIndexCacheSize = config.getInteger("max_index_cache_size", DEFAULT_UNIQUE_METRICS_INDEXED); indexExpiryInMinutes = config.getInteger("index_expiry_minutes", DEFAULT_INDEX_EXPIRY_MINUTES); timeout = config.getInteger("default_timeout_ms", DEFAULT_TIMEOUT_MS); metricsIndexed = new AtomicInteger(0); metricsPut = new AtomicInteger(0); metricsErrors = new AtomicInteger(0); eventBus = vertx.eventBus(); // create the list of workers connections = new ArrayList<>(hosts.size()); initializeConnections(startedResult); createMessageHandlers(); outputConfig(); // initialize the in memory index cache distinctMetrics = CacheBuilder.newBuilder().maximumSize(maxIndexCacheSize) .expireAfterWrite(DEFAULT_INDEX_EXPIRY_MINUTES, TimeUnit.MINUTES) .build(new CacheLoader<String, Boolean>() { public Boolean load(String key) throws Exception { return true; } }); // start listening for incoming messages eventBus.consumer(address, this); initStatsReporting(); } private void initStatsReporting() { reportingTimerId = vertx.setPeriodic(FIVE_MINUTES_MILLI, (timerId) -> { logger.info(String.format( "Currently indexing %d metrics, metrics indexed: %d put: %d errors: %d this period", distinctMetrics.size(), metricsIndexed.getAndSet(0), metricsPut.getAndSet(0), metricsErrors.getAndSet(0))); }); } /** * Dump the config that we are using out */ private void outputConfig() { StringBuilder builder = new StringBuilder(); builder.append("Config[address=").append(address).append(", maxTags=").append(maxTags) .append(", max_index_cache_size=").append(maxIndexCacheSize).append(", index_expiry_in_minutes=") .append(indexExpiryInMinutes).append(", default_timeout_ms=").append(timeout).append(", hosts='") .append(hosts.encode()).append("']"); logger.info(builder.toString()); } /** * Setup our client connections * * @param startedResult the startup callback for loading the module */ private void initializeConnections(Future<Void> startedResult) { try { for (int i = 0; i < hosts.size(); i++) { JsonObject jsonHost = hosts.getJsonObject(i); connections.add( vertx.createHttpClient(new HttpClientOptions().setDefaultHost(jsonHost.getString("host")) .setDefaultPort(jsonHost.getInteger("port")).setKeepAlive(true).setTcpNoDelay(true) .setConnectTimeout(timeout).setTryUseCompression(true))); } } catch (Exception ex) { startedResult.fail(ex.getLocalizedMessage()); return; } // all connections added startedResult.complete(); } @Override public void stop() { logger.info("Shutting down vertx-bosun..."); if (reportingTimerId != -1) { vertx.cancelTimer(reportingTimerId); reportingTimerId = -1; } } /** * Handles round robin'ing through the client connections * * @return the next client connection to use */ private HttpClient getNextHost() { int nextIndex = currentConnectionIndex.incrementAndGet(); if (nextIndex >= hosts.size()) { nextIndex = 0; currentConnectionIndex.set(nextIndex); } return connections.get(nextIndex); } /** * Setup message listeners for specific actions. */ private void createMessageHandlers() { handlers = new HashMap<>(); handlers.put(PUT_COMMAND, this::doPut); handlers.put(INDEX_COMMAND, this::doIndex); } /** * Handles posting to the put endpoint * * @param message the message to send */ private void doPut(Message<JsonObject> message) { OpenTsDbMetric metric = getMetricFromMessage(message); if (metric == null) { return; } metricsPut.incrementAndGet(); sendData(PUT_API, metric.asJson().encode(), message); } /** * Handles posting to the index endpoint * * @param message the message to send */ private void doIndex(Message<JsonObject> message) { OpenTsDbMetric metric = getMetricFromMessage(message); if (metric == null) { return; } // ignore it we've seen it lately String key = metric.getDistinctKey(); if (distinctMetrics.getIfPresent(key) != null) { message.reply(new JsonObject().put(RESULT_FIELD, BosunResponse.EXISTS_MSG)); return; } // cache it distinctMetrics.put(key, true); metricsIndexed.incrementAndGet(); sendData(INDEX_API, metric.asJson().encode(), message); } /** * Convert the event bus message to a metric object we can work with. * * @param message the event bus message * @return a metric object that can be sent to Bosun */ private OpenTsDbMetric getMetricFromMessage(Message<JsonObject> message) { OpenTsDbMetric metric = null; try { metric = new OpenTsDbMetric(message.body()); } catch (IllegalArgumentException ex) { sendError(message, ex.getMessage()); return null; } if (!metric.validate(maxTags)) { sendError(message, String.format("Cannot send more than %d tags, %d were attempted", maxTags, metric.tags.size())); metric = null; } return metric; } /** * Send data to the bosun instance * * @param api the api on bosun to send to * @param data the json data to send * @param message the event bus message the request originated from */ private void sendData(String api, String data, Message message) { HttpClient client = getNextHost(); Buffer buffer = Buffer.buffer(data.getBytes()); client.post(api).exceptionHandler(error -> { sendError(message, "Got ex contacting bosun, " + error.getLocalizedMessage()); }).handler(response -> { int statusCode = response.statusCode(); // is it 2XX if (statusCode >= HttpResponseStatus.OK.code() && statusCode < HttpResponseStatus.MULTIPLE_CHOICES.code()) { message.reply(new JsonObject().put(RESULT_FIELD, BosunResponse.OK_MSG)); } else { response.bodyHandler(responseData -> { sendError(message, "got non 200 response from bosun, error: " + responseData, statusCode); }); } }).setTimeout(timeout).putHeader(HttpHeaders.CONTENT_LENGTH, buffer.length() + "") .putHeader(HttpHeaders.CONTENT_TYPE, MediaType.JSON_UTF_8.toString()).write(buffer).end(); } /** * Handles processing metric requests off the event bus * * @param message the metrics message */ @Override public void handle(Message<JsonObject> message) { String action = message.body().getString(ACTION_FIELD); if (action == null) { sendError(message, "You must specify an action"); } Consumer<Message<JsonObject>> handler = handlers.get(action); if (handler != null) { handler.accept(message); } else { sendError(message, "Invalid action: " + action + " specified."); } } /** * Send an error message back to the message sender * * @param message the message to reply to * @param error the error text * @param errorCode an error code defaults to DEFAULT_MSG_ERROR_CODE, in the case of HTTP failures you'll get a * status back. */ private void sendError(Message message, String error, int errorCode) { metricsErrors.incrementAndGet(); message.fail(errorCode, error); } private void sendError(Message message, String error) { sendError(message, error, DEFAULT_MSG_ERROR_CODE); } }