Java tutorial
/** * Licensed to DigitalPebble Ltd under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * DigitalPebble licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.digitalpebble.storm.crawler.librato.metrics; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import backtype.storm.metric.api.IMetricsConsumer; import backtype.storm.task.IErrorReporter; import backtype.storm.task.TopologyContext; import com.fasterxml.jackson.databind.ObjectMapper; import com.librato.metrics.HttpPoster; import com.librato.metrics.HttpPoster.Response; import com.librato.metrics.LibratoBatch; import com.librato.metrics.NingHttpPoster; import com.librato.metrics.Sanitizer; import com.librato.metrics.Versions; /** * Sends the metrics to Librato */ public class LibratoMetricsConsumer implements IMetricsConsumer { private static final int DEFAULT_BATCH_SIZE = 500; private static final Logger LOG = LoggerFactory.getLogger(LibratoMetricsConsumer.class); private static final String LIB_VERSION = Versions .getVersion("META-INF/maven/com.librato.metrics/librato-java/pom.properties", LibratoBatch.class); private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private final Sanitizer sanitizer = new Sanitizer() { @Override public String apply(String name) { return Sanitizer.LAST_PASS.apply(name); } }; private int postBatchSize = DEFAULT_BATCH_SIZE; private long timeout = 30; private final TimeUnit timeoutUnit = TimeUnit.SECONDS; private String userAgent = null; private HttpPoster httpPoster; private Set<String> metricsToKeep = new HashSet<String>(); @Override public void prepare(Map stormConf, Object registrationArgument, TopologyContext context, IErrorReporter errorReporter) { // TODO configure timeouts // this.timeout = timeout; // this.timeoutUnit = timeoutUnit; // this.postBatchSize = postBatchSize; String agentIdentifier = (String) stormConf.get("librato.agent"); if (agentIdentifier == null) agentIdentifier = "storm"; String token = (String) stormConf.get("librato.token"); String username = (String) stormConf.get("librato.username"); String apiUrl = (String) stormConf.get("librato.api.url"); if (apiUrl == null) apiUrl = "https://metrics-api.librato.com/v1/metrics"; // check that the values are not null if (StringUtils.isBlank(token)) throw new RuntimeException("librato.token not set"); if (StringUtils.isBlank(username)) throw new RuntimeException("librato.username not set"); this.userAgent = String.format("%s librato-java/%s", agentIdentifier, LIB_VERSION); this.httpPoster = NingHttpPoster.newPoster(username, token, apiUrl); // get the list of metrics names to keep if any String metrics2keep = (String) stormConf.get("librato.metrics.to.keep"); if (metrics2keep != null) { String[] mets = metrics2keep.split(","); for (String m : mets) metricsToKeep.add(m.trim().toLowerCase()); } } // post(String source, long epoch) @Override public void handleDataPoints(TaskInfo taskInfo, Collection<DataPoint> dataPoints) { final Map<String, Object> payloadMap = new HashMap<String, Object>(); payloadMap.put("source", taskInfo.srcComponentId + "_" + taskInfo.srcWorkerHost + "_" + taskInfo.srcTaskId); payloadMap.put("measure_time", taskInfo.timestamp); final List<Map<String, Object>> gaugeData = new ArrayList<Map<String, Object>>(); final List<Map<String, Object>> counterData = new ArrayList<Map<String, Object>>(); int counter = 0; final Iterator<DataPoint> datapointsIterator = dataPoints.iterator(); while (datapointsIterator.hasNext()) { final DataPoint dataPoint = datapointsIterator.next(); // ignore datapoint with a value which is not a map if (!(dataPoint.value instanceof Map)) continue; // a counter or a gauge // convention if its name contains '_counter' // then treat it as a counter boolean isCounter = false; if (dataPoint.name.contains("_counter")) { isCounter = true; dataPoint.name = dataPoint.name.replaceFirst("_counter", ""); } if (!metricsToKeep.isEmpty()) { if (!metricsToKeep.contains(dataPoint.name.toLowerCase())) { continue; } } try { Map<String, Number> metric = (Map<String, Number>) dataPoint.value; for (Map.Entry<String, Number> entry : metric.entrySet()) { String metricId = entry.getKey(); Number val = entry.getValue(); final Map<String, Object> data = new HashMap<String, Object>(); data.put("name", sanitizer.apply(dataPoint.name + "_" + metricId)); data.put("value", val); if (isCounter) counterData.add(data); else // use as gauge gaugeData.add(data); counter++; if (counter % postBatchSize == 0 || (!datapointsIterator.hasNext() && (!counterData.isEmpty() || !gaugeData.isEmpty()))) { final String countersKey = "counters"; final String gaugesKey = "gauges"; payloadMap.put(countersKey, counterData); payloadMap.put(gaugesKey, gaugeData); postPortion(payloadMap); payloadMap.remove(gaugesKey); payloadMap.remove(countersKey); gaugeData.clear(); counterData.clear(); } } } catch (RuntimeException e) { LOG.error(e.getMessage()); } } LOG.debug("Posted {} measurements", counter); } @Override public void cleanup() { } private void postPortion(Map<String, Object> chunk) { try { final String payload = OBJECT_MAPPER.writeValueAsString(chunk); final Future<Response> future = httpPoster.post(userAgent, payload); final Response response = future.get(timeout, timeoutUnit); final int statusCode = response.getStatusCode(); if (statusCode < 200 || statusCode >= 300) { LOG.error("Received an error from Librato API. Code : {}, Message: {}", statusCode, response.getBody()); } } catch (Exception e) { LOG.error("Unable to post to Librato API", e); } } }