Java tutorial
package com.splunk.logging; /** * @copyright * * Copyright 2013-2015 Splunk, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"): you may * not use this file except in compliance with the License. You may obtain * a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations * under the License. */ import org.apache.http.HttpResponse; import org.apache.http.client.methods.HttpPost; import org.apache.http.concurrent.FutureCallback; import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.conn.ssl.SSLContexts; import org.apache.http.conn.ssl.TrustStrategy; import org.apache.http.entity.StringEntity; import org.apache.http.impl.nio.client.CloseableHttpAsyncClient; import org.apache.http.impl.nio.client.HttpAsyncClients; import org.apache.http.util.EntityUtils; import org.json.simple.JSONArray; import org.json.simple.JSONObject; import javax.net.ssl.SSLContext; import java.io.IOException; import java.io.Serializable; import java.security.cert.X509Certificate; import java.util.Dictionary; import java.util.Timer; import java.util.TimerTask; import java.util.List; import java.util.LinkedList; import java.util.Map; import java.util.Locale; /** * This is an internal helper class that sends logging events to Splunk http event collector. */ final class HttpEventCollectorSender extends TimerTask implements HttpEventCollectorMiddleware.IHttpSender { public static final String MetadataTimeTag = "time"; public static final String MetadataHostTag = "host"; public static final String MetadataIndexTag = "index"; public static final String MetadataSourceTag = "source"; public static final String MetadataSourceTypeTag = "sourcetype"; private static final String AuthorizationHeaderTag = "Authorization"; private static final String AuthorizationHeaderScheme = "Splunk %s"; private static final String HttpEventCollectorUriPath = "/services/collector/event/1.0"; private static final String HttpContentType = "application/json; profile=urn:splunk:event:1.0; charset=utf-8"; private static final String SendModeSequential = "sequential"; private static final String SendModeSParallel = "parallel"; /** * Sender operation mode. Parallel means that all HTTP requests are * asynchronous and may be indexed out of order. Sequential mode guarantees * sequential order of the indexed events. */ public enum SendMode { Sequential, Parallel }; /** * Recommended default values for events batching. */ public static final int DefaultBatchInterval = 10 * 1000; // 10 seconds public static final int DefaultBatchSize = 10 * 1024; // 10KB public static final int DefaultBatchCount = 10; // 10 events private String url; private String token; private long maxEventsBatchCount; private long maxEventsBatchSize; private Dictionary<String, String> metadata; private Timer timer; private List<HttpEventCollectorEventInfo> eventsBatch = new LinkedList<HttpEventCollectorEventInfo>(); private long eventsBatchSize = 0; // estimated total size of events batch private CloseableHttpAsyncClient httpClient; private boolean disableCertificateValidation = false; private SendMode sendMode = SendMode.Sequential; private HttpEventCollectorMiddleware middleware = new HttpEventCollectorMiddleware(); /** * Initialize HttpEventCollectorSender * @param Url http event collector input server * @param token application token * @param delay batching delay * @param maxEventsBatchCount max number of events in a batch * @param maxEventsBatchSize max size of batch * @param metadata events metadata */ public HttpEventCollectorSender(final String Url, final String token, long delay, long maxEventsBatchCount, long maxEventsBatchSize, String sendModeStr, Dictionary<String, String> metadata) { this.url = Url + HttpEventCollectorUriPath; this.token = token; // when size configuration setting is missing it's treated as "infinity", // i.e., any value is accepted. if (maxEventsBatchCount == 0 && maxEventsBatchSize > 0) { maxEventsBatchCount = Long.MAX_VALUE; } else if (maxEventsBatchSize == 0 && maxEventsBatchCount > 0) { maxEventsBatchSize = Long.MAX_VALUE; } this.maxEventsBatchCount = maxEventsBatchCount; this.maxEventsBatchSize = maxEventsBatchSize; this.metadata = metadata; if (sendModeStr != null) { if (sendModeStr.equals(SendModeSequential)) this.sendMode = SendMode.Sequential; else if (sendModeStr.equals(SendModeSParallel)) this.sendMode = SendMode.Parallel; else throw new IllegalArgumentException("Unknown send mode: " + sendModeStr); } if (delay > 0) { // start heartbeat timer timer = new Timer(); timer.scheduleAtFixedRate(this, delay, delay); } } public void addMiddleware(HttpEventCollectorMiddleware.HttpSenderMiddleware middleware) { this.middleware.add(middleware); } /** * Send a single logging event * @note in case of batching the event isn't sent immediately * @param severity event severity level (info, warning, etc.) * @param message event text */ public synchronized void send(final String severity, final String message, final String logger_name, final String thread_name, Map<String, String> properties, final String exception_message, Serializable marker) { // create event info container and add it to the batch HttpEventCollectorEventInfo eventInfo = new HttpEventCollectorEventInfo(severity, message, logger_name, thread_name, properties, exception_message, marker); queueEventInfo(eventInfo); } /** * Send a single logging event including the full information from the throwable. * * Contrast with the alternate send method that only sends the throwable's message. */ public synchronized void send(final String severity, final String message, final String logger_name, final String thread_name, Map<String, String> properties, final Throwable throwable, Serializable marker) { // create event info container and add it to the batch HttpEventCollectorEventInfo eventInfo = new HttpEventCollectorEventInfo(severity, message, logger_name, thread_name, properties, HttpEventCollectorThrowableInfo.buildFromThrowable(throwable), marker); queueEventInfo(eventInfo); } /** * Queues an event info for sending. If the number and size (measured roughly by message and severity length) * exceed the max batch size, then queued events will be flushed immediately. * * @param eventInfo the info that should be queued for sending. */ private void queueEventInfo(final HttpEventCollectorEventInfo eventInfo) { eventsBatch.add(eventInfo); eventsBatchSize += eventInfo.getSeverity().length() + eventInfo.getMessage().length(); if (eventsBatch.size() >= maxEventsBatchCount || eventsBatchSize > maxEventsBatchSize) { flush(); } } /** * Flush all pending events */ public synchronized void flush() { if (eventsBatch.size() > 0) { postEventsAsync(eventsBatch); } // Clear the batch. A new list should be created because events are // sending asynchronously and "previous" instance of eventsBatch object // is still in use. eventsBatch = new LinkedList<HttpEventCollectorEventInfo>(); eventsBatchSize = 0; } /** * Close events sender */ public void close() { if (timer != null) timer.cancel(); flush(); } /** * Timer heartbeat */ @Override // TimerTask public void run() { flush(); } /** * Disable https certificate validation of the splunk server. * This functionality is for development purpose only. */ public void disableCertificateValidation() { disableCertificateValidation = true; } @SuppressWarnings("unchecked") private static void putIfPresent(JSONObject collection, String tag, String value) { if (value != null && value.length() > 0) { collection.put(tag, value); } } @SuppressWarnings("unchecked") private String serializeEventInfo(HttpEventCollectorEventInfo eventInfo) { // create event json content // // cf: http://dev.splunk.com/view/event-collector/SP-CAAAE6P // JSONObject event = new JSONObject(); // event timestamp and metadata putIfPresent(event, MetadataTimeTag, String.format(Locale.US, "%.3f", eventInfo.getTime())); putIfPresent(event, MetadataHostTag, metadata.get(MetadataHostTag)); putIfPresent(event, MetadataIndexTag, metadata.get(MetadataIndexTag)); putIfPresent(event, MetadataSourceTag, metadata.get(MetadataSourceTag)); putIfPresent(event, MetadataSourceTypeTag, metadata.get(MetadataSourceTypeTag)); // event body JSONObject body = new JSONObject(); putIfPresent(body, "severity", eventInfo.getSeverity()); putIfPresent(body, "message", eventInfo.getMessage()); putIfPresent(body, "logger", eventInfo.getLoggerName()); putIfPresent(body, "thread", eventInfo.getThreadName()); // add an exception record if and only if there is one // in practice, the message also has the exception information attached if (eventInfo.getExceptionMessage() != null) { putIfPresent(body, "exception", eventInfo.getExceptionMessage()); } final JSONObject throwableJson = writeThrowableInfoToJson(eventInfo.getThrowableInfo()); if (throwableJson != null) { body.put("throwable", throwableJson); } // add properties if and only if there are any final Map<String, String> props = eventInfo.getProperties(); if (props != null && !props.isEmpty()) { body.put("properties", props); } // add marker if and only if there is one final Serializable marker = eventInfo.getMarker(); if (marker != null) { putIfPresent(body, "marker", marker.toString()); } // join event and body event.put("event", body); return event.toString(); } @SuppressWarnings("unchecked") // JSONObject does not understand that Map should be parameterised. private JSONObject writeThrowableInfoToJson(HttpEventCollectorThrowableInfo throwableInfo) { if (throwableInfo == null) { return null; } final JSONObject json = new JSONObject(); putIfPresent(json, "throwable_message", throwableInfo.getMessage()); putIfPresent(json, "throwable_class", throwableInfo.getClassName()); final JSONArray stackTrace = new JSONArray(); json.put("stack_trace", stackTrace); if (throwableInfo.getStackTraceElements() != null) { for (final String strackTraceElement : throwableInfo.getStackTraceElements()) { stackTrace.add(strackTraceElement); } } final JSONObject cause = writeThrowableInfoToJson(throwableInfo.getCause()); if (cause != null) { json.put("cause", cause); } return json; } private void startHttpClient() { if (httpClient != null) { // http client is already started return; } // limit max number of async requests in sequential mode, 0 means "use // default limit" int maxConnTotal = sendMode == SendMode.Sequential ? 1 : 0; if (!disableCertificateValidation) { // create an http client that validates certificates httpClient = HttpAsyncClients.custom().setMaxConnTotal(maxConnTotal).build(); } else { // create strategy that accepts all certificates TrustStrategy acceptingTrustStrategy = new TrustStrategy() { public boolean isTrusted(X509Certificate[] certificate, String type) { return true; } }; SSLContext sslContext = null; try { sslContext = SSLContexts.custom().loadTrustMaterial(null, acceptingTrustStrategy).build(); httpClient = HttpAsyncClients.custom().setMaxConnTotal(maxConnTotal) .setHostnameVerifier(SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER) .setSSLContext(sslContext).build(); } catch (Exception e) { } } httpClient.start(); } // Currently we never close http client. This method is added for symmetry // with startHttpClient. @SuppressWarnings("unused") // Private method not used but is wanted. private void stopHttpClient() throws SecurityException { if (httpClient != null) { try { httpClient.close(); } catch (IOException e) { } httpClient = null; } } private void postEventsAsync(final List<HttpEventCollectorEventInfo> events) { this.middleware.postEvents(events, this, new HttpEventCollectorMiddleware.IHttpSenderCallback() { @Override public void completed(int statusCode, String reply) { if (statusCode != 200) { HttpEventCollectorErrorHandler.error(events, new HttpEventCollectorErrorHandler.ServerErrorException(reply)); } } @Override public void failed(Exception ex) { HttpEventCollectorErrorHandler.error(eventsBatch, new HttpEventCollectorErrorHandler.ServerErrorException(ex.getMessage())); } }); } public void postEvents(final List<HttpEventCollectorEventInfo> events, final HttpEventCollectorMiddleware.IHttpSenderCallback callback) { startHttpClient(); // make sure http client is started final String encoding = "utf-8"; // convert events list into a string StringBuilder eventsBatchString = new StringBuilder(); for (HttpEventCollectorEventInfo eventInfo : events) eventsBatchString.append(serializeEventInfo(eventInfo)); // create http request final HttpPost httpPost = new HttpPost(url); httpPost.setHeader(AuthorizationHeaderTag, String.format(AuthorizationHeaderScheme, token)); StringEntity entity = new StringEntity(eventsBatchString.toString(), encoding); entity.setContentType(HttpContentType); httpPost.setEntity(entity); httpClient.execute(httpPost, new FutureCallback<HttpResponse>() { @Override public void completed(HttpResponse response) { String reply = ""; int httpStatusCode = response.getStatusLine().getStatusCode(); // read reply only in case of a server error if (httpStatusCode != 200) { try { reply = EntityUtils.toString(response.getEntity(), encoding); } catch (IOException e) { reply = e.getMessage(); } } callback.completed(httpStatusCode, reply); } @Override public void failed(Exception ex) { callback.failed(ex); } @Override public void cancelled() { } }); } }