Java tutorial
// This file is part of OpenTSDB. // Copyright (C) 2010-2012 The OpenTSDB Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 2.1 of the License, or (at your // option) any later version. This program is distributed in the hope that it // will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty // of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser // General Public License for more details. You should have received a copy // of the GNU Lesser General Public License along with this program. If not, // see <http://www.gnu.org/licenses/>. package net.opentsdb.tsd; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.PrintWriter; import java.net.URL; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.NoSuchElementException; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicInteger; import static java.util.concurrent.TimeUnit.MILLISECONDS; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.databind.JsonMappingException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import net.opentsdb.core.Aggregator; import net.opentsdb.core.Aggregators; import net.opentsdb.core.Const; import net.opentsdb.core.DataPoint; import net.opentsdb.core.DataPoints; import net.opentsdb.core.Query; import net.opentsdb.core.RateOptions; import net.opentsdb.core.TSDB; import net.opentsdb.core.Tags; import net.opentsdb.graph.Plot; import net.opentsdb.stats.Histogram; import net.opentsdb.stats.StatsCollector; import net.opentsdb.uid.NoSuchUniqueName; import net.opentsdb.utils.DateTime; import net.opentsdb.utils.JSON; /** * Stateless handler of HTTP graph requests (the {@code /q} endpoint). */ final class GraphHandler implements HttpRpc { private static final Logger LOG = LoggerFactory.getLogger(GraphHandler.class); private static final boolean IS_WINDOWS = System.getProperty("os.name", "").contains("Windows"); /** Number of times we had to do all the work up to running Gnuplot. */ private static final AtomicInteger graphs_generated = new AtomicInteger(); /** Number of times a graph request was served from disk, no work needed. */ private static final AtomicInteger graphs_diskcache_hit = new AtomicInteger(); /** Keep track of the latency of graphing requests. */ private static final Histogram graphlatency = new Histogram(16000, (short) 2, 100); /** Keep track of the latency (in ms) introduced by running Gnuplot. */ private static final Histogram gnuplotlatency = new Histogram(16000, (short) 2, 100); /** Executor to run Gnuplot in separate bounded thread pool. */ private final ThreadPoolExecutor gnuplot; /** * Constructor. */ public GraphHandler() { // Gnuplot is mostly CPU bound and does only a little bit of IO at the // beginning to read the input data and at the end to write its output. // We want to avoid running too many Gnuplot instances concurrently as // it can steal a significant number of CPU cycles from us. Instead, we // allow only one per core, and we nice it (the nicing is done in the // shell script we use to start Gnuplot). Similarly, the queue we use // is sized so as to have a fixed backlog per core. final int ncores = Runtime.getRuntime().availableProcessors(); gnuplot = new ThreadPoolExecutor(ncores, ncores, // Thread pool of a fixed size. /* 5m = */ 300000, MILLISECONDS, // How long to keep idle threads. new ArrayBlockingQueue<Runnable>(20 * ncores), // XXX Don't hardcode? thread_factory); // ArrayBlockingQueue does not scale as much as LinkedBlockingQueue in terms // of throughput but we don't need high throughput here. We use ABQ instead // of LBQ because it creates far fewer references. } public void execute(final TSDB tsdb, final HttpQuery query) { if (!query.hasQueryStringParam("json") && !query.hasQueryStringParam("png") && !query.hasQueryStringParam("ascii")) { String uri = query.request().getUri(); if (uri.length() < 4) { // Shouldn't happen... uri = "/"; // But just in case, redirect. } else { uri = "/#" + uri.substring(3); // Remove "/q?" } query.redirect(uri); return; } try { doGraph(tsdb, query); } catch (IOException e) { query.internalError(e); } catch (IllegalArgumentException e) { query.badRequest(e.getMessage()); } } private void doGraph(final TSDB tsdb, final HttpQuery query) throws IOException { final String basepath = getGnuplotBasePath(tsdb, query); long start_time = DateTime.parseDateTimeString(query.getRequiredQueryStringParam("start"), query.getQueryStringParam("tz")); final boolean nocache = query.hasQueryStringParam("nocache"); if (start_time == -1) { throw BadRequestException.missingParameter("start"); } else { // temp fixup to seconds from ms until the rest of TSDB supports ms // Note you can't append this to the DateTime.parseDateTimeString() call as // it clobbers -1 results start_time /= 1000; } long end_time = DateTime.parseDateTimeString(query.getQueryStringParam("end"), query.getQueryStringParam("tz")); final long now = System.currentTimeMillis() / 1000; if (end_time == -1) { end_time = now; } else { // temp fixup to seconds from ms until the rest of TSDB supports ms // Note you can't append this to the DateTime.parseDateTimeString() call as // it clobbers -1 results end_time /= 1000; } final int max_age = computeMaxAge(query, start_time, end_time, now); if (!nocache && isDiskCacheHit(query, end_time, max_age, basepath)) { return; } Query[] tsdbqueries; List<String> options; tsdbqueries = parseQuery(tsdb, query); options = query.getQueryStringParams("o"); if (options == null) { options = new ArrayList<String>(tsdbqueries.length); for (int i = 0; i < tsdbqueries.length; i++) { options.add(""); } } else if (options.size() != tsdbqueries.length) { throw new BadRequestException( options.size() + " `o' parameters, but " + tsdbqueries.length + " `m' parameters."); } for (final Query tsdbquery : tsdbqueries) { try { tsdbquery.setStartTime(start_time); } catch (IllegalArgumentException e) { throw new BadRequestException("start time: " + e.getMessage()); } try { tsdbquery.setEndTime(end_time); } catch (IllegalArgumentException e) { throw new BadRequestException("end time: " + e.getMessage()); } } final Plot plot = new Plot(start_time, end_time, DateTime.timezones.get(query.getQueryStringParam("tz"))); setPlotDimensions(query, plot); setPlotParams(query, plot); final int nqueries = tsdbqueries.length; @SuppressWarnings("unchecked") final HashSet<String>[] aggregated_tags = new HashSet[nqueries]; int npoints = 0; for (int i = 0; i < nqueries; i++) { try { // execute the TSDB query! // XXX This is slow and will block Netty. TODO(tsuna): Don't block. // TODO(tsuna): Optimization: run each query in parallel. final DataPoints[] series = tsdbqueries[i].run(); for (final DataPoints datapoints : series) { plot.add(datapoints, options.get(i)); aggregated_tags[i] = new HashSet<String>(); aggregated_tags[i].addAll(datapoints.getAggregatedTags()); npoints += datapoints.aggregatedSize(); } } catch (RuntimeException e) { logInfo(query, "Query failed (stack trace coming): " + tsdbqueries[i]); throw e; } tsdbqueries[i] = null; // free() } tsdbqueries = null; // free() if (query.hasQueryStringParam("ascii")) { respondAsciiQuery(query, max_age, basepath, plot); return; } try { gnuplot.execute(new RunGnuplot(query, max_age, plot, basepath, aggregated_tags, npoints)); } catch (RejectedExecutionException e) { query.internalError(new Exception("Too many requests pending," + " please try again later", e)); } } /** * Decides how long we're going to allow the client to cache our response. * <p> * Based on the query, we'll decide whether or not we want to allow the * client to cache our response and for how long. * @param query The query to serve. * @param start_time The start time on the query (32-bit unsigned int, secs). * @param end_time The end time on the query (32-bit unsigned int, seconds). * @param now The current time (32-bit unsigned int, seconds). * @return A positive integer, in seconds. */ private static int computeMaxAge(final HttpQuery query, final long start_time, final long end_time, final long now) { // If the end time is in the future (1), make the graph uncacheable. // Otherwise, if the end time is far enough in the past (2) such that // no TSD can still be writing to rows for that time span and it's not // specified in a relative fashion (3) (e.g. "1d-ago"), make the graph // cacheable for a day since it's very unlikely that any data will change // for this time span. // Otherwise (4), allow the client to cache the graph for ~0.1% of the // time span covered by the request e.g., for 1h of data, it's OK to // serve something 3s stale, for 1d of data, 84s stale. if (end_time > now) { // (1) return 0; } else if (end_time < now - Const.MAX_TIMESPAN // (2) && !DateTime.isRelativeDate(query.getQueryStringParam("start")) // (3) && !DateTime.isRelativeDate(query.getQueryStringParam("end"))) { return 86400; } else { // (4) return (int) (end_time - start_time) >> 10; } } // Runs Gnuplot in a subprocess to generate the graph. private static final class RunGnuplot implements Runnable { private final HttpQuery query; private final int max_age; private final Plot plot; private final String basepath; private final HashSet<String>[] aggregated_tags; private final int npoints; public RunGnuplot(final HttpQuery query, final int max_age, final Plot plot, final String basepath, final HashSet<String>[] aggregated_tags, final int npoints) { this.query = query; this.max_age = max_age; this.plot = plot; if (IS_WINDOWS) this.basepath = basepath.replace("\\", "\\\\").replace("/", "\\\\"); else this.basepath = basepath; this.aggregated_tags = aggregated_tags; this.npoints = npoints; } public void run() { try { execute(); } catch (BadRequestException e) { query.badRequest(e.getMessage()); } catch (GnuplotException e) { query.badRequest("<pre>" + e.getMessage() + "</pre>"); } catch (RuntimeException e) { query.internalError(e); } catch (IOException e) { query.internalError(e); } } private void execute() throws IOException { final int nplotted = runGnuplot(query, basepath, plot); if (query.hasQueryStringParam("json")) { final HashMap<String, Object> results = new HashMap<String, Object>(); results.put("plotted", nplotted); results.put("points", npoints); // 1.0 returned an empty inner array if the 1st hashset was null, to do // the same we need to fudge it with an empty set if (aggregated_tags != null && aggregated_tags.length > 0 && aggregated_tags[0] == null) { aggregated_tags[0] = new HashSet<String>(); } results.put("etags", aggregated_tags); results.put("timing", query.processingTimeMillis()); query.sendReply(JSON.serializeToBytes(results)); writeFile(query, basepath + ".json", JSON.serializeToBytes(results)); } else if (query.hasQueryStringParam("png")) { query.sendFile(basepath + ".png", max_age); } else { query.internalError(new Exception("Should never be here!")); } // TODO(tsuna): Expire old files from the on-disk cache. graphlatency.add(query.processingTimeMillis()); graphs_generated.incrementAndGet(); } } /** Shuts down the thread pool used to run Gnuplot. */ public void shutdown() { gnuplot.shutdown(); } /** * Collects the stats and metrics tracked by this instance. * @param collector The collector to use. */ public static void collectStats(final StatsCollector collector) { collector.record("http.latency", graphlatency, "type=graph"); collector.record("http.latency", gnuplotlatency, "type=gnuplot"); collector.record("http.graph.requests", graphs_diskcache_hit, "cache=disk"); collector.record("http.graph.requests", graphs_generated, "cache=miss"); } /** Returns the base path to use for the Gnuplot files. */ private String getGnuplotBasePath(final TSDB tsdb, final HttpQuery query) { final Map<String, List<String>> q = query.getQueryString(); q.remove("ignore"); // Super cheap caching mechanism: hash the query string. final HashMap<String, List<String>> qs = new HashMap<String, List<String>>(q); // But first remove the parameters that don't influence the output. qs.remove("png"); qs.remove("json"); qs.remove("ascii"); return tsdb.getConfig().getDirectoryName("tsd.http.cachedir") + Integer.toHexString(qs.hashCode()); } /** * Checks whether or not it's possible to re-serve this query from disk. * @param query The query to serve. * @param end_time The end time on the query (32-bit unsigned int, seconds). * @param max_age The maximum time (in seconds) we wanna allow clients to * cache the result in case of a cache hit. * @param basepath The base path used for the Gnuplot files. * @return {@code true} if this request was served from disk (in which * case processing can stop here), {@code false} otherwise (in which case * the query needs to be processed). */ private boolean isDiskCacheHit(final HttpQuery query, final long end_time, final int max_age, final String basepath) throws IOException { final String cachepath = basepath + (query.hasQueryStringParam("ascii") ? ".txt" : ".png"); final File cachedfile = new File(cachepath); if (cachedfile.exists()) { final long bytes = cachedfile.length(); if (bytes < 21) { // Minimum possible size for a PNG: 21 bytes. // For .txt files, <21 bytes is almost impossible. logWarn(query, "Cached " + cachepath + " is too small (" + bytes + " bytes) to be valid. Ignoring it."); return false; } if (staleCacheFile(query, end_time, max_age, cachedfile)) { return false; } if (query.hasQueryStringParam("json")) { HashMap<String, Object> map = loadCachedJson(query, end_time, max_age, basepath); if (map == null) { map = new HashMap<String, Object>(); } map.put("timing", query.processingTimeMillis()); map.put("cachehit", "disk"); query.sendReply(JSON.serializeToBytes(map)); } else if (query.hasQueryStringParam("png") || query.hasQueryStringParam("ascii")) { query.sendFile(cachepath, max_age); } else { query.sendReply(HttpQuery.makePage("TSDB Query", "Your graph is ready", "<img src=\"" + query.request().getUri() + "&png\"/><br/>" + "<small>(served from disk cache)</small>")); } graphs_diskcache_hit.incrementAndGet(); return true; } // We didn't find an image. Do a negative cache check. If we've seen // this query before but there was no result, we at least wrote the JSON. final HashMap<String, Object> map = loadCachedJson(query, end_time, max_age, basepath); // If we don't have a JSON file it's a complete cache miss. If we have // one, and it says 0 data points were plotted, it's a negative cache hit. if (map == null || !map.containsKey("plotted") || ((Integer) map.get("plotted")) == 0) { return false; } if (query.hasQueryStringParam("json")) { map.put("timing", query.processingTimeMillis()); map.put("cachehit", "disk"); query.sendReply(JSON.serializeToBytes(map)); } else if (query.hasQueryStringParam("png")) { query.sendReply(" "); // Send back an empty response... } else { query.sendReply(HttpQuery.makePage("TSDB Query", "No results", "Sorry, your query didn't return anything.<br/>" + "<small>(served from disk cache)</small>")); } graphs_diskcache_hit.incrementAndGet(); return true; } /** * Returns whether or not the given cache file can be used or is stale. * @param query The query to serve. * @param end_time The end time on the query (32-bit unsigned int, seconds). * @param max_age The maximum time (in seconds) we wanna allow clients to * cache the result in case of a cache hit. If the file is exactly that * old, it is not considered stale. * @param cachedfile The file to check for staleness. */ private static boolean staleCacheFile(final HttpQuery query, final long end_time, final long max_age, final File cachedfile) { final long mtime = cachedfile.lastModified() / 1000; if (mtime <= 0) { return true; // File doesn't exist, or can't be read. } final long now = System.currentTimeMillis() / 1000; // How old is the cached file, in seconds? final long staleness = now - mtime; if (staleness < 0) { // Can happen if the mtime is "in the future". logWarn(query, "Not using file @ " + cachedfile + " with weird" + " mtime in the future: " + mtime); return true; // Play it safe, pretend we can't use this file. } // Case 1: The end time is an absolute point in the past. // We might be able to re-use the cached file. if (0 < end_time && end_time < now) { // If the file was created prior to the end time, maybe we first // executed this query while the result was uncacheable. We can // tell by looking at the mtime on the file. If the file was created // before the query end time, then it contains partial results that // shouldn't be served again. return mtime < end_time; } // Case 2: The end time of the query is now or in the future. // The cached file contains partial data and can only be re-used if it's // not too old. if (staleness > max_age) { logInfo(query, "Cached file @ " + cachedfile.getPath() + " is " + staleness + "s stale, which is more than its limit of " + max_age + "s, and needs to be regenerated."); return true; } return false; } /** * Writes the given byte array into a file. * This function logs an error but doesn't throw if it fails. * @param query The query being handled (for logging purposes). * @param path The path to write to. * @param contents The contents to write into the file. */ private static void writeFile(final HttpQuery query, final String path, final byte[] contents) { try { final FileOutputStream out = new FileOutputStream(path); try { out.write(contents); } finally { out.close(); } } catch (FileNotFoundException e) { logError(query, "Failed to create file " + path, e); } catch (IOException e) { logError(query, "Failed to write file " + path, e); } } /** * Reads a file into a byte array. * @param query The query being handled (for logging purposes). * @param file The file to read. * @param max_length The maximum number of bytes to read from the file. * @return {@code null} if the file doesn't exist or is empty or couldn't be * read, otherwise a byte array of up to {@code max_length} bytes. */ private static byte[] readFile(final HttpQuery query, final File file, final int max_length) { final int length = (int) file.length(); if (length <= 0) { return null; } FileInputStream in; try { in = new FileInputStream(file.getPath()); } catch (FileNotFoundException e) { return null; } try { final byte[] buf = new byte[Math.min(length, max_length)]; final int read = in.read(buf); if (read != buf.length) { logError(query, "When reading " + file + ": read only " + read + " bytes instead of " + buf.length); return null; } return buf; } catch (IOException e) { logError(query, "Error while reading " + file, e); return null; } finally { try { in.close(); } catch (IOException e) { logError(query, "Error while closing " + file, e); } } } /** * Attempts to read the cached {@code .json} file for this query. * @param query The query to serve. * @param end_time The end time on the query (32-bit unsigned int, seconds). * @param max_age The maximum time (in seconds) we wanna allow clients to * cache the result in case of a cache hit. * @param basepath The base path used for the Gnuplot files. * @return {@code null} in case no file was found, or the contents of the * file if it was found. * @throws IOException If the file cannot be loaded * @throws JsonMappingException If the JSON cannot be parsed to a HashMap * @throws JsonParseException If the JSON is improperly formatted */ @SuppressWarnings("unchecked") private HashMap<String, Object> loadCachedJson(final HttpQuery query, final long end_time, final long max_age, final String basepath) throws JsonParseException, JsonMappingException, IOException { final String json_path = basepath + ".json"; File json_cache = new File(json_path); if (staleCacheFile(query, end_time, max_age, json_cache)) { return null; } final byte[] json = readFile(query, json_cache, 4096); if (json == null) { return null; } json_cache = null; return (HashMap<String, Object>) JSON.parseToObject(json, HashMap.class); } /** Parses the {@code wxh} query parameter to set the graph dimension. */ static void setPlotDimensions(final HttpQuery query, final Plot plot) { final String wxh = query.getQueryStringParam("wxh"); if (wxh != null && !wxh.isEmpty()) { final int wxhlength = wxh.length(); if (wxhlength < 7) { // 100x100 minimum. throw new BadRequestException("Parameter wxh too short: " + wxh); } final int x = wxh.indexOf('x', 3); // Start at 2 as min size is 100x100 if (x < 0) { throw new BadRequestException("Invalid wxh parameter: " + wxh); } try { final short width = Short.parseShort(wxh.substring(0, x)); final short height = Short.parseShort(wxh.substring(x + 1, wxhlength)); try { plot.setDimensions(width, height); } catch (IllegalArgumentException e) { throw new BadRequestException("Invalid wxh parameter: " + wxh + ", " + e.getMessage()); } } catch (NumberFormatException e) { throw new BadRequestException("Can't parse wxh '" + wxh + "': " + e.getMessage()); } } } /** * Formats and quotes the given string so it's a suitable Gnuplot string. * @param s The string to stringify. * @return A string suitable for use as a literal string in Gnuplot. */ private static String stringify(final String s) { final StringBuilder buf = new StringBuilder(1 + s.length() + 1); buf.append('"'); HttpQuery.escapeJson(s, buf); // Abusing this function gets the job done. buf.append('"'); return buf.toString(); } /** * Pops out of the query string the given parameter. * @param querystring The query string. * @param param The name of the parameter to pop out. * @return {@code null} if the parameter wasn't passed, otherwise the * value of the last occurrence of the parameter. */ private static String popParam(final Map<String, List<String>> querystring, final String param) { final List<String> params = querystring.remove(param); if (params == null) { return null; } return params.get(params.size() - 1); } /** * Applies the plot parameters from the query to the given plot. * @param query The query from which to get the query string. * @param plot The plot on which to apply the parameters. */ static void setPlotParams(final HttpQuery query, final Plot plot) { final HashMap<String, String> params = new HashMap<String, String>(); final Map<String, List<String>> querystring = query.getQueryString(); String value; if ((value = popParam(querystring, "yrange")) != null) { params.put("yrange", value); } if ((value = popParam(querystring, "y2range")) != null) { params.put("y2range", value); } if ((value = popParam(querystring, "ylabel")) != null) { params.put("ylabel", stringify(value)); } if ((value = popParam(querystring, "y2label")) != null) { params.put("y2label", stringify(value)); } if ((value = popParam(querystring, "yformat")) != null) { params.put("format y", stringify(value)); } if ((value = popParam(querystring, "y2format")) != null) { params.put("format y2", stringify(value)); } if ((value = popParam(querystring, "xformat")) != null) { params.put("format x", stringify(value)); } if ((value = popParam(querystring, "ylog")) != null) { params.put("logscale y", ""); } if ((value = popParam(querystring, "y2log")) != null) { params.put("logscale y2", ""); } if ((value = popParam(querystring, "key")) != null) { params.put("key", value); } if ((value = popParam(querystring, "title")) != null) { params.put("title", stringify(value)); } if ((value = popParam(querystring, "bgcolor")) != null) { params.put("bgcolor", value); } if ((value = popParam(querystring, "fgcolor")) != null) { params.put("fgcolor", value); } if ((value = popParam(querystring, "smooth")) != null) { params.put("smooth", value); } // This must remain after the previous `if' in order to properly override // any previous `key' parameter if a `nokey' parameter is given. if ((value = popParam(querystring, "nokey")) != null) { params.put("key", null); } plot.setParams(params); } /** * Runs Gnuplot in a subprocess to generate the graph. * <strong>This function will block</strong> while Gnuplot is running. * @param query The query being handled (for logging purposes). * @param basepath The base path used for the Gnuplot files. * @param plot The plot object to generate Gnuplot's input files. * @return The number of points plotted by Gnuplot (0 or more). * @throws IOException if the Gnuplot files can't be written, or * the Gnuplot subprocess fails to start, or we can't read the * graph from the file it produces, or if we have been interrupted. * @throws GnuplotException if Gnuplot returns non-zero. */ static int runGnuplot(final HttpQuery query, final String basepath, final Plot plot) throws IOException { final int nplotted = plot.dumpToFiles(basepath); final long start_time = System.nanoTime(); final Process gnuplot = new ProcessBuilder(GNUPLOT, basepath + ".out", basepath + ".err", basepath + ".gnuplot").start(); final int rv; try { rv = gnuplot.waitFor(); // Couldn't find how to do this asynchronously. } catch (InterruptedException e) { Thread.currentThread().interrupt(); // Restore the interrupted status. throw new IOException("interrupted", e); // I hate checked exceptions. } finally { // We need to always destroy() the Process, otherwise we "leak" file // descriptors and pipes. Unless I'm blind, this isn't actually // documented in the Javadoc of the !@#$%^ JDK, and in Java 6 there's no // way to ask the stupid-ass ProcessBuilder to not create fucking pipes. // I think when the GC kicks in the JVM may run some kind of a finalizer // that closes the pipes, because I've never seen this issue on long // running TSDs, except where ulimit -n was low (the default, 1024). gnuplot.destroy(); } gnuplotlatency.add((int) ((System.nanoTime() - start_time) / 1000000)); if (rv != 0) { final byte[] stderr = readFile(query, new File(basepath + ".err"), 4096); // Sometimes Gnuplot will error out but still create the file. new File(basepath + ".png").delete(); if (stderr == null) { throw new GnuplotException(rv); } throw new GnuplotException(new String(stderr)); } // Remove the files for stderr/stdout if they're empty. deleteFileIfEmpty(basepath + ".out"); deleteFileIfEmpty(basepath + ".err"); return nplotted; } private static void deleteFileIfEmpty(final String path) { final File file = new File(path); if (file.length() <= 0) { file.delete(); } } /** * Respond to a query that wants the output in ASCII. * <p> * When a query specifies the "ascii" query string parameter, we send the * data points back to the client in plain text instead of sending a PNG. * @param query The query we're currently serving. * @param max_age The maximum time (in seconds) we wanna allow clients to * cache the result in case of a cache hit. * @param basepath The base path used for the Gnuplot files. * @param plot The plot object to generate Gnuplot's input files. */ private static void respondAsciiQuery(final HttpQuery query, final int max_age, final String basepath, final Plot plot) { final String path = basepath + ".txt"; PrintWriter asciifile; try { asciifile = new PrintWriter(path); } catch (IOException e) { query.internalError(e); return; } try { final StringBuilder tagbuf = new StringBuilder(); for (final DataPoints dp : plot.getDataPoints()) { final String metric = dp.metricName(); tagbuf.setLength(0); for (final Map.Entry<String, String> tag : dp.getTags().entrySet()) { tagbuf.append(' ').append(tag.getKey()).append('=').append(tag.getValue()); } for (final DataPoint d : dp) { asciifile.print(metric); asciifile.print(' '); asciifile.print((d.timestamp() / 1000)); asciifile.print(' '); if (d.isInteger()) { asciifile.print(d.longValue()); } else { final double value = d.doubleValue(); if (value != value || Double.isInfinite(value)) { throw new IllegalStateException( "NaN or Infinity:" + value + " d=" + d + ", query=" + query); } asciifile.print(value); } asciifile.print(tagbuf); asciifile.print('\n'); } } } finally { asciifile.close(); } try { query.sendFile(path, max_age); } catch (IOException e) { query.internalError(e); } } /** * Parses the {@code /q} query in a list of {@link Query} objects. * @param tsdb The TSDB to use. * @param query The HTTP query for {@code /q}. * @return The corresponding {@link Query} objects. * @throws BadRequestException if the query was malformed. * @throws IllegalArgumentException if the metric or tags were malformed. */ private static Query[] parseQuery(final TSDB tsdb, final HttpQuery query) { final List<String> ms = query.getQueryStringParams("m"); if (ms == null) { throw BadRequestException.missingParameter("m"); } final Query[] tsdbqueries = new Query[ms.size()]; int nqueries = 0; for (final String m : ms) { // m is of the following forms: // agg:[interval-agg:][rate[{counter[,[countermax][,resetvalue]]}]:] // metric[{tag=value,...}] // Where the parts in square brackets `[' .. `]' are optional. final String[] parts = Tags.splitString(m, ':'); int i = parts.length; if (i < 2 || i > 4) { throw new BadRequestException("Invalid parameter m=" + m + " (" + (i < 2 ? "not enough" : "too many") + " :-separated parts)"); } final Aggregator agg = getAggregator(parts[0]); i--; // Move to the last part (the metric name). final HashMap<String, String> parsedtags = new HashMap<String, String>(); final String metric = Tags.parseWithMetric(parts[i], parsedtags); final boolean rate = parts[--i].startsWith("rate"); final RateOptions rate_options = QueryRpc.parseRateOptions(rate, parts[i]); if (rate) { i--; // Move to the next part. } final Query tsdbquery = tsdb.newQuery(); try { tsdbquery.setTimeSeries(metric, parsedtags, agg, rate, rate_options); } catch (NoSuchUniqueName e) { throw new BadRequestException(e.getMessage()); } // downsampling function & interval. if (i > 0) { final int dash = parts[1].indexOf('-', 1); // 1st char can't be `-'. if (dash < 0) { throw new BadRequestException("Invalid downsampling specifier '" + parts[1] + "' in m=" + m); } Aggregator downsampler; try { downsampler = Aggregators.get(parts[1].substring(dash + 1)); } catch (NoSuchElementException e) { throw new BadRequestException("No such downsampling function: " + parts[1].substring(dash + 1)); } final long interval = DateTime.parseDuration(parts[1].substring(0, dash)); tsdbquery.downsample(interval, downsampler); } else { tsdbquery.downsample(1000, agg); } tsdbqueries[nqueries++] = tsdbquery; } return tsdbqueries; } /** * Returns the aggregator with the given name. * @param name Name of the aggregator to get. * @throws BadRequestException if there's no aggregator with this name. */ private static final Aggregator getAggregator(final String name) { try { return Aggregators.get(name); } catch (NoSuchElementException e) { throw new BadRequestException("No such aggregation function: " + name); } } private static final PlotThdFactory thread_factory = new PlotThdFactory(); private static final class PlotThdFactory implements ThreadFactory { private final AtomicInteger id = new AtomicInteger(0); public Thread newThread(final Runnable r) { return new Thread(r, "Gnuplot #" + id.incrementAndGet()); } } /** Name of the wrapper script we use to execute Gnuplot. */ private static final String WRAPPER = IS_WINDOWS ? "mygnuplot.bat" : "mygnuplot.sh"; /** Path to the wrapper script. */ private static final String GNUPLOT; static { GNUPLOT = findGnuplotHelperScript(); } /** * Iterate through the class path and look for the Gnuplot helper script. * @return The path to the wrapper script. */ private static String findGnuplotHelperScript() { final URL url = GraphHandler.class.getClassLoader().getResource(WRAPPER); if (url == null) { throw new RuntimeException("Couldn't find " + WRAPPER + " on the" + " CLASSPATH: " + System.getProperty("java.class.path")); } final String path = url.getFile(); LOG.debug("Using Gnuplot wrapper at {}", path); final File file = new File(path); final String error; if (!file.exists()) { error = "non-existent"; } else if (!file.canExecute()) { error = "non-executable"; } else if (!file.canRead()) { error = "unreadable"; } else { return path; } throw new RuntimeException("The " + WRAPPER + " found on the" + " CLASSPATH (" + path + ") is a " + error + " file... WTF?" + " CLASSPATH=" + System.getProperty("java.class.path")); } // ---------------- // // Logging helpers. // // ---------------- // static void logInfo(final HttpQuery query, final String msg) { LOG.info(query.channel().toString() + ' ' + msg); } static void logWarn(final HttpQuery query, final String msg) { LOG.warn(query.channel().toString() + ' ' + msg); } static void logError(final HttpQuery query, final String msg) { LOG.error(query.channel().toString() + ' ' + msg); } static void logError(final HttpQuery query, final String msg, final Throwable e) { LOG.error(query.channel().toString() + ' ' + msg, e); } }