Java tutorial
// This file is part of OpenTSDB. // Copyright (C) 2010-2012 The OpenTSDB Authors. // // This program is free software: you can redistribute it and/or modify it // under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 2.1 of the License, or (at your // option) any later version. This program is distributed in the hope that it // will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty // of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser // General Public License for more details. You should have received a copy // of the GNU Lesser General Public License along with this program. If not, // see <http://www.gnu.org/licenses/>. package net.opentsdb.core; import net.opentsdb.uid.NoSuchUniqueId; import net.opentsdb.uid.NoSuchUniqueName; import org.apache.commons.lang.time.DateFormatUtils; import org.hbase.async.Bytes; import org.hbase.async.HBaseException; import org.hbase.async.KeyValue; import org.hbase.async.Scanner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.nio.charset.Charset; import java.util.*; /** * Non-synchronized implementation of {@link net.opentsdb.core.Query}. */ final class TsdbQuery implements Query { private static final Logger LOG = LoggerFactory.getLogger(TsdbQuery.class); /** * Charset to use with our server-side row-filter. * We use this one because it preserves every possible byte unchanged. */ private static final Charset CHARSET = Charset.forName("UTF-8"); /** * The TSDB we belong to. */ private final TSDB tsdb; /** * Value used for timestamps that are uninitialized. */ private static final int UNSET = -1; /** * Start time (UNIX timestamp in seconds) on 32 bits ("unsigned" int). */ private int start_time = UNSET; /** * End time (UNIX timestamp in seconds) on 32 bits ("unsigned" int). */ private int end_time = UNSET; /** * ID of the metric being looked up. */ private byte[] metric; /** * Tags of the metrics being looked up. * Each tag is a byte array holding the ID of both the name and value * of the tag. * Invariant: an element cannot be both in this array and in group_bys. */ private ArrayList<byte[]> tags; /** * If true, use rate of change instead of actual values. */ private boolean rate; private final String DATE_PATTERN = "yyyy-MM-dd HH:mm:ss"; private ArrayList<ArrayList<KeyValue>> rows = new ArrayList<ArrayList<KeyValue>>(); /** * Constructor. */ public TsdbQuery(final TSDB tsdb) { this.tsdb = tsdb; } public void setStartTime(final long timestamp) { if ((timestamp & 0xFFFFFFFF00000000L) != 0) { throw new IllegalArgumentException("Invalid timestamp: " + timestamp); } else if (end_time != UNSET && timestamp >= getEndTime()) { throw new IllegalArgumentException( "new start time (" + timestamp + ") is greater than or equal to end time: " + getEndTime()); } // Keep the 32 bits. start_time = (int) timestamp; } public long getStartTime() { if (start_time == UNSET) { throw new IllegalStateException("setStartTime was never called!"); } return start_time & 0x00000000FFFFFFFFL; } public void setEndTime(final long timestamp) { if ((timestamp & 0xFFFFFFFF00000000L) != 0) { throw new IllegalArgumentException("Invalid timestamp: " + timestamp); } else if (start_time != UNSET && timestamp <= getStartTime()) { throw new IllegalArgumentException( "new end time (" + timestamp + ") is less than or equal to start time: " + getStartTime()); } // Keep the 32 bits. end_time = (int) timestamp; } public long getEndTime() { if (end_time == UNSET) { setEndTime(System.currentTimeMillis() / 1000); } return end_time; } public void setTimeSeries(final String metric, final Map<String, String> tags, final boolean rate) throws NoSuchUniqueName { this.metric = tsdb.metrics.getId(metric); this.tags = Tags.resolveAll(tsdb, tags); this.rate = rate; } public void run() throws HBaseException { collectRows(); } private void collectRows() { final Scanner scanner = getScanner(); try { ArrayList<ArrayList<KeyValue>> rows; while ((rows = scanner.nextRows().joinUninterruptibly()) != null) { this.rows.addAll(rows); } } catch (RuntimeException e) { throw e; } catch (Exception e) { throw new RuntimeException("Should never be here", e); } } @Override public Vector getVector(char monitDataTypeParm) { DataType dataType = DataType.getDataType(monitDataTypeParm); if (this.rows == null) { return new Vector(); } else { Vector<String[]> vector = new Vector<String[]>(); for (int i = 0; i < this.rows.size(); i++) { List<KeyValue> oneList = rows.get(i); if (i == 0) { selectStartRows(oneList, vector, dataType); } else if (i == this.rows.size() - 1) { selectEndRows(oneList, vector, dataType); } else { selectMiddleRows(oneList, vector, dataType); } } return vector; } } private void selectMiddleRows(List<KeyValue> oneList, Vector<String[]> vector, DataType dataType) { for (KeyValue kv : oneList) { String[] strArray = new String[] { dataType.getData(kv.value()), DateFormatUtils.format( new Date((Bytes.getUnsignedInt(kv.key(), tsdb.metrics.width()) + (short) (Bytes.getShort(kv.qualifier()) >> Const.FLAG_BITS)) * 1000), DATE_PATTERN) }; vector.add(strArray); } } private void selectEndRows(List<KeyValue> oneList, Vector<String[]> vector, DataType dataType) { final long base_time = (end_time - (end_time % Const.MAX_TIMESPAN)); short endMinAndSec = (short) (end_time - base_time); int myBaseTime = 0; for (int i = 0; i < oneList.size(); i++) { KeyValue kv = oneList.get(i); if (i == 0) { myBaseTime = (int) Bytes.getUnsignedInt(kv.key(), tsdb.metrics.width()); } short myMinAndSec = (short) (Bytes.getShort(kv.qualifier()) >> Const.FLAG_BITS); //?????,??? if (myBaseTime < base_time || (myBaseTime == base_time && myMinAndSec <= endMinAndSec)) { String[] strArray = new String[] { dataType.getData(kv.value()), DateFormatUtils.format(new Date( (Bytes.getUnsignedInt(kv.key(), tsdb.metrics.width()) + myMinAndSec) * 1000), DATE_PATTERN) }; vector.add(strArray); } } } private void selectStartRows(List<KeyValue> oneList, Vector<String[]> vector, DataType dataType) { final int base_time = (start_time - (start_time % Const.MAX_TIMESPAN)); short startMinAndSec = (short) (start_time - base_time); int myBaseTime = 0; for (int i = 0; i < oneList.size(); i++) { KeyValue kv = oneList.get(i); if (i == 0) { myBaseTime = (int) Bytes.getUnsignedInt(kv.key(), tsdb.metrics.width()); } short myMinAndSec = (short) (Bytes.getShort(kv.qualifier()) >> Const.FLAG_BITS); //????,??? if (myBaseTime > base_time || (myBaseTime == base_time && myMinAndSec >= startMinAndSec)) { String[] strArray = new String[] { dataType.getData(kv.value()), DateFormatUtils.format(new Date( (Bytes.getUnsignedInt(kv.key(), tsdb.metrics.width()) + myMinAndSec) * 1000), DATE_PATTERN) }; vector.add(strArray); } } } @Override public Set<byte[]> getRowkeys() { Set<byte[]> set = new HashSet<byte[]>(); for (ArrayList<KeyValue> oneList : this.rows) { for (KeyValue kv : oneList) { set.add(kv.key()); } } return set; } /** * Creates the {@link org.hbase.async.Scanner} to use for this query. */ Scanner getScanner() throws HBaseException { final short metric_width = tsdb.metrics.width(); final byte[] start_row = new byte[metric_width + Const.TIMESTAMP_BYTES]; final byte[] end_row = new byte[metric_width + Const.TIMESTAMP_BYTES]; // We search at least one row before and one row after the start & end // time we've been given as it's quite likely that the exact timestamp // we're looking for is in the middle of a row. Plus, a number of things // rely on having a few extra data points before & after the exact start // & end dates in order to do proper rate calculation or downsampling near // the "edges" of the graph. final int startBaseTime = (start_time - (start_time % Const.MAX_TIMESPAN)); final int endBaseTime = (end_time - (end_time % Const.MAX_TIMESPAN)) + 3600; Bytes.setInt(start_row, startBaseTime, metric_width); Bytes.setInt(end_row, (end_time == UNSET ? -1 // Will scan until the end (0xFFF...). : endBaseTime), metric_width); System.arraycopy(metric, 0, start_row, 0, metric_width); System.arraycopy(metric, 0, end_row, 0, metric_width); final Scanner scanner = tsdb.client.newScanner(tsdb.table); scanner.setStartKey(start_row); scanner.setStopKey(end_row); scanner.setFamily(TSDB.FAMILY); return scanner; } public String toString() { final StringBuilder buf = new StringBuilder(); buf.append("TsdbQuery(start_time=").append(getStartTime()).append(", end_time=").append(getEndTime()) .append(", metric=").append(Arrays.toString(metric)); try { buf.append(" (").append(tsdb.metrics.getName(metric)); } catch (NoSuchUniqueId e) { buf.append(" (<").append(e.getMessage()).append('>'); } try { buf.append("), tags=").append(Tags.resolveIds(tsdb, tags)); } catch (NoSuchUniqueId e) { buf.append("), tags=<").append(e.getMessage()).append('>'); } buf.append(", rate=").append(rate).append(", group_bys=("); buf.append("))"); return buf.toString(); } protected static abstract class DataType { public abstract String getData(byte[] bytes); private static DataType dataInt = new DataInt(); private static DataType dataLong = new DataLong(); private static DataType dataString = new DataString(); private static DataType dataDouble = new DataDouble(); private static DataType dataChar = new DataChar(); public static DataType getDataType(char type) { DataType dataType = null; switch (type) { case 'i': dataType = dataInt; break; case 'd': dataType = dataDouble; break; case 'l': dataType = dataLong; break; case 's': dataType = dataString; break; case 'c': dataType = dataChar; default: break; } return dataType; } } private static class DataInt extends DataType { @Override public String getData(byte[] bytes) { return String.valueOf(getIntFromBytes(bytes)); } } private static class DataLong extends DataType { @Override public String getData(byte[] bytes) { return String.valueOf(getLongFromBytes(bytes)); } } private static class DataDouble extends DataType { @Override public String getData(byte[] bytes) { return String.valueOf(getDoubleFromBytes(bytes)); } } private static class DataString extends DataType { @Override public String getData(byte[] bytes) { return new String(bytes); } } private static class DataChar extends DataType { @Override public String getData(byte[] bytes) { return String.valueOf(getCharFromBytes(bytes)); } } public static int getIntFromBytes(byte[] valueBytesParm) { int returnInt = -1; try { int ch1 = (valueBytesParm[0] & 0xFF); int ch2 = (valueBytesParm[1] & 0xFF); int ch3 = (valueBytesParm[2] & 0xFF); int ch4 = (valueBytesParm[3] & 0xFF); returnInt = ((ch1 << 24) + (ch2 << 16) + (ch3 << 8) + (ch4 << 0)); } catch (Exception e) { e.printStackTrace(); } return returnInt; } public static long getLongFromBytes(byte[] valueBytesParm) { long returnLong = -1; try { byte[] tmpHighBytes = new byte[4]; System.arraycopy(valueBytesParm, 0, tmpHighBytes, 0, 4); byte[] tmpLowBytes = new byte[4]; System.arraycopy(valueBytesParm, 4, tmpLowBytes, 0, 4); returnLong = ((long) (getIntFromBytes(tmpHighBytes)) << 32) + (getIntFromBytes(tmpLowBytes) & 0xFFFFFFFFL); } catch (Exception e) { e.printStackTrace(); } return returnLong; } public static double getDoubleFromBytes(byte[] byteValues) { long tmpLongValue = getLongFromBytes(byteValues); return Double.longBitsToDouble(tmpLongValue); } public static char getCharFromBytes(byte[] valueBytesParm) { char returnChar = 0; try { int ch1 = (valueBytesParm[0] & 0xFF); int ch2 = (valueBytesParm[1] & 0xFF); returnChar = (char) ((ch1 << 8) + (ch2 << 0)); } catch (Exception e) { e.printStackTrace(); } return returnChar; } }