Java tutorial
/* * Copyright 2014 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of * the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations under * the License. */ package co.cask.cdap.data2.dataset2.lib.timeseries; import co.cask.cdap.api.common.Bytes; import co.cask.cdap.api.dataset.lib.cube.DimensionValue; import co.cask.cdap.api.dataset.lib.cube.MeasureType; import co.cask.cdap.api.dataset.lib.cube.Measurement; import co.cask.cdap.api.dataset.table.Row; import co.cask.cdap.api.dataset.table.Scanner; import co.cask.cdap.api.metrics.MetricsCollector; import co.cask.cdap.common.utils.ImmutablePair; import co.cask.cdap.data2.dataset2.lib.table.FuzzyRowFilter; import co.cask.cdap.data2.dataset2.lib.table.MetricsTable; import com.google.common.base.Function; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.Closeable; import java.io.IOException; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.NavigableMap; import java.util.Set; import javax.annotation.Nullable; /** * Table for storing {@link Fact}s. * * Thread safe as long as the passed into the constructor datasets are thread safe (usually is not the case). */ public final class FactTable implements Closeable { private static final Logger LOG = LoggerFactory.getLogger(FactTable.class); private static final int MAX_ROLL_TIME = 0xfffe; // hard limits on some ops to stay on safe side private static final int MAX_RECORDS_TO_SCAN_DURING_SEARCH = 10 * 1000 * 1000; private static final int MAX_SCANS_DURING_SEARCH = 10 * 1000; private static final Function<byte[], Long> BYTES_TO_LONG = new Function<byte[], Long>() { @Override public Long apply(byte[] input) { return Bytes.toLong(input); } }; private static final Function<NavigableMap<byte[], byte[]>, NavigableMap<byte[], Long>> TRANSFORM_MAP_BYTE_ARRAY_TO_LONG = new Function<NavigableMap<byte[], byte[]>, NavigableMap<byte[], Long>>() { @Override public NavigableMap<byte[], Long> apply(NavigableMap<byte[], byte[]> input) { return Maps.transformValues(input, BYTES_TO_LONG); } }; private final MetricsTable timeSeriesTable; private final EntityTable entityTable; private final FactCodec codec; private final int resolution; // todo: should not be used outside of codec private final int rollTime; private final String putCountMetric; private final String incrementCountMetric; @Nullable private MetricsCollector metrics; /** * Creates an instance of {@link FactTable}. * * @param timeSeriesTable A table for storing facts information. * @param entityTable The table for storing dimension encoding mappings. * @param resolution Resolution in seconds * @param rollTime Number of resolution for writing to a new row with a new timebase. * Meaning the differences between timebase of two consecutive rows divided by * resolution seconds. It essentially defines how many columns per row in the table. * This value should be < 65535. */ public FactTable(MetricsTable timeSeriesTable, EntityTable entityTable, int resolution, int rollTime) { // Two bytes for column name, which is a delta timestamp Preconditions.checkArgument(rollTime <= MAX_ROLL_TIME, "Rolltime should be <= " + MAX_ROLL_TIME); this.entityTable = entityTable; this.timeSeriesTable = timeSeriesTable; this.codec = new FactCodec(entityTable, resolution, rollTime); this.resolution = resolution; this.rollTime = rollTime; this.putCountMetric = "factTable." + resolution + ".put.count"; this.incrementCountMetric = "factTable." + resolution + ".increment.count"; } public void setMetricsCollector(MetricsCollector metrics) { this.metrics = metrics; } public void add(List<Fact> facts) { // Simply collecting all rows/cols/values that need to be put to the underlying table. NavigableMap<byte[], NavigableMap<byte[], byte[]>> gaugesTable = Maps.newTreeMap(Bytes.BYTES_COMPARATOR); NavigableMap<byte[], NavigableMap<byte[], byte[]>> incrementsTable = Maps .newTreeMap(Bytes.BYTES_COMPARATOR); for (Fact fact : facts) { for (Measurement measurement : fact.getMeasurements()) { byte[] rowKey = codec.createRowKey(fact.getDimensionValues(), measurement.getName(), fact.getTimestamp()); byte[] column = codec.createColumn(fact.getTimestamp()); if (MeasureType.COUNTER == measurement.getType()) { inc(incrementsTable, rowKey, column, measurement.getValue()); } else { set(gaugesTable, rowKey, column, Bytes.toBytes(measurement.getValue())); } } } NavigableMap<byte[], NavigableMap<byte[], Long>> convertedIncrementsTable = Maps .transformValues(incrementsTable, TRANSFORM_MAP_BYTE_ARRAY_TO_LONG); NavigableMap<byte[], NavigableMap<byte[], Long>> convertedGaugesTable = Maps.transformValues(gaugesTable, TRANSFORM_MAP_BYTE_ARRAY_TO_LONG); // todo: replace with single call, to be able to optimize rpcs in underlying table timeSeriesTable.put(convertedGaugesTable); timeSeriesTable.increment(convertedIncrementsTable); if (metrics != null) { metrics.increment(putCountMetric, convertedGaugesTable.size()); metrics.increment(incrementCountMetric, convertedIncrementsTable.size()); } } public FactScanner scan(FactScan scan) { return new FactScanner(getScanner(scan), codec, scan.getStartTs(), scan.getEndTs(), scan.getMeasureNames()); } private Scanner getScanner(FactScan scan) { // use null if no metrics or more than one metrics are provided in the scan String measureName = scan.getMeasureNames().size() == 1 ? scan.getMeasureNames().iterator().next() : null; byte[] startRow = codec.createStartRowKey(scan.getDimensionValues(), measureName, scan.getStartTs(), false); byte[] endRow = codec.createEndRowKey(scan.getDimensionValues(), measureName, scan.getEndTs(), false); byte[][] columns; if (Arrays.equals(startRow, endRow)) { // If on the same timebase, we only need subset of columns long timeBase = scan.getStartTs() / rollTime * rollTime; int startCol = (int) (scan.getStartTs() - timeBase) / resolution; int endCol = (int) (scan.getEndTs() - timeBase) / resolution; columns = new byte[endCol - startCol + 1][]; for (int i = 0; i < columns.length; i++) { columns[i] = Bytes.toBytes((short) (startCol + i)); } } endRow = Bytes.stopKeyForPrefix(endRow); FuzzyRowFilter fuzzyRowFilter = createFuzzyRowFilter(scan, startRow); if (LOG.isTraceEnabled()) { LOG.trace( "Scanning fact table {} with scan: {}; constructed startRow: {}, endRow: {}, fuzzyRowFilter: {}", timeSeriesTable, scan, toPrettyLog(startRow), toPrettyLog(endRow), fuzzyRowFilter); } return timeSeriesTable.scan(startRow, endRow, fuzzyRowFilter); } /** * Delete entries in fact table. * @param scan specifies deletion criteria */ public void delete(FactScan scan) { Scanner scanner = getScanner(scan); try { Row row; while ((row = scanner.next()) != null) { List<byte[]> columns = Lists.newArrayList(); boolean exhausted = false; for (byte[] column : row.getColumns().keySet()) { long ts = codec.getTimestamp(row.getRow(), column); if (ts < scan.getStartTs()) { continue; } if (ts > scan.getEndTs()) { exhausted = true; break; } columns.add(column); } // todo: do deletes efficiently, in batches, not one-by-one timeSeriesTable.delete(row.getRow(), columns.toArray(new byte[columns.size()][])); if (exhausted) { break; } } } finally { scanner.close(); } } /** * Searches for first non-null valued dimensions in records that contain given list of dimensions and match given * dimension values in given time range. Returned dimension values are those that are not defined in given * dimension values. * @param allDimensionNames list of all dimension names to be present in the record * @param dimensionSlice dimension values to filter by, {@code null} means any non-null value. * @param startTs start of the time range, in seconds * @param endTs end of the time range, in seconds * @return {@link Set} of {@link DimensionValue}s */ // todo: pass a limit on number of dimensionValues returned // todo: kinda not cool API when we expect null values in a map... public Set<DimensionValue> findSingleDimensionValue(List<String> allDimensionNames, Map<String, String> dimensionSlice, long startTs, long endTs) { // Algorithm, briefly: // We scan in the records which have given allDimensionNames. We use dimensionSlice as a criteria for scan. // If record from the scan has non-null values in the dimensions which are not specified in dimensionSlice, // we use first of such dimension as a value to return. // When we find value to return, since we only fill a single dimension, we are not interested in drilling down // further and instead attempt to fast-forward (jump) to a record that has different value in that dimension. // Thus we find all results. List<DimensionValue> allDimensions = Lists.newArrayList(); List<DimensionValue> filledDimension = Lists.newArrayList(); List<Integer> dimToFillIndexes = Lists.newArrayList(); for (int i = 0; i < allDimensionNames.size(); i++) { String dimensionName = allDimensionNames.get(i); if (!dimensionSlice.containsKey(dimensionName)) { dimToFillIndexes.add(i); allDimensions.add(new DimensionValue(dimensionName, null)); } else { DimensionValue dimensionValue = new DimensionValue(dimensionName, dimensionSlice.get(dimensionName)); filledDimension.add(dimensionValue); allDimensions.add(dimensionValue); } } // If provided dimensions contain all values filled in, there's nothing to look for if (dimToFillIndexes.isEmpty()) { return Collections.emptySet(); } Set<DimensionValue> result = Sets.newHashSet(); int scans = 0; int scannedRecords = 0; // build a scan byte[] startRow = codec.createStartRowKey(allDimensions, null, startTs, false); byte[] endRow = codec.createEndRowKey(allDimensions, null, endTs, false); endRow = Bytes.stopKeyForPrefix(endRow); FuzzyRowFilter fuzzyRowFilter = createFuzzyRowFilter( new FactScan(startTs, endTs, ImmutableList.<String>of(), allDimensions), startRow); Scanner scanner = timeSeriesTable.scan(startRow, endRow, fuzzyRowFilter); scans++; try { Row rowResult; while ((rowResult = scanner.next()) != null) { scannedRecords++; // todo: make configurable if (scannedRecords > MAX_RECORDS_TO_SCAN_DURING_SEARCH) { break; } byte[] rowKey = rowResult.getRow(); // filter out columns by time range (scan configuration only filters whole rows) if (codec.getTimestamp(rowKey, codec.createColumn(startTs)) < startTs) { continue; } if (codec.getTimestamp(rowKey, codec.createColumn(endTs)) > endTs) { // we're done with scanner break; } List<DimensionValue> dimensionValues = codec.getDimensionValues(rowResult.getRow()); // At this point, we know that the record is in right time range and its dimensions matches given. // We try find first non-null valued dimension in the record that was not in given dimensions: we use it to form // next drill down suggestion int filledIndex = -1; for (int index : dimToFillIndexes) { // todo: it may be not efficient, if dimensionValues is not array-backed list: i.e. if access by index is // not fast DimensionValue dimensionValue = dimensionValues.get(index); if (dimensionValue.getValue() != null) { result.add(dimensionValue); filledIndex = index; break; } } // Ss soon as we find dimension to fill, we are not interested into drilling down further (by contract, we fill // single dimension value). Thus, we try to jump to the record that has greater value in that dimension. // todo: fast-forwarding (jumping) should be done on server-side (CDAP-1421) if (filledIndex >= 0) { scanner.close(); scanner = null; scans++; if (scans > MAX_SCANS_DURING_SEARCH) { break; } startRow = codec.getNextRowKey(rowResult.getRow(), filledIndex); scanner = timeSeriesTable.scan(startRow, endRow, fuzzyRowFilter); } } } finally { if (scanner != null) { scanner.close(); } } LOG.trace("search for dimensions completed, scans performed: {}, scanned records: {}", scans, scannedRecords); return result; } /** * Finds all measure names of the facts that match given {@link DimensionValue}s and time range. * @param allDimensionNames list of all dimension names to be present in the fact record * @param dimensionSlice dimension values to filter by, {@code null} means any non-null value. * @param startTs start timestamp, in sec * @param endTs end timestamp, in sec * @return {@link Set} of measure names */ // todo: pass a limit on number of measures returned public Set<String> findMeasureNames(List<String> allDimensionNames, Map<String, String> dimensionSlice, long startTs, long endTs) { List<DimensionValue> allDimensions = Lists.newArrayList(); for (String dimensionName : allDimensionNames) { allDimensions.add(new DimensionValue(dimensionName, dimensionSlice.get(dimensionName))); } byte[] startRow = codec.createStartRowKey(allDimensions, null, startTs, false); byte[] endRow = codec.createEndRowKey(allDimensions, null, endTs, false); endRow = Bytes.stopKeyForPrefix(endRow); FuzzyRowFilter fuzzyRowFilter = createFuzzyRowFilter( new FactScan(startTs, endTs, ImmutableList.<String>of(), allDimensions), startRow); Set<String> measureNames = Sets.newHashSet(); int scannedRecords = 0; // todo: make configurable Scanner scanner = timeSeriesTable.scan(startRow, endRow, fuzzyRowFilter); try { Row rowResult; while ((rowResult = scanner.next()) != null) { scannedRecords++; if (scannedRecords > MAX_RECORDS_TO_SCAN_DURING_SEARCH) { break; } byte[] rowKey = rowResult.getRow(); // filter out columns by time range (scan configuration only filters whole rows) if (codec.getTimestamp(rowKey, codec.createColumn(startTs)) < startTs) { continue; } if (codec.getTimestamp(rowKey, codec.createColumn(endTs)) > endTs) { // we're done with scanner break; } measureNames.add(codec.getMeasureName(rowResult.getRow())); } } finally { scanner.close(); } LOG.trace("search for metrics completed, scanned records: {}", scannedRecords); return measureNames; } @Override public void close() throws IOException { timeSeriesTable.close(); entityTable.close(); } public static byte[][] getSplits(int aggGroupsCount) { return FactCodec.getSplits(aggGroupsCount); } @Nullable private FuzzyRowFilter createFuzzyRowFilter(FactScan scan, byte[] startRow) { // we need to always use a fuzzy row filter as it is the only one to do the matching of values // if we are querying only one metric, we will use fixed metricName for filter, // if there are no metrics or more than one metrics to query we use `ANY` fuzzy filter. String measureName = (scan.getMeasureNames().size() == 1) ? scan.getMeasureNames().iterator().next() : null; byte[] fuzzyRowMask = codec.createFuzzyRowMask(scan.getDimensionValues(), measureName); // note: we can use startRow, as it will contain all "fixed" parts of the key needed return new FuzzyRowFilter(ImmutableList.of(new ImmutablePair<>(startRow, fuzzyRowMask))); } // todo: shouldn't we aggregate "before" writing to FactTable? We could do it really efficient outside // also: the underlying datasets will do aggregation in memory anyways private static void inc(NavigableMap<byte[], NavigableMap<byte[], byte[]>> incrementsTable, byte[] rowKey, byte[] column, long value) { byte[] oldValue = get(incrementsTable, rowKey, column); long newValue = value; if (oldValue != null) { if (Bytes.SIZEOF_LONG == oldValue.length) { newValue = Bytes.toLong(oldValue) + value; } else if (Bytes.SIZEOF_INT == oldValue.length) { // In 2.4 and older versions we stored it as int newValue = Bytes.toInt(oldValue) + value; } else { // should NEVER happen, unless the table is screwed up manually throw new IllegalStateException( String.format("Could not parse measure @row %s @column %s value %s as int or long", Bytes.toStringBinary(rowKey), Bytes.toStringBinary(column), Bytes.toStringBinary(oldValue))); } } set(incrementsTable, rowKey, column, Bytes.toBytes(newValue)); } private static byte[] get(NavigableMap<byte[], NavigableMap<byte[], byte[]>> table, byte[] row, byte[] column) { NavigableMap<byte[], byte[]> rowMap = table.get(row); return rowMap == null ? null : rowMap.get(column); } private static void set(NavigableMap<byte[], NavigableMap<byte[], byte[]>> table, byte[] row, byte[] column, byte[] value) { NavigableMap<byte[], byte[]> rowMap = table.get(row); if (rowMap == null) { rowMap = Maps.newTreeMap(Bytes.BYTES_COMPARATOR); table.put(row, rowMap); } rowMap.put(column, value); } private String toPrettyLog(byte[] key) { StringBuilder sb = new StringBuilder("{"); for (byte b : key) { String enc = String.valueOf((int) b) + " "; sb.append(enc.substring(0, 5)); } sb.append("}"); return sb.toString(); } }