Java tutorial
/** * Copyright (C) 2014-2016 LinkedIn Corp. (pinot-core@linkedin.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /** * Copyright (C) 2014-2015 LinkedIn Corp. (pinot-core@linkedin.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.linkedin.pinot.perf; import com.linkedin.pinot.core.segment.memory.PinotDataBuffer; import com.linkedin.pinot.core.segment.store.SegmentDirectory; import java.io.File; import java.io.IOException; import java.nio.channels.FileChannel; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Random; import org.apache.commons.configuration.ConfigurationException; import org.roaringbitmap.buffer.ImmutableRoaringBitmap; import org.roaringbitmap.buffer.MutableRoaringBitmap; import com.linkedin.pinot.common.data.FieldSpec.DataType; import com.linkedin.pinot.common.segment.ReadMode; import com.linkedin.pinot.core.segment.index.readers.BitmapInvertedIndexReader; import com.linkedin.pinot.core.segment.index.ColumnMetadata; import com.linkedin.pinot.core.segment.index.readers.InvertedIndexReader; import com.linkedin.pinot.core.segment.index.SegmentMetadataImpl; import com.linkedin.pinot.core.segment.index.column.ColumnIndexContainer; import com.linkedin.pinot.core.segment.index.readers.ImmutableDictionaryReader; import com.linkedin.pinot.core.segment.index.readers.IntDictionary; public class BitmapPerformanceBenchmark { public static void iterationSpeed(String indexSegmentDir, String column) throws Exception { File indexSegment = new File(indexSegmentDir); SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(indexSegment); Map<String, BitmapInvertedIndexReader> bitMapIndexMap = new HashMap<String, BitmapInvertedIndexReader>(); Map<String, Integer> cardinalityMap = new HashMap<String, Integer>(); Map<String, ImmutableDictionaryReader> dictionaryMap = new HashMap<String, ImmutableDictionaryReader>(); File bitMapIndexFile = new File(indexSegmentDir, column + ".bitmap.inv"); ColumnMetadata columnMetadata = segmentMetadata.getColumnMetadataFor(column); int cardinality = columnMetadata.getCardinality(); cardinalityMap.put(column, cardinality); PinotDataBuffer bitMapDataBuffer = PinotDataBuffer.fromFile(bitMapIndexFile, ReadMode.mmap, FileChannel.MapMode.READ_ONLY, "testing"); BitmapInvertedIndexReader bitmapInvertedIndex = new BitmapInvertedIndexReader(bitMapDataBuffer, cardinality); File dictionaryFile = new File(indexSegmentDir + "/" + column + ".dict"); SegmentDirectory segmentDirectory = SegmentDirectory.createFromLocalFS(indexSegment, segmentMetadata, ReadMode.mmap); SegmentDirectory.Reader segmentReader = segmentDirectory.createReader(); ColumnIndexContainer container = ColumnIndexContainer.init(segmentReader, columnMetadata, null); ImmutableDictionaryReader dictionary = container.getDictionary(); dictionaryMap.put(column, dictionary); // System.out.println(column + ":\t" + MemoryUtil.deepMemoryUsageOf(bitmapInvertedIndex)); bitMapIndexMap.put(column, bitmapInvertedIndex); int dictId = dictionary.indexOf("na.us"); ImmutableRoaringBitmap immutable = bitmapInvertedIndex.getImmutable(dictId); Iterator<Integer> iterator = immutable.iterator(); int count = 0; long start = System.currentTimeMillis(); while (iterator.hasNext()) { iterator.next(); count = count + 1; } long end = System.currentTimeMillis(); System.out.println(" matched: " + count + " Time to iterate:" + (end - start)); bitMapDataBuffer.close(); } public static void andSpeed(String indexSegmentDir) { } public static void main(String[] args) throws Exception { File indexDir = new File("/home/kgopalak/pinot_perf/index_dir/scinPricing_OFFLINE"); File[] segmentDirs = indexDir.listFiles(); for (File indexSegmentDir : segmentDirs) { iterationSpeed(indexSegmentDir.getAbsolutePath(), "dimension_geo"); } } public static void benchmarkIntersetionAndUnion(String indexSegmentDir) throws ConfigurationException, IOException, Exception { File[] listFiles = new File(indexSegmentDir).listFiles(); File indexDir = new File(indexSegmentDir); SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(indexDir); Map<String, BitmapInvertedIndexReader> bitMapIndexMap = new HashMap<String, BitmapInvertedIndexReader>(); Map<String, Integer> cardinalityMap = new HashMap<String, Integer>(); Map<String, ImmutableDictionaryReader> dictionaryMap = new HashMap<String, ImmutableDictionaryReader>(); for (File file : listFiles) { if (!file.getName().endsWith("bitmap.inv")) { continue; } String column = file.getName().replaceAll(".bitmap.inv", ""); ColumnMetadata columnMetadata = segmentMetadata.getColumnMetadataFor(column); int cardinality = columnMetadata.getCardinality(); cardinalityMap.put(column, cardinality); System.out.println(column + "\t\t\t" + cardinality + " \t" + columnMetadata.getDataType()); PinotDataBuffer bitmapDataBuffer = PinotDataBuffer.fromFile(file, ReadMode.mmap, FileChannel.MapMode.READ_ONLY, "testing"); BitmapInvertedIndexReader bitmapInvertedIndex = new BitmapInvertedIndexReader(bitmapDataBuffer, cardinality); File dictionaryFile = new File(indexSegmentDir + "/" + column + ".dict"); SegmentDirectory segmentDirectory = SegmentDirectory.createFromLocalFS(indexDir, segmentMetadata, ReadMode.mmap); SegmentDirectory.Reader segmentReader = segmentDirectory.createReader(); ColumnIndexContainer container = ColumnIndexContainer.init(segmentReader, columnMetadata, null); ImmutableDictionaryReader dictionary = container.getDictionary(); if (columnMetadata.getDataType() == DataType.INT) { System.out.println("BitmapPerformanceBenchmark.main()"); assert dictionary instanceof IntDictionary; } dictionaryMap.put(column, dictionary); // System.out.println(column + ":\t" + MemoryUtil.deepMemoryUsageOf(bitmapInvertedIndex)); bitMapIndexMap.put(column, bitmapInvertedIndex); bitmapDataBuffer.close(); } List<String> dimensionNamesList = segmentMetadata.getSchema().getDimensionNames(); Collections.shuffle(dimensionNamesList); int NUM_TEST = 100; final int MAX_DIMENSIONS_PER_DIMENSION = 1; int MAX_DIMENSIONS_IN_WHERE_CLAUSE = 3; Random random = new Random(); for (int numDimensions = 1; numDimensions <= MAX_DIMENSIONS_IN_WHERE_CLAUSE; numDimensions++) { for (int numValuesPerDimension = 1; numValuesPerDimension <= MAX_DIMENSIONS_PER_DIMENSION; numValuesPerDimension++) { int runCount = 0; while (runCount < NUM_TEST) { Collections.shuffle(dimensionNamesList); List<ImmutableRoaringBitmap> bitMaps = new ArrayList<ImmutableRoaringBitmap>(); List<String> columnNameValuePairs = new ArrayList<String>(); for (int i = 0; i < numDimensions; i++) { String columnName = dimensionNamesList.get(i); InvertedIndexReader bitmapInvertedIndex = bitMapIndexMap.get(columnName); for (int j = 0; j < numValuesPerDimension; j++) { int dictId = random.nextInt(cardinalityMap.get(columnName)); String dictValue = dictionaryMap.get(columnName).getStringValue(dictId); columnNameValuePairs.add(columnName + ":" + dictValue); ImmutableRoaringBitmap immutable = bitmapInvertedIndex.getImmutable(dictId); bitMaps.add(immutable); } } System.out.println("START**********************************"); int[] cardinality = new int[bitMaps.size()]; int[] sizes = new int[bitMaps.size()]; for (int i = 0; i < bitMaps.size(); i++) { ImmutableRoaringBitmap immutableRoaringBitmap = bitMaps.get(i); cardinality[i] = immutableRoaringBitmap.getCardinality(); sizes[i] = immutableRoaringBitmap.getSizeInBytes(); } System.out.println("\t#bitmaps:" + bitMaps.size()); System.out.println("\tinput values:" + columnNameValuePairs); System.out.println("\tinput cardinality:" + Arrays.toString(cardinality)); System.out.println("\tinput sizes:" + Arrays.toString(sizes)); and(bitMaps); or(bitMaps); System.out.println("END**********************************"); runCount = runCount + 1; } } } } private static ImmutableRoaringBitmap or(List<ImmutableRoaringBitmap> bitMaps) { if (bitMaps.size() == 1) { return bitMaps.get(0); } MutableRoaringBitmap answer = ImmutableRoaringBitmap.or(bitMaps.get(0), bitMaps.get(1)); long start = System.currentTimeMillis(); for (int i = 2; i < bitMaps.size(); i++) { answer.or(bitMaps.get(i)); } long end = System.currentTimeMillis(); bitMaps.get(0).getCardinality(); System.out.println("OR operation Took " + (end - start)); System.out.println("\toutput cardinality:" + answer.getCardinality()); System.out.println("\toutout sizes:" + answer.getSizeInBytes()); return answer; } private static ImmutableRoaringBitmap and(List<ImmutableRoaringBitmap> bitMaps) { if (bitMaps.size() == 1) { return bitMaps.get(0); } MutableRoaringBitmap answer = ImmutableRoaringBitmap.and(bitMaps.get(0), bitMaps.get(1)); long start = System.currentTimeMillis(); for (int i = 2; i < bitMaps.size(); i++) { answer.and(bitMaps.get(i)); } long end = System.currentTimeMillis(); int[] cardinality = new int[bitMaps.size()]; int[] sizes = new int[bitMaps.size()]; for (int i = 0; i < bitMaps.size(); i++) { ImmutableRoaringBitmap immutableRoaringBitmap = bitMaps.get(i); cardinality[i] = immutableRoaringBitmap.getCardinality(); sizes[i] = immutableRoaringBitmap.getSizeInBytes(); } bitMaps.get(0).getCardinality(); System.out.println("AND operation Took " + (end - start)); System.out.println("\toutput cardinality:" + answer.getCardinality()); System.out.println("\toutout sizes:" + answer.getSizeInBytes()); return answer; } }