com.linkedin.pinot.perf.BitmapPerformanceBenchmark.java Source code

Java tutorial

Introduction

Here is the source code for com.linkedin.pinot.perf.BitmapPerformanceBenchmark.java

Source

/**
 * Copyright (C) 2014-2016 LinkedIn Corp. (pinot-core@linkedin.com)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/**
 * Copyright (C) 2014-2015 LinkedIn Corp. (pinot-core@linkedin.com)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.linkedin.pinot.perf;

import com.linkedin.pinot.core.segment.memory.PinotDataBuffer;
import com.linkedin.pinot.core.segment.store.SegmentDirectory;
import java.io.File;
import java.io.IOException;
import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Random;

import org.apache.commons.configuration.ConfigurationException;
import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
import org.roaringbitmap.buffer.MutableRoaringBitmap;

import com.linkedin.pinot.common.data.FieldSpec.DataType;
import com.linkedin.pinot.common.segment.ReadMode;
import com.linkedin.pinot.core.segment.index.readers.BitmapInvertedIndexReader;
import com.linkedin.pinot.core.segment.index.ColumnMetadata;
import com.linkedin.pinot.core.segment.index.readers.InvertedIndexReader;
import com.linkedin.pinot.core.segment.index.SegmentMetadataImpl;
import com.linkedin.pinot.core.segment.index.column.ColumnIndexContainer;
import com.linkedin.pinot.core.segment.index.readers.ImmutableDictionaryReader;
import com.linkedin.pinot.core.segment.index.readers.IntDictionary;

public class BitmapPerformanceBenchmark {

    public static void iterationSpeed(String indexSegmentDir, String column) throws Exception {
        File indexSegment = new File(indexSegmentDir);
        SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(indexSegment);
        Map<String, BitmapInvertedIndexReader> bitMapIndexMap = new HashMap<String, BitmapInvertedIndexReader>();
        Map<String, Integer> cardinalityMap = new HashMap<String, Integer>();
        Map<String, ImmutableDictionaryReader> dictionaryMap = new HashMap<String, ImmutableDictionaryReader>();
        File bitMapIndexFile = new File(indexSegmentDir, column + ".bitmap.inv");
        ColumnMetadata columnMetadata = segmentMetadata.getColumnMetadataFor(column);
        int cardinality = columnMetadata.getCardinality();
        cardinalityMap.put(column, cardinality);

        PinotDataBuffer bitMapDataBuffer = PinotDataBuffer.fromFile(bitMapIndexFile, ReadMode.mmap,
                FileChannel.MapMode.READ_ONLY, "testing");
        BitmapInvertedIndexReader bitmapInvertedIndex = new BitmapInvertedIndexReader(bitMapDataBuffer,
                cardinality);
        File dictionaryFile = new File(indexSegmentDir + "/" + column + ".dict");
        SegmentDirectory segmentDirectory = SegmentDirectory.createFromLocalFS(indexSegment, segmentMetadata,
                ReadMode.mmap);
        SegmentDirectory.Reader segmentReader = segmentDirectory.createReader();
        ColumnIndexContainer container = ColumnIndexContainer.init(segmentReader, columnMetadata, null);
        ImmutableDictionaryReader dictionary = container.getDictionary();
        dictionaryMap.put(column, dictionary);
        // System.out.println(column + ":\t" + MemoryUtil.deepMemoryUsageOf(bitmapInvertedIndex));
        bitMapIndexMap.put(column, bitmapInvertedIndex);
        int dictId = dictionary.indexOf("na.us");
        ImmutableRoaringBitmap immutable = bitmapInvertedIndex.getImmutable(dictId);
        Iterator<Integer> iterator = immutable.iterator();
        int count = 0;
        long start = System.currentTimeMillis();
        while (iterator.hasNext()) {
            iterator.next();
            count = count + 1;
        }
        long end = System.currentTimeMillis();
        System.out.println(" matched: " + count + " Time to iterate:" + (end - start));
        bitMapDataBuffer.close();
    }

    public static void andSpeed(String indexSegmentDir) {

    }

    public static void main(String[] args) throws Exception {
        File indexDir = new File("/home/kgopalak/pinot_perf/index_dir/scinPricing_OFFLINE");
        File[] segmentDirs = indexDir.listFiles();
        for (File indexSegmentDir : segmentDirs) {
            iterationSpeed(indexSegmentDir.getAbsolutePath(), "dimension_geo");
        }
    }

    public static void benchmarkIntersetionAndUnion(String indexSegmentDir)
            throws ConfigurationException, IOException, Exception {
        File[] listFiles = new File(indexSegmentDir).listFiles();
        File indexDir = new File(indexSegmentDir);
        SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(indexDir);
        Map<String, BitmapInvertedIndexReader> bitMapIndexMap = new HashMap<String, BitmapInvertedIndexReader>();
        Map<String, Integer> cardinalityMap = new HashMap<String, Integer>();
        Map<String, ImmutableDictionaryReader> dictionaryMap = new HashMap<String, ImmutableDictionaryReader>();
        for (File file : listFiles) {
            if (!file.getName().endsWith("bitmap.inv")) {
                continue;
            }
            String column = file.getName().replaceAll(".bitmap.inv", "");
            ColumnMetadata columnMetadata = segmentMetadata.getColumnMetadataFor(column);
            int cardinality = columnMetadata.getCardinality();
            cardinalityMap.put(column, cardinality);
            System.out.println(column + "\t\t\t" + cardinality + "  \t" + columnMetadata.getDataType());
            PinotDataBuffer bitmapDataBuffer = PinotDataBuffer.fromFile(file, ReadMode.mmap,
                    FileChannel.MapMode.READ_ONLY, "testing");
            BitmapInvertedIndexReader bitmapInvertedIndex = new BitmapInvertedIndexReader(bitmapDataBuffer,
                    cardinality);
            File dictionaryFile = new File(indexSegmentDir + "/" + column + ".dict");
            SegmentDirectory segmentDirectory = SegmentDirectory.createFromLocalFS(indexDir, segmentMetadata,
                    ReadMode.mmap);
            SegmentDirectory.Reader segmentReader = segmentDirectory.createReader();
            ColumnIndexContainer container = ColumnIndexContainer.init(segmentReader, columnMetadata, null);
            ImmutableDictionaryReader dictionary = container.getDictionary();
            if (columnMetadata.getDataType() == DataType.INT) {
                System.out.println("BitmapPerformanceBenchmark.main()");
                assert dictionary instanceof IntDictionary;
            }
            dictionaryMap.put(column, dictionary);
            // System.out.println(column + ":\t" + MemoryUtil.deepMemoryUsageOf(bitmapInvertedIndex));
            bitMapIndexMap.put(column, bitmapInvertedIndex);
            bitmapDataBuffer.close();
        }

        List<String> dimensionNamesList = segmentMetadata.getSchema().getDimensionNames();

        Collections.shuffle(dimensionNamesList);
        int NUM_TEST = 100;
        final int MAX_DIMENSIONS_PER_DIMENSION = 1;
        int MAX_DIMENSIONS_IN_WHERE_CLAUSE = 3;
        Random random = new Random();
        for (int numDimensions = 1; numDimensions <= MAX_DIMENSIONS_IN_WHERE_CLAUSE; numDimensions++) {
            for (int numValuesPerDimension = 1; numValuesPerDimension <= MAX_DIMENSIONS_PER_DIMENSION; numValuesPerDimension++) {
                int runCount = 0;
                while (runCount < NUM_TEST) {
                    Collections.shuffle(dimensionNamesList);
                    List<ImmutableRoaringBitmap> bitMaps = new ArrayList<ImmutableRoaringBitmap>();
                    List<String> columnNameValuePairs = new ArrayList<String>();
                    for (int i = 0; i < numDimensions; i++) {
                        String columnName = dimensionNamesList.get(i);
                        InvertedIndexReader bitmapInvertedIndex = bitMapIndexMap.get(columnName);
                        for (int j = 0; j < numValuesPerDimension; j++) {
                            int dictId = random.nextInt(cardinalityMap.get(columnName));
                            String dictValue = dictionaryMap.get(columnName).getStringValue(dictId);
                            columnNameValuePairs.add(columnName + ":" + dictValue);
                            ImmutableRoaringBitmap immutable = bitmapInvertedIndex.getImmutable(dictId);
                            bitMaps.add(immutable);

                        }
                    }
                    System.out.println("START**********************************");
                    int[] cardinality = new int[bitMaps.size()];
                    int[] sizes = new int[bitMaps.size()];
                    for (int i = 0; i < bitMaps.size(); i++) {
                        ImmutableRoaringBitmap immutableRoaringBitmap = bitMaps.get(i);
                        cardinality[i] = immutableRoaringBitmap.getCardinality();
                        sizes[i] = immutableRoaringBitmap.getSizeInBytes();
                    }
                    System.out.println("\t#bitmaps:" + bitMaps.size());
                    System.out.println("\tinput values:" + columnNameValuePairs);

                    System.out.println("\tinput cardinality:" + Arrays.toString(cardinality));
                    System.out.println("\tinput sizes:" + Arrays.toString(sizes));

                    and(bitMaps);
                    or(bitMaps);
                    System.out.println("END**********************************");

                    runCount = runCount + 1;
                }
            }
        }
    }

    private static ImmutableRoaringBitmap or(List<ImmutableRoaringBitmap> bitMaps) {
        if (bitMaps.size() == 1) {
            return bitMaps.get(0);
        }
        MutableRoaringBitmap answer = ImmutableRoaringBitmap.or(bitMaps.get(0), bitMaps.get(1));
        long start = System.currentTimeMillis();
        for (int i = 2; i < bitMaps.size(); i++) {
            answer.or(bitMaps.get(i));
        }

        long end = System.currentTimeMillis();

        bitMaps.get(0).getCardinality();
        System.out.println("OR operation Took " + (end - start));
        System.out.println("\toutput cardinality:" + answer.getCardinality());
        System.out.println("\toutout sizes:" + answer.getSizeInBytes());
        return answer;

    }

    private static ImmutableRoaringBitmap and(List<ImmutableRoaringBitmap> bitMaps) {
        if (bitMaps.size() == 1) {
            return bitMaps.get(0);
        }
        MutableRoaringBitmap answer = ImmutableRoaringBitmap.and(bitMaps.get(0), bitMaps.get(1));
        long start = System.currentTimeMillis();
        for (int i = 2; i < bitMaps.size(); i++) {
            answer.and(bitMaps.get(i));
        }

        long end = System.currentTimeMillis();
        int[] cardinality = new int[bitMaps.size()];
        int[] sizes = new int[bitMaps.size()];
        for (int i = 0; i < bitMaps.size(); i++) {
            ImmutableRoaringBitmap immutableRoaringBitmap = bitMaps.get(i);
            cardinality[i] = immutableRoaringBitmap.getCardinality();
            sizes[i] = immutableRoaringBitmap.getSizeInBytes();
        }
        bitMaps.get(0).getCardinality();
        System.out.println("AND operation Took " + (end - start));
        System.out.println("\toutput cardinality:" + answer.getCardinality());
        System.out.println("\toutout sizes:" + answer.getSizeInBytes());
        return answer;

    }
}