com.linkedin.pinot.integration.tests.FileBasedSentineTest.java Source code

Introduction

Here is the source code for com.linkedin.pinot.integration.tests.FileBasedSentineTest.java

Source

/**
 * Copyright (C) 2014-2016 LinkedIn Corp. (pinot-core@linkedin.com)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.linkedin.pinot.integration.tests;

import com.linkedin.pinot.controller.helix.ControllerTest;

import com.linkedin.pinot.segments.v1.creator.SegmentTestUtils;
import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit;

import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.math.IntRange;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.json.JSONObject;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;

import com.linkedin.pinot.common.data.FieldSpec;
import com.linkedin.pinot.common.data.FieldSpec.DataType;
import com.linkedin.pinot.common.data.FieldSpec.FieldType;
import com.linkedin.pinot.core.data.readers.FileFormat;
import com.linkedin.pinot.core.indexsegment.generator.SegmentGeneratorConfig;
import com.linkedin.pinot.core.segment.creator.SegmentIndexCreationDriver;
import com.linkedin.pinot.core.segment.creator.impl.SegmentCreationDriverFactory;
import com.linkedin.pinot.tools.data.generator.DataGenerator;
import com.linkedin.pinot.tools.data.generator.DataGeneratorSpec;

/**
 * Sep 14, 2014
 */

public class FileBasedSentineTest extends ControllerTest {
    private static final Logger LOGGER = LoggerFactory.getLogger(FileBasedSentineTest.class);
    private static URL url;
    private static final String AVRO_FILE_PATH = "/tmp/avroFiles";
    FileBasedServerBrokerStarters starter;
    DataGenerator generator;
    private static File avroDataDir = new File(AVRO_FILE_PATH);

    @BeforeClass
    public void setup() throws Exception {
        url = new URL("http://localhost:" + FileBasedServerBrokerStarters.BROKER_CLIENT_PORT + "/query");

        // lets generate data
        final String[] columns = { "dimention1", "dimention2", "dimention3", "dimention4", "metric1",
                "daysSinceEpoch" };
        final Map<String, DataType> dataTypes = new HashMap<String, FieldSpec.DataType>();
        final Map<String, FieldType> fieldTypes = new HashMap<String, FieldType>();

        final Map<String, TimeUnit> timeUnits = new HashMap<String, TimeUnit>();
        final Map<String, Integer> cardinality = new HashMap<String, Integer>();

        // Crate empty range map as the signature of DataGeneratorSpec has changed, and this test does not
        // use metric/time as fieldType.
        final Map<String, IntRange> range = new HashMap<String, IntRange>();

        for (final String col : columns) {
            if (col.equals("dimention1")) {
                dataTypes.put(col, DataType.STRING);
                cardinality.put(col, 1000);
            } else {
                dataTypes.put(col, DataType.INT);
                cardinality.put(col, 1000);
            }
            fieldTypes.put(col, FieldType.DIMENSION);
        }

        if (avroDataDir.exists()) {
            FileUtils.deleteDirectory(avroDataDir);
        }

        final DataGeneratorSpec spec = new DataGeneratorSpec(Arrays.asList(columns), cardinality, range, dataTypes,
                fieldTypes, timeUnits, FileFormat.AVRO, avroDataDir.getAbsolutePath(), true);
        generator = new DataGenerator();
        generator.init(spec);
        generator.generate(100000L, 2);
        // lets make segments now

        final File bootstrapDir = new File(FileBasedServerBrokerStarters.SERVER_BOOTSTRAP_DIR);

        if (bootstrapDir.exists()) {
            FileUtils.deleteDirectory(bootstrapDir);
        }

        bootstrapDir.mkdir();

        int counter = 0;
        for (final File avro : avroDataDir.listFiles()) {
            for (final String table : FileBasedServerBrokerStarters.TABLE_NAMES) {
                final SegmentGeneratorConfig genConfig = SegmentTestUtils
                        .getSegmentGenSpecWithSchemAndProjectedColumns(avro,
                                new File(bootstrapDir, "segment-" + counter), "daysSinceEpoch", TimeUnit.DAYS,
                                table);

                final SegmentIndexCreationDriver driver = SegmentCreationDriverFactory.get(null);
                driver.init(genConfig);
                driver.build();

                counter++;
            }
        }

        // lets start the server and the broker now

        starter = new FileBasedServerBrokerStarters();
        starter.startAll();

        // pick some values from here if you need to use it for running filter queries

        final JSONObject selectionRequestResponse = postQuery("select * from 'table1' limit 100",
                "http://localhost:" + FileBasedServerBrokerStarters.BROKER_CLIENT_PORT);

        System.out.println(selectionRequestResponse.toString(1));
    }

    @AfterClass
    public void tearDown() throws IOException {
        starter.stopAll();

        FileUtils.deleteDirectory(new File(FileBasedServerBrokerStarters.SERVER_BOOTSTRAP_DIR));
        FileUtils.deleteDirectory(avroDataDir);
        FileUtils.deleteDirectory(new File(FileBasedServerBrokerStarters.SERVER_INDEX_DIR));
    }

    @Test
    public void test1() {

    }
}