Java tutorial
/** * Copyright (C) 2014-2016 LinkedIn Corp. (pinot-core@linkedin.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.linkedin.pinot.tools; import static com.linkedin.pinot.tools.Quickstart.printStatus; import java.io.File; import java.io.IOException; import org.apache.commons.io.FileUtils; import org.apache.log4j.Level; import org.json.JSONException; import com.google.common.collect.Lists; import com.linkedin.pinot.common.data.Schema; import com.linkedin.pinot.common.utils.KafkaStarterUtils; import com.linkedin.pinot.common.utils.ZkStarter; import com.linkedin.pinot.core.data.readers.FileFormat; import com.linkedin.pinot.tools.Quickstart.color; import com.linkedin.pinot.tools.admin.command.QuickstartRunner; import com.linkedin.pinot.tools.streams.AirlineDataStream; import kafka.server.KafkaServerStartable; /** * * * */ public class HybridQuickstart { private File _offlineQuickStartDataDir; private File _realtimeQuickStartDataDir; KafkaServerStartable kafkaStarter; private ZkStarter.ZookeeperInstance _zookeeperInstance; private QuickstartTableRequest prepareOfflineTableRequest() throws IOException { _offlineQuickStartDataDir = new File("quickStartData" + System.currentTimeMillis()); if (!_offlineQuickStartDataDir.exists()) { _offlineQuickStartDataDir.mkdir(); } _offlineQuickStartDataDir = new File("quickStartData" + System.currentTimeMillis()); if (!_offlineQuickStartDataDir.exists()) { _offlineQuickStartDataDir.mkdir(); } File schemaFile = new File(_offlineQuickStartDataDir + "/airlineStats.schema"); File dataFile = new File(_offlineQuickStartDataDir + "/airline.avro"); File tableCreationJsonFileName = new File(_offlineQuickStartDataDir + "/airlineStatsOffline_hybrid.json"); FileUtils.copyURLToFile(Quickstart.class.getClassLoader().getResource("sample_data/airlineStats.schema"), schemaFile); FileUtils.copyURLToFile(Quickstart.class.getClassLoader().getResource("sample_data/airline.avro"), dataFile); FileUtils.copyURLToFile( Quickstart.class.getClassLoader().getResource("sample_data/airlineStatsOffline_hybrid.json"), tableCreationJsonFileName); String tableName = "airlineStats"; return new QuickstartTableRequest(tableName, schemaFile, tableCreationJsonFileName, _offlineQuickStartDataDir, FileFormat.AVRO); } private QuickstartTableRequest prepareRealtimeTableRequest() throws IOException { _realtimeQuickStartDataDir = new File("quickStartData" + System.currentTimeMillis()); String quickStartDataDirName = _realtimeQuickStartDataDir.getName(); if (!_realtimeQuickStartDataDir.exists()) { _realtimeQuickStartDataDir.mkdir(); } File schema = new File(quickStartDataDirName + "/airlineStats.schema"); File tableCreate = new File(quickStartDataDirName + "/airlineStatsRealtime_hybrid.json"); FileUtils.copyURLToFile( RealtimeQuickStart.class.getClassLoader().getResource("sample_data/airlineStats.schema"), schema); FileUtils.copyURLToFile(RealtimeQuickStart.class.getClassLoader() .getResource("sample_data/airlineStatsRealtime_hybrid.json"), tableCreate); return new QuickstartTableRequest("airlineStats", schema, tableCreate); } private void startKafka() { _zookeeperInstance = ZkStarter.startLocalZkServer(); kafkaStarter = KafkaStarterUtils.startServer(KafkaStarterUtils.DEFAULT_KAFKA_PORT, KafkaStarterUtils.DEFAULT_BROKER_ID, KafkaStarterUtils.DEFAULT_ZK_STR, KafkaStarterUtils.getDefaultKafkaConfiguration()); KafkaStarterUtils.createTopic("airlineStatsEvents", KafkaStarterUtils.DEFAULT_ZK_STR, 10); } public void execute() throws JSONException, Exception { QuickstartTableRequest offlineRequest = prepareOfflineTableRequest(); QuickstartTableRequest realtimeTableRequest = prepareRealtimeTableRequest(); File tempDir = new File("/tmp/" + System.currentTimeMillis()); tempDir.mkdir(); final QuickstartRunner runner = new QuickstartRunner( Lists.newArrayList(offlineRequest, realtimeTableRequest), 2, 2, 1, tempDir, false); printStatus(color.YELLOW, "***** starting kafka *****"); startKafka(); printStatus(color.YELLOW, "***** starting 2 servers, 2 brokers and 1 controller *****"); runner.startAll(); printStatus(color.YELLOW, "***** creating a server tenant with name airline *****"); runner.createServerTenantWith(1, 1, "airline"); printStatus(color.YELLOW, "***** creating a broker tenant with name airline_broker *****"); runner.createBrokerTenantWith(2, "airline_broker"); printStatus(color.YELLOW, "***** adding airline schema *****"); runner.addSchema(); printStatus(color.YELLOW, "***** adding airline offline and realtime table *****"); runner.addTable(); printStatus(color.YELLOW, "***** adding airline offline segment *****"); runner.buildSegment(); printStatus(color.YELLOW, "***** pushing airline offline segment to the controller *****"); runner.pushSegment(); File dataFile = new File(_offlineQuickStartDataDir + "/airline.avro"); File schemaFile = new File(_offlineQuickStartDataDir + "/airlineStats.schema"); printStatus(color.YELLOW, "***** publishing data to kafka for airline realtime to start consuming event stream *****"); final AirlineDataStream stream = new AirlineDataStream(Schema.fromFile(schemaFile), dataFile); stream.run(); printStatus(color.YELLOW, "***** Pinot Hybrid with hybrid table setup is complete *****"); printStatus(color.YELLOW, "***** 1. Sequence of operations *****"); printStatus(color.YELLOW, "***** 2. Started 1 controller instances where tenant creation is enabled *****"); printStatus(color.YELLOW, "***** 3. Started 2 servers and 2 brokers *****"); printStatus(color.YELLOW, "***** 4. created a server tenant with 1 offline and 1 realtime instance *****"); printStatus(color.YELLOW, "***** 5. Created a broker tenant with 2 instances *****"); printStatus(color.YELLOW, "***** 6. Added a schema *****"); printStatus(color.YELLOW, "***** 7. Created a offline and a realtime table with the tenant names created abover *****"); printStatus(color.YELLOW, "***** 8. Created and published an offline segment *****"); printStatus(color.YELLOW, "***** Also Started publishing a kafka stream for the realtime instance so start consuming *****"); printStatus(color.YELLOW, "***** go to http://localhost:9000/query to run a few queries *****"); Runtime.getRuntime().addShutdownHook(new Thread() { @Override public void run() { try { stream.shutdown(); Thread.sleep(2000); printStatus(color.GREEN, "***** shutting down hybrid quick start *****"); runner.stop(); runner.clean(); FileUtils.deleteDirectory(_offlineQuickStartDataDir); FileUtils.deleteDirectory(_realtimeQuickStartDataDir); KafkaStarterUtils.stopServer(kafkaStarter); ZkStarter.stopLocalZkServer(_zookeeperInstance); } catch (Exception e) { } } }); } public static void main(String[] args) throws Exception { org.apache.log4j.Logger.getRootLogger().setLevel(Level.ERROR); HybridQuickstart st = new HybridQuickstart(); st.execute(); while (true) { } } }