Java tutorial
/** * Copyright (C) 2014-2016 LinkedIn Corp. (pinot-core@linkedin.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.linkedin.pinot.integration.tests; import com.linkedin.pinot.client.Connection; import com.linkedin.pinot.client.ConnectionFactory; import com.linkedin.pinot.tools.admin.PinotAdministrator; import com.linkedin.pinot.util.TestUtils; import java.io.File; import java.io.IOException; import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import jnr.constants.platform.Signal; import jnr.posix.POSIXFactory; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.Test; /** * Monkeys and chaos. */ public class ChaosMonkeyIntegrationTest { private static final Logger LOGGER = LoggerFactory.getLogger(ChaosMonkeyIntegrationTest.class); private static final String TOTAL_RECORD_COUNT = "1000"; private static final String SEGMENT_COUNT = "25"; private final List<Process> _processes = new ArrayList<>(); private final String AVRO_DIR = "/tmp/temp-avro-" + getClass().getName(); private final String SEGMENT_DIR = "/tmp/temp-segment-" + getClass().getName(); private Process runAdministratorCommand(String[] args) { String classpath = System.getProperty("java.class.path"); List<String> completeArgs = new ArrayList<>(); completeArgs.add("java"); completeArgs.add("-Xms4G"); completeArgs.add("-Xmx4G"); completeArgs.add("-cp"); completeArgs.add(classpath); completeArgs.add(PinotAdministrator.class.getName()); completeArgs.addAll(Arrays.asList(args)); try { Process process = new ProcessBuilder(completeArgs).redirectError(ProcessBuilder.Redirect.INHERIT) .redirectOutput(ProcessBuilder.Redirect.INHERIT).start(); synchronized (_processes) { _processes.add(process); } return process; } catch (IOException e) { throw new RuntimeException(e); } } private void sendSignalToProcess(Process process, Signal signal) { int processPid = getProcessPid(process); if (processPid != -1) { LOGGER.info("Sending signal {} to process {}", signal.intValue(), processPid); POSIXFactory.getNativePOSIX().kill(processPid, signal.intValue()); } } private int getProcessPid(Process process) { Class<? extends Process> clazz = process.getClass(); try { Field field = clazz.getDeclaredField("pid"); field.setAccessible(true); return field.getInt(process); } catch (NoSuchFieldException e) { return -1; } catch (IllegalAccessException e) { return -1; } } private Process startZookeeper() { return runAdministratorCommand(new String[] { "StartZookeeper", "-zkPort", "2191" }); } private Process startController() { return runAdministratorCommand(new String[] { "StartController", "-zkAddress", "localhost:2191", "-controllerPort", "39000", "-dataDir", "/tmp/ChaosMonkeyClusterController" }); } private Process startBroker() { return runAdministratorCommand( new String[] { "StartBroker", "-brokerPort", "8099", "-zkAddress", "localhost:2191" }); } private Process startServer() { return runAdministratorCommand( new String[] { "StartServer", "-serverPort", "8098", "-zkAddress", "localhost:2191", "-dataDir", "/tmp/ChaosMonkeyCluster/data", "-segmentDir", "/tmp/ChaosMonkeyCluster/segments" }); } private void generateData() throws InterruptedException { String schemaFile = TestUtils.getFileFromResourceUrl( ChaosMonkeyIntegrationTest.class.getClassLoader().getResource("chaos-monkey-schema.json")); String schemaAnnotationsFile = TestUtils.getFileFromResourceUrl(ChaosMonkeyIntegrationTest.class .getClassLoader().getResource("chaos-monkey-schema-annotations.json")); runAdministratorCommand(new String[] { "GenerateData", "-numRecords", TOTAL_RECORD_COUNT, "-numFiles", SEGMENT_COUNT, "-schemaFile", schemaFile, "-schemaAnnotationFile", schemaAnnotationsFile, "-overwrite", "-outDir", AVRO_DIR }).waitFor(); } private void createTable() throws InterruptedException { String createTableFile = TestUtils.getFileFromResourceUrl( ChaosMonkeyIntegrationTest.class.getClassLoader().getResource("chaos-monkey-create-table.json")); runAdministratorCommand( new String[] { "AddTable", "-controllerPort", "39000", "-filePath", createTableFile, "-exec" }) .waitFor(); } private void convertData() throws InterruptedException { String schemaFile = TestUtils.getFileFromResourceUrl( ChaosMonkeyIntegrationTest.class.getClassLoader().getResource("chaos-monkey-schema.json")); runAdministratorCommand(new String[] { "CreateSegment", "-schemaFile", schemaFile, "-dataDir", AVRO_DIR, "-tableName", "myTable", "-segmentName", "my_table", "-outDir", SEGMENT_DIR, "-overwrite" }) .waitFor(); } private void uploadData() throws InterruptedException { runAdministratorCommand( new String[] { "UploadSegment", "-controllerPort", "39000", "-segmentDir", SEGMENT_DIR }).waitFor(); } private int countRecords() { Connection connection = ConnectionFactory.fromHostList("localhost:8099"); return connection.execute("select count(*) from myTable").getResultSet(0).getInt(0); } @Test(enabled = false) public void testShortZookeeperFreeze() throws Exception { testFreezeZookeeper(10000L); } @Test(enabled = false) public void testLongZookeeperFreeze() throws Exception { testFreezeZookeeper(60000L); } public void testFreezeZookeeper(long freezeLength) throws Exception { Process zookeeper = startZookeeper(); Thread.sleep(1000L); startController(); Thread.sleep(3000L); startServer(); startBroker(); Thread.sleep(3000L); createTable(); generateData(); convertData(); uploadData(); Thread.sleep(5000L); long timeInTwoMinutes = System.currentTimeMillis() + 120000L; int currentRecordCount = countRecords(); int expectedRecordCount = Integer.parseInt(TOTAL_RECORD_COUNT); while (currentRecordCount != expectedRecordCount && System.currentTimeMillis() < timeInTwoMinutes) { Thread.sleep(1000L); currentRecordCount = countRecords(); } Assert.assertEquals(currentRecordCount, expectedRecordCount, "All segments did not load within 120 seconds"); sendSignalToProcess(zookeeper, Signal.SIGSTOP); Thread.sleep(freezeLength); sendSignalToProcess(zookeeper, Signal.SIGCONT); Thread.sleep(5000L); timeInTwoMinutes = System.currentTimeMillis() + 120000L; currentRecordCount = countRecords(); while (currentRecordCount != expectedRecordCount && System.currentTimeMillis() < timeInTwoMinutes) { Thread.sleep(1000L); currentRecordCount = countRecords(); } Assert.assertEquals(currentRecordCount, expectedRecordCount, "Record count still inconsistent 120 seconds after zookeeper restart"); } @AfterMethod public void tearDown() { for (Process process : _processes) { process.destroy(); } FileUtils.deleteQuietly(new File(AVRO_DIR)); FileUtils.deleteQuietly(new File(SEGMENT_DIR)); } }