Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package gobblin.kafka.writer; import java.io.File; import java.io.FileReader; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.lang.management.ManagementFactory; import java.util.Properties; import org.apache.commons.io.FileUtils; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; import com.google.common.io.Closer; import org.testng.Assert; import kafka.consumer.ConsumerIterator; import lombok.extern.slf4j.Slf4j; import gobblin.kafka.KafkaTestBase; import gobblin.runtime.JobLauncher; import gobblin.runtime.JobLauncherFactory; /** * Tests that set up a complete standalone Gobblin pipeline along with a Kafka suite */ @Slf4j public class Kafka08DataWriterIntegrationTest { private static final String JOB_PROPS_DIR = "gobblin-modules/gobblin-kafka-08/resource/job-props/"; private static final String TEST_LAUNCHER_PROPERTIES_FILE = JOB_PROPS_DIR + "testKafkaIngest.properties"; private static final String TEST_INGEST_PULL_FILE = JOB_PROPS_DIR + "testKafkaIngest.pull"; private Properties gobblinProps; private Properties jobProps; private KafkaTestBase kafkaTestHelper; private static final String TOPIC = Kafka08DataWriterIntegrationTest.class.getName(); @BeforeClass public void setup() throws Exception { kafkaTestHelper = new KafkaTestBase(); this.gobblinProps = new Properties(); gobblinProps.load(new FileReader(TEST_LAUNCHER_PROPERTIES_FILE)); this.jobProps = new Properties(); jobProps.load(new FileReader(TEST_INGEST_PULL_FILE)); replaceProperties(gobblinProps, "{$topic}", TOPIC); replaceProperties(gobblinProps, "{$kafkaPort}", "" + kafkaTestHelper.getKafkaServerPort()); replaceProperties(jobProps, "{$topic}", TOPIC); replaceProperties(jobProps, "{$kafkaPort}", "" + kafkaTestHelper.getKafkaServerPort()); kafkaTestHelper.startServers(); } private void replaceProperties(Properties props, String searchString, String replacementString) { for (String key : props.stringPropertyNames()) { String value = props.getProperty(key); if (value.contains(searchString)) { String replacedValue = value.replace(searchString, replacementString); props.setProperty(key, replacedValue); } } } @Test public void testErrors() throws Exception { log.warn("Process id = " + ManagementFactory.getRuntimeMXBean().getName()); int numRecordsPerExtract = 5; int numParallel = 2; int errorEvery = 2000; int totalRecords = numRecordsPerExtract * numParallel; int totalSuccessful = totalRecords / errorEvery + totalRecords % errorEvery; { Closer closer = Closer.create(); try { kafkaTestHelper.provisionTopic(TOPIC); jobProps.setProperty("source.numRecordsPerExtract", "" + numRecordsPerExtract); jobProps.setProperty("source.numParallelism", "" + numParallel); jobProps.setProperty("writer.kafka.producerConfig.flaky.errorType", "regex"); // all records from partition 0 will be dropped. jobProps.setProperty("writer.kafka.producerConfig.flaky.regexPattern", ":index:0.*"); jobProps.setProperty("job.commit.policy", "partial"); jobProps.setProperty("publish.at.job.level", "false"); totalSuccessful = 5; // number of records in partition 1 JobLauncher jobLauncher = closer .register(JobLauncherFactory.newJobLauncher(gobblinProps, jobProps)); jobLauncher.launchJob(null); } catch (Exception e) { log.error("Failed to run job with exception ", e); Assert.fail("Should not throw exception on running the job"); } finally { closer.close(); } // test records written testRecordsWritten(totalSuccessful, TOPIC); } boolean trySecond = true; if (trySecond) { Closer closer = Closer.create(); try { jobProps.setProperty("source.numRecordsPerExtract", "" + numRecordsPerExtract); jobProps.setProperty("source.numParallelism", "" + numParallel); jobProps.setProperty("writer.kafka.producerConfig.flaky.errorType", "nth"); jobProps.setProperty("writer.kafka.producerConfig.flaky.errorEvery", "" + errorEvery); JobLauncher jobLauncher = closer .register(JobLauncherFactory.newJobLauncher(gobblinProps, jobProps)); jobLauncher.launchJob(null); totalSuccessful = totalRecords / errorEvery + totalRecords % errorEvery; } catch (Exception e) { log.error("Failed to run job with exception ", e); Assert.fail("Should not throw exception on running the job"); } finally { closer.close(); } } // test records written testRecordsWritten(totalSuccessful, TOPIC); } private void testRecordsWritten(int totalSuccessful, String topic) throws UnsupportedEncodingException { final ConsumerIterator<byte[], byte[]> iterator = kafkaTestHelper.getIteratorForTopic(topic); for (int i = 0; i < totalSuccessful; ++i) { String message = new String(iterator.next().message(), "UTF-8"); log.debug(String.format("%d of %d: Message consumed: %s", (i + 1), totalSuccessful, message)); } } @AfterClass public void stopServers() throws IOException { try { kafkaTestHelper.stopClients(); } finally { kafkaTestHelper.stopServers(); } } @AfterClass @BeforeClass public void cleanup() throws Exception { File file = new File("gobblin-kafka/testOutput"); FileUtils.deleteDirectory(file); } }