Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package gobblin.util; import java.io.IOException; import java.util.Arrays; import java.util.List; import java.util.regex.Pattern; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.slf4j.LoggerFactory; import org.testng.Assert; import org.testng.annotations.Test; import gobblin.configuration.ConfigurationKeys; import gobblin.configuration.SourceState; import gobblin.configuration.WorkUnitState; import gobblin.source.workunit.Extract; import gobblin.source.workunit.Extract.TableType; import gobblin.source.workunit.MultiWorkUnit; import gobblin.source.workunit.WorkUnit; /** * Unit tests for {@link JobLauncherUtils}. * * @author Yinan Li */ @Test(groups = { "gobblin.util" }) public class JobLauncherUtilsTest { private static final String JOB_NAME = "foo"; private static final Pattern PATTERN = Pattern.compile("job_" + JOB_NAME + "_\\d+"); private String jobId; @Test public void testNewJobId() { this.jobId = JobLauncherUtils.newJobId(JOB_NAME); Assert.assertTrue(PATTERN.matcher(this.jobId).matches()); } @Test(dependsOnMethods = "testNewJobId") public void testNewTaskId() { Assert.assertEquals(JobLauncherUtils.newTaskId(this.jobId, 0), this.jobId.replace("job", "task") + "_0"); Assert.assertEquals(JobLauncherUtils.newTaskId(this.jobId, 1), this.jobId.replace("job", "task") + "_1"); } @Test(dependsOnMethods = "testNewJobId") public void testNewMultiTaskId() { Assert.assertEquals(JobLauncherUtils.newMultiTaskId(this.jobId, 0), this.jobId.replace("job", "multitask") + "_0"); Assert.assertEquals(JobLauncherUtils.newMultiTaskId(this.jobId, 1), this.jobId.replace("job", "multitask") + "_1"); } @Test public void testFlattenWorkUnits() { List<WorkUnit> workUnitsOnly = Arrays.asList(WorkUnit.createEmpty(), WorkUnit.createEmpty(), WorkUnit.createEmpty()); Assert.assertEquals(JobLauncherUtils.flattenWorkUnits(workUnitsOnly).size(), 3); MultiWorkUnit multiWorkUnit1 = MultiWorkUnit.createEmpty(); multiWorkUnit1.addWorkUnits( Arrays.asList(WorkUnit.createEmpty(), WorkUnit.createEmpty(), WorkUnit.createEmpty())); MultiWorkUnit multiWorkUnit2 = MultiWorkUnit.createEmpty(); multiWorkUnit1.addWorkUnits( Arrays.asList(WorkUnit.createEmpty(), WorkUnit.createEmpty(), WorkUnit.createEmpty())); List<WorkUnit> workUnitsAndMultiWorkUnits = Arrays.asList(WorkUnit.createEmpty(), WorkUnit.createEmpty(), WorkUnit.createEmpty(), multiWorkUnit1, multiWorkUnit2); Assert.assertEquals(JobLauncherUtils.flattenWorkUnits(workUnitsAndMultiWorkUnits).size(), 9); } @Test public void testDeleteStagingData() throws IOException { FileSystem fs = FileSystem.getLocal(new Configuration()); Path rootDir = new Path("gobblin-test/job-launcher-utils-test"); Path writerStagingDir0 = new Path(rootDir, "staging/fork_0"); Path writerStagingDir1 = new Path(rootDir, "staging/fork_1"); Path writerOutputDir0 = new Path(rootDir, "output/fork_0"); Path writerOutputDir1 = new Path(rootDir, "output/fork_1"); String writerPath0 = "test0"; String writerPath1 = "test1"; try { WorkUnitState state = new WorkUnitState(); state.setProp(ConfigurationKeys.FORK_BRANCHES_KEY, "2"); state.setProp( ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_FILE_SYSTEM_URI, 2, 0), ConfigurationKeys.LOCAL_FS_URI); state.setProp( ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_FILE_SYSTEM_URI, 2, 1), ConfigurationKeys.LOCAL_FS_URI); state.setProp(ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_STAGING_DIR, 2, 0), writerStagingDir0.toString()); state.setProp(ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_STAGING_DIR, 2, 1), writerStagingDir1.toString()); state.setProp(ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_OUTPUT_DIR, 2, 0), writerOutputDir0.toString()); state.setProp(ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_OUTPUT_DIR, 2, 1), writerOutputDir1.toString()); state.setProp(ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_FILE_PATH, 2, 0), writerPath0); state.setProp(ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_FILE_PATH, 2, 1), writerPath1); Path writerStagingPath0 = new Path(writerStagingDir0, writerPath0); fs.mkdirs(writerStagingPath0); Path writerStagingPath1 = new Path(writerStagingDir1, writerPath1); fs.mkdirs(writerStagingPath1); Path writerOutputPath0 = new Path(writerOutputDir0, writerPath0); fs.mkdirs(writerOutputPath0); Path writerOutputPath1 = new Path(writerOutputDir1, writerPath1); fs.mkdirs(writerOutputPath1); JobLauncherUtils.cleanTaskStagingData(state, LoggerFactory.getLogger(JobLauncherUtilsTest.class)); Assert.assertFalse(fs.exists(writerStagingPath0)); Assert.assertFalse(fs.exists(writerStagingPath1)); Assert.assertFalse(fs.exists(writerOutputPath0)); Assert.assertFalse(fs.exists(writerOutputPath1)); } finally { fs.delete(rootDir, true); } } @Test public void testDeleteStagingDataWithOutWriterFilePath() throws IOException { FileSystem fs = FileSystem.getLocal(new Configuration()); String branchName0 = "fork_0"; String branchName1 = "fork_1"; String namespace = "gobblin.test"; String tableName = "test-table"; Path rootDir = new Path("gobblin-test/job-launcher-utils-test"); Path writerStagingDir0 = new Path(rootDir, "staging" + Path.SEPARATOR + branchName0); Path writerStagingDir1 = new Path(rootDir, "staging" + Path.SEPARATOR + branchName1); Path writerOutputDir0 = new Path(rootDir, "output" + Path.SEPARATOR + branchName0); Path writerOutputDir1 = new Path(rootDir, "output" + Path.SEPARATOR + branchName1); try { SourceState sourceState = new SourceState(); WorkUnitState state = new WorkUnitState( WorkUnit.create(new Extract(sourceState, TableType.APPEND_ONLY, namespace, tableName))); state.setProp(ConfigurationKeys.FORK_BRANCHES_KEY, "2"); state.setProp(ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.FORK_BRANCH_NAME_KEY, 2, 0), branchName0); state.setProp(ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.FORK_BRANCH_NAME_KEY, 2, 1), branchName1); state.setProp( ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_FILE_SYSTEM_URI, 2, 0), ConfigurationKeys.LOCAL_FS_URI); state.setProp( ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_FILE_SYSTEM_URI, 2, 1), ConfigurationKeys.LOCAL_FS_URI); state.setProp(ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_STAGING_DIR, 2, 0), writerStagingDir0.toString()); state.setProp(ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_STAGING_DIR, 2, 1), writerStagingDir1.toString()); state.setProp(ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_OUTPUT_DIR, 2, 0), writerOutputDir0.toString()); state.setProp(ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_OUTPUT_DIR, 2, 1), writerOutputDir1.toString()); Path writerStagingPath0 = new Path(writerStagingDir0, ForkOperatorUtils.getPathForBranch(state, state.getExtract().getOutputFilePath(), 2, 0)); fs.mkdirs(writerStagingPath0); Path writerStagingPath1 = new Path(writerStagingDir1, ForkOperatorUtils.getPathForBranch(state, state.getExtract().getOutputFilePath(), 2, 1)); fs.mkdirs(writerStagingPath1); Path writerOutputPath0 = new Path(writerOutputDir0, ForkOperatorUtils.getPathForBranch(state, state.getExtract().getOutputFilePath(), 2, 0)); fs.mkdirs(writerOutputPath0); Path writerOutputPath1 = new Path(writerOutputDir1, ForkOperatorUtils.getPathForBranch(state, state.getExtract().getOutputFilePath(), 2, 1)); fs.mkdirs(writerOutputPath1); JobLauncherUtils.cleanTaskStagingData(state, LoggerFactory.getLogger(JobLauncherUtilsTest.class)); Assert.assertFalse(fs.exists(writerStagingPath0)); Assert.assertFalse(fs.exists(writerStagingPath1)); Assert.assertFalse(fs.exists(writerOutputPath0)); Assert.assertFalse(fs.exists(writerOutputPath1)); } finally { fs.delete(rootDir, true); } } }