Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package gobblin.scheduler; import java.io.File; import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; import java.io.Reader; import java.nio.file.Files; import java.util.Properties; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.Assert; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; import com.google.common.base.Function; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.google.common.util.concurrent.ServiceManager; import gobblin.configuration.ConfigurationKeys; import gobblin.testing.AssertWithBackoff; /** * Unit tests for the job configuration file monitor in {@link gobblin.scheduler.JobScheduler}. * * @author Yinan Li */ @Test(groups = { "gobblin.scheduler" }) public class JobConfigFileMonitorTest { private static final String JOB_CONFIG_FILE_DIR = "gobblin-test/resource/job-conf"; private String jobConfigDir; private ServiceManager serviceManager; private JobScheduler jobScheduler; private File newJobConfigFile; private class GetNumScheduledJobs implements Function<Void, Integer> { @Override public Integer apply(Void input) { return JobConfigFileMonitorTest.this.jobScheduler.getScheduledJobs().size(); } } @BeforeClass public void setUp() throws Exception { this.jobConfigDir = Files .createTempDirectory(String.format("gobblin-test_%s_job-conf", this.getClass().getSimpleName())) .toString(); FileUtils.forceDeleteOnExit(new File(this.jobConfigDir)); FileUtils.copyDirectory(new File(JOB_CONFIG_FILE_DIR), new File(jobConfigDir)); Properties properties = new Properties(); try (Reader schedulerPropsReader = new FileReader("gobblin-test/resource/gobblin.test.properties")) { properties.load(schedulerPropsReader); } properties.setProperty(ConfigurationKeys.JOB_CONFIG_FILE_DIR_KEY, jobConfigDir); properties.setProperty(ConfigurationKeys.JOB_CONFIG_FILE_GENERAL_PATH_KEY, jobConfigDir); properties.setProperty(ConfigurationKeys.JOB_CONFIG_FILE_MONITOR_POLLING_INTERVAL_KEY, "1000"); properties.setProperty(ConfigurationKeys.METRICS_ENABLED_KEY, "false"); SchedulerService quartzService = new SchedulerService(new Properties()); this.jobScheduler = new JobScheduler(properties, quartzService); this.serviceManager = new ServiceManager(Lists.newArrayList(quartzService, this.jobScheduler)); this.serviceManager.startAsync().awaitHealthy(10, TimeUnit.SECONDS); ; } @Test public void testAddNewJobConfigFile() throws Exception { final Logger log = LoggerFactory.getLogger("testAddNewJobConfigFile"); log.info("testAddNewJobConfigFile: start"); AssertWithBackoff assertWithBackoff = AssertWithBackoff.create().logger(log).timeoutMs(15000); assertWithBackoff.assertEquals(new GetNumScheduledJobs(), 3, "3 scheduled jobs"); /* Set a time gap, to let the monitor recognize the "3-file" status as old status, so that new added file can be discovered */ Thread.sleep(1000); // Create a new job configuration file by making a copy of an existing // one and giving a different job name Properties jobProps = new Properties(); jobProps.load(new FileReader(new File(this.jobConfigDir, "GobblinTest1.pull"))); jobProps.setProperty(ConfigurationKeys.JOB_NAME_KEY, "Gobblin-test-new"); this.newJobConfigFile = new File(this.jobConfigDir, "Gobblin-test-new.pull"); jobProps.store(new FileWriter(this.newJobConfigFile), null); assertWithBackoff.assertEquals(new GetNumScheduledJobs(), 4, "4 scheduled jobs"); Set<String> jobNames = Sets.newHashSet(this.jobScheduler.getScheduledJobs()); Set<String> expectedJobNames = ImmutableSet.<String>builder() .add("GobblinTest1", "GobblinTest2", "GobblinTest3", "Gobblin-test-new").build(); Assert.assertEquals(jobNames, expectedJobNames); log.info("testAddNewJobConfigFile: end"); } @Test(dependsOnMethods = { "testAddNewJobConfigFile" }) public void testChangeJobConfigFile() throws Exception { final Logger log = LoggerFactory.getLogger("testChangeJobConfigFile"); log.info("testChangeJobConfigFile: start"); Assert.assertEquals(this.jobScheduler.getScheduledJobs().size(), 4); // Make a change to the new job configuration file Properties jobProps = new Properties(); jobProps.load(new FileReader(this.newJobConfigFile)); jobProps.setProperty(ConfigurationKeys.JOB_COMMIT_POLICY_KEY, "partial"); jobProps.setProperty(ConfigurationKeys.JOB_NAME_KEY, "Gobblin-test-new2"); jobProps.store(new FileWriter(this.newJobConfigFile), null); AssertWithBackoff.create().logger(log).timeoutMs(30000).assertEquals(new GetNumScheduledJobs(), 4, "4 scheduled jobs"); final Set<String> expectedJobNames = ImmutableSet.<String>builder() .add("GobblinTest1", "GobblinTest2", "GobblinTest3", "Gobblin-test-new2").build(); AssertWithBackoff.create().logger(log).timeoutMs(30000).assertEquals(new Function<Void, Set<String>>() { @Override public Set<String> apply(Void input) { return Sets.newHashSet(JobConfigFileMonitorTest.this.jobScheduler.getScheduledJobs()); } }, expectedJobNames, "Job change detected"); log.info("testChangeJobConfigFile: end"); } @Test(dependsOnMethods = { "testChangeJobConfigFile" }) public void testUnscheduleJob() throws Exception { final Logger log = LoggerFactory.getLogger("testUnscheduleJob"); log.info("testUnscheduleJob: start"); Assert.assertEquals(this.jobScheduler.getScheduledJobs().size(), 4); // Disable the new job by setting job.disabled=true Properties jobProps = new Properties(); jobProps.load(new FileReader(this.newJobConfigFile)); jobProps.setProperty(ConfigurationKeys.JOB_DISABLED_KEY, "true"); jobProps.store(new FileWriter(this.newJobConfigFile), null); AssertWithBackoff.create().logger(log).timeoutMs(7500).assertEquals(new GetNumScheduledJobs(), 3, "3 scheduled jobs"); Set<String> jobNames = Sets.newHashSet(this.jobScheduler.getScheduledJobs()); Assert.assertEquals(jobNames.size(), 3); Assert.assertTrue(jobNames.contains("GobblinTest1")); Assert.assertTrue(jobNames.contains("GobblinTest2")); Assert.assertTrue(jobNames.contains("GobblinTest3")); log.info("testUnscheduleJob: end"); } @AfterClass public void tearDown() throws TimeoutException, IOException { if (jobConfigDir != null) { FileUtils.forceDelete(new File(jobConfigDir)); } this.serviceManager.stopAsync().awaitStopped(30, TimeUnit.SECONDS); } }