Java tutorial
/** * Copyright 2015 Cloudera Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.kitesdk.data.oozie; import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.List; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.oozie.service.ServiceException; import org.apache.oozie.service.Services; import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; import org.junit.Assume; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import org.kitesdk.data.DatasetDescriptor; import org.kitesdk.data.spi.DefaultConfiguration; import com.google.common.base.Joiner; import com.google.common.io.Files; @RunWith(Parameterized.class) public class TestKiteConfigurationService extends MiniDFSTest { protected static final String NAMESPACE = "ns1"; protected static final String NAME = "provider_test1"; @Parameterized.Parameters public static Collection<Object[]> data() { Object[][] data = new Object[][] { { false }, // default to local FS { true } }; // default to distributed FS return Arrays.asList(data); } // whether this should use the DFS provided by MiniDFSTest protected boolean distributed; protected Configuration conf; protected DatasetDescriptor testDescriptor; protected FileSystem fs; private Configuration startingConf; private String startingOozieHome; private File serviceTempDir; Path kiteConfigPath = null; public TestKiteConfigurationService(boolean distributed) { this.distributed = distributed; } private static Configuration originalKiteConf; @BeforeClass public static void defaultConfiguration() { originalKiteConf = DefaultConfiguration.get(); } @AfterClass public static void restoreConfiguration() { DefaultConfiguration.set(originalKiteConf); } @After public void removeDataPath() throws IOException { // restore configuration DefaultConfiguration.set(startingConf); if (serviceTempDir != null) { FileUtils.deleteDirectory(serviceTempDir); serviceTempDir = null; } if (kiteConfigPath != null) { fs.delete(kiteConfigPath, true); kiteConfigPath = null; } if (Services.get() != null) { Services.get().destroy(); } if (startingOozieHome == null) { System.clearProperty("oozie.home.dir"); } else { System.setProperty("oozie.home.dir", startingOozieHome); startingOozieHome = null; } } @Before public void setUp() throws IOException, URISyntaxException { this.conf = (distributed ? MiniDFSTest.getConfiguration() : new Configuration()); this.fs = FileSystem.get(conf); startingOozieHome = System.getProperty("oozie.home.dir"); serviceTempDir = Files.createTempDir(); } @Test public void noKiteConfiguration() throws ServiceException, FileNotFoundException, IOException { Assume.assumeTrue(!distributed); kiteConfigPath = new Path("target/kite-conf-serv"); setupKiteConfigurationService(kiteConfigPath, true, false, false); Assert.assertNull(Services.get().get(KiteConfigurationService.class).getKiteConf()); } @Test public void singleKiteConfiguration() throws ServiceException, IOException { Assume.assumeTrue(!distributed); kiteConfigPath = new Path("target/kite-conf-serv"); setupKiteConfigurationService(kiteConfigPath, true, true, false); Configuration serviceKiteConf = Services.get().get(KiteConfigurationService.class).getKiteConf(); Assert.assertNotNull(serviceKiteConf); Assert.assertEquals("test.value", serviceKiteConf.get("test.property")); } @Test public void multipleKiteConfigurations() throws ServiceException, IOException { Assume.assumeTrue(!distributed); kiteConfigPath = new Path("target/kite-conf-serv"); setupKiteConfigurationService(kiteConfigPath, true, true, true); Configuration serviceKiteConf = Services.get().get(KiteConfigurationService.class).getKiteConf(); Assert.assertNotNull(serviceKiteConf); Assert.assertEquals("test.value", serviceKiteConf.get("test.property")); Assert.assertEquals("second.value", serviceKiteConf.get("second.property")); } @Test public void hdfsKiteConfigurations() throws ServiceException, IOException { Assume.assumeTrue(distributed); kiteConfigPath = new Path("hdfs:///testing/kite-conf-serv"); setupKiteConfigurationService(kiteConfigPath, true, true, true); Configuration serviceKiteConf = Services.get().get(KiteConfigurationService.class).getKiteConf(); Assert.assertNotNull(serviceKiteConf); Assert.assertEquals("test.value", serviceKiteConf.get("test.property")); Assert.assertEquals("second.value", serviceKiteConf.get("second.property")); } private void setupKiteConfigurationService(Path kiteConfigLocation, boolean loadKiteService, boolean loadPrimaryConfig, boolean loadSecondaryConfig) throws IOException, ServiceException { File confDir = new File(serviceTempDir, "conf"); File hadoopConfDir = new File(confDir, "hadoop-conf"); File actionConfDir = new File(confDir, "action-conf"); confDir.mkdirs(); hadoopConfDir.mkdirs(); actionConfDir.mkdirs(); Path kiteConfDir = new Path(kiteConfigLocation, "kite-conf"); fs.mkdirs(kiteConfDir); // these may need to be forced to local FS File oozieSiteTarget = new File(confDir, "oozie-site.xml"); File hadoopConfTarget = new File(hadoopConfDir, "hadoop-site.xml"); // we'll want tests for this with both local and hdfs files Path primaryConfTarget = new Path(kiteConfDir, "primary-site.xml"); Path secondaryConfTarget = new Path(kiteConfDir, "secondary-site.xml"); Configuration oozieSiteConf = new Configuration(false); List<String> configs = new ArrayList<String>(); if (loadPrimaryConfig) { configs.add(primaryConfTarget.toString()); } if (loadSecondaryConfig) { configs.add(secondaryConfTarget.toString()); } if (!configs.isEmpty()) { oozieSiteConf.set("oozie.service.KiteConfigurationService.kite.configuration", Joiner.on(",").join(configs)); } oozieSiteConf.set("oozie.services", "org.apache.oozie.service.HadoopAccessorService"); FileOutputStream oozieStream = new FileOutputStream(oozieSiteTarget); oozieSiteConf.writeXml(oozieStream); oozieStream.close(); FileOutputStream hadoopStream = new FileOutputStream(hadoopConfTarget); conf.writeXml(hadoopStream); hadoopStream.close(); if (loadPrimaryConfig) { Configuration primaryConf = new Configuration(false); primaryConf.set("test.property", "test.value"); FSDataOutputStream primaryStream = fs.create(primaryConfTarget); primaryConf.writeXml(primaryStream); primaryStream.close(); } if (loadSecondaryConfig) { Configuration secondaryConf = new Configuration(false); secondaryConf.set("second.property", "second.value"); FSDataOutputStream secondaryStream = fs.create(secondaryConfTarget); secondaryConf.writeXml(secondaryStream); secondaryStream.close(); } // set to the temp directory System.setProperty("oozie.home.dir", serviceTempDir.getAbsolutePath()); Services services = new Services(); services.init(); if (loadKiteService) { services.setService(KiteConfigurationService.class); } } }