Java tutorial
/******************************************************************************* * * Pentaho Big Data * * Copyright (C) 2002-2017 by Hitachi Vantara : http://www.pentaho.com * ******************************************************************************* * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * ******************************************************************************/ package org.pentaho.hadoop.shim; import org.apache.commons.vfs2.AllFileSelector; import org.apache.commons.vfs2.FileObject; import org.apache.commons.vfs2.FileSystemException; import org.apache.commons.vfs2.FileType; import org.apache.commons.vfs2.VFS; import org.apache.commons.vfs2.impl.DefaultFileSystemManager; import org.jboss.shrinkwrap.api.ShrinkWrap; import org.jboss.shrinkwrap.api.exporter.ZipExporter; import org.jboss.shrinkwrap.api.spec.JavaArchive; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; import org.pentaho.hadoop.shim.api.ShimProperties; import org.pentaho.hadoop.shim.spi.HadoopShim; import org.pentaho.hadoop.shim.spi.MockHadoopShim; import java.io.IOException; import java.io.InputStream; import java.lang.reflect.Field; import java.net.URL; import java.util.Arrays; import java.util.List; import java.util.Properties; import static org.junit.Assert.*; public class HadoopConfigurationLocatorTest { private static String HADOOP_CONFIGURATIONS_PATH = System.getProperty("java.io.tmpdir") + "/hadoop-configurations"; private static FileObject configFile; @BeforeClass public static void setup() throws Exception { // Create a test hadoop configuration "a" FileObject ramRoot = VFS.getManager().resolveFile(HADOOP_CONFIGURATIONS_PATH); FileObject aConfigFolder = ramRoot.resolveFile("a"); if (aConfigFolder.exists()) { aConfigFolder.delete(new AllFileSelector()); } aConfigFolder.createFolder(); assertEquals(FileType.FOLDER, aConfigFolder.getType()); // Create the properties file for the configuration as hadoop-configurations/a/config.properties configFile = aConfigFolder.resolveFile("config.properties"); Properties p = new Properties(); p.setProperty("name", "Test Configuration A"); p.setProperty("classpath", ""); p.setProperty("ignore.classes", ""); p.setProperty("library.path", ""); p.setProperty("required.classes", HadoopConfigurationLocatorTest.class.getName()); p.store(configFile.getContent().getOutputStream(), "Test Configuration A"); configFile.close(); // Create the implementation jar FileObject implJar = aConfigFolder.resolveFile("a-config.jar"); implJar.createFile(); // Use ShrinkWrap to create the jar and write it out to VFS JavaArchive archive = ShrinkWrap.create(JavaArchive.class, "a-configuration.jar") .addAsServiceProvider(HadoopShim.class, MockHadoopShim.class).addClass(MockHadoopShim.class); archive.as(ZipExporter.class).exportTo(implJar.getContent().getOutputStream()); } @Test(expected = ConfigurationException.class) public void init_invalidDirectory() throws FileSystemException, ConfigurationException { HadoopConfigurationLocator locator = new HadoopConfigurationLocator(); locator.init(VFS.getManager().resolveFile("ram://bogus-path"), new MockActiveHadoopConfigurationLocator(), new DefaultFileSystemManager()); } @Test(expected = NullPointerException.class) public void init_null_basedir() throws FileSystemException, ConfigurationException { HadoopConfigurationLocator locator = new HadoopConfigurationLocator(); locator.init(null, new MockActiveHadoopConfigurationLocator(), new DefaultFileSystemManager()); } @Test(expected = NullPointerException.class) public void init_null_activeLocator() throws FileSystemException, ConfigurationException { HadoopConfigurationLocator locator = new HadoopConfigurationLocator(); locator.init(VFS.getManager().resolveFile(HADOOP_CONFIGURATIONS_PATH), null, new DefaultFileSystemManager()); } @Test(expected = NullPointerException.class) public void init_null_fsm() throws FileSystemException, ConfigurationException { HadoopConfigurationLocator locator = new HadoopConfigurationLocator(); locator.init(VFS.getManager().resolveFile(HADOOP_CONFIGURATIONS_PATH), new MockActiveHadoopConfigurationLocator(), null); } @Test public void init_MissingRequiredClasses() throws IOException { Properties properties = new Properties(); InputStream inputStream = configFile.getContent().getInputStream(); properties.load(inputStream); inputStream.close(); properties.setProperty("required.classes", "this.class.does.not.Exist"); properties.store(configFile.getContent().getOutputStream(), "Test Configuration A"); configFile.close(); HadoopConfigurationLocator locator = new HadoopConfigurationLocator(); try { locator.init(VFS.getManager().resolveFile(HADOOP_CONFIGURATIONS_PATH), new MockActiveHadoopConfigurationLocator("a"), new DefaultFileSystemManager()); Assert.fail("Should have got exception "); } catch (ConfigurationException e) { assertEquals( "Unable to load class this.class.does.not.Exist that is required to start the Test Configuration A Hadoop Shim", e.getCause().getMessage()); } finally { properties.setProperty("required.classes", HadoopConfigurationLocator.class.getName()); properties.store(configFile.getContent().getOutputStream(), "Test Configuration A"); configFile.close(); } } public void init() throws FileSystemException, ConfigurationException { HadoopConfigurationLocator locator = new HadoopConfigurationLocator(); locator.init(VFS.getManager().resolveFile(HADOOP_CONFIGURATIONS_PATH), new MockActiveHadoopConfigurationLocator("a"), new DefaultFileSystemManager()); assertEquals(1, locator.getConfigurations().size()); assertEquals(VFS.getManager().resolveFile(HADOOP_CONFIGURATIONS_PATH).resolveFile("a"), locator.getConfiguration("a").getLocation()); } @Test public void hasConfiguration() throws Exception { HadoopConfigurationLocator locator = new HadoopConfigurationLocator(); locator.init(VFS.getManager().resolveFile(HADOOP_CONFIGURATIONS_PATH), new MockActiveHadoopConfigurationLocator("a"), new DefaultFileSystemManager()); assertTrue(locator.hasConfiguration("a")); } @Test(expected = RuntimeException.class) public void hasConfiguration_not_intialized() { HadoopConfigurationLocator locator = new HadoopConfigurationLocator(); locator.hasConfiguration(null); } @Test public void getConfiguration() throws Exception { HadoopConfigurationLocator locator = new HadoopConfigurationLocator(); locator.init(VFS.getManager().resolveFile(HADOOP_CONFIGURATIONS_PATH), new MockActiveHadoopConfigurationLocator("a"), new DefaultFileSystemManager()); HadoopConfiguration a = locator.getConfiguration("a"); assertNotNull(a); assertEquals("a", a.getIdentifier()); assertEquals("Test Configuration A", a.getName()); } @Test(expected = ConfigurationException.class) public void getConfiguration_unknown_id() throws Exception { HadoopConfigurationLocator locator = new HadoopConfigurationLocator(); locator.init(VFS.getManager().resolveFile(HADOOP_CONFIGURATIONS_PATH), new MockActiveHadoopConfigurationLocator(), new DefaultFileSystemManager()); locator.getConfiguration("unknown"); } @Test(expected = RuntimeException.class) public void getConfiguration_not_intialized() throws ConfigurationException { HadoopConfigurationLocator locator = new HadoopConfigurationLocator(); locator.getConfiguration(null); } @Test(expected = RuntimeException.class) public void getConfigurations_not_intialized() { HadoopConfigurationLocator locator = new HadoopConfigurationLocator(); locator.getConfigurations(); } @Test public void getActiveConfiguration() throws Exception { HadoopConfigurationLocator locator = new HadoopConfigurationLocator(); locator.init(VFS.getManager().resolveFile(HADOOP_CONFIGURATIONS_PATH), new MockActiveHadoopConfigurationLocator("a"), new DefaultFileSystemManager()); HadoopConfiguration a = locator.getActiveConfiguration(); assertNotNull(a); } @Test(expected = RuntimeException.class) public void getActiveConfiguration_not_intialized() throws ConfigurationException { HadoopConfigurationLocator locator = new HadoopConfigurationLocator(); locator.getActiveConfiguration(); } @Test(expected = NullPointerException.class) public void registerNativeLibraryPath_null_path() throws SecurityException, NoSuchFieldException, IllegalArgumentException, IllegalAccessException { HadoopConfigurationLocator locator = new HadoopConfigurationLocator(); locator.registerNativeLibraryPath(null); } @Test public void registerNativeLibraryPaths() throws SecurityException, NoSuchFieldException, IllegalArgumentException, IllegalAccessException { HadoopConfigurationLocator locator = new HadoopConfigurationLocator(); locator.registerNativeLibraryPaths("test,ing"); Field f = ClassLoader.class.getDeclaredField("usr_paths"); boolean accessible = f.isAccessible(); f.setAccessible(true); try { String[] usrPaths = (String[]) f.get(null); assertTrue(Arrays.asList(usrPaths).contains("test")); assertTrue(Arrays.asList(usrPaths).contains("ing")); } finally { f.setAccessible(accessible); } } @Test public void registerNativeLibraryPaths_no_duplicates() throws SecurityException, NoSuchFieldException, IllegalArgumentException, IllegalAccessException { HadoopConfigurationLocator locator = new HadoopConfigurationLocator(); locator.registerNativeLibraryPaths("test,ing"); Field f = ClassLoader.class.getDeclaredField("usr_paths"); boolean accessible = f.isAccessible(); f.setAccessible(true); try { String[] usrPaths = (String[]) f.get(null); int pathCount = usrPaths.length; locator.registerNativeLibraryPaths("ing"); usrPaths = (String[]) f.get(null); assertTrue(Arrays.asList(usrPaths).contains("test")); assertTrue(Arrays.asList(usrPaths).contains("ing")); assertEquals(pathCount, usrPaths.length); } finally { f.setAccessible(accessible); } } @Test(expected = ConfigurationException.class) public void createConfigurationLoader_null_root() throws ConfigurationException { HadoopConfigurationLocator locator = new HadoopConfigurationLocator(); locator.createConfigurationLoader(null, null, null, new ShimProperties()); } @Test(expected = ConfigurationException.class) public void createConfigurationLoader_root_not_a_folder() throws Exception { HadoopConfigurationLocator locator = new HadoopConfigurationLocator(); // Try to create a configuration based on a file, not a folder FileObject buildProperties = VFS.getManager().resolveFile("ram:///test.file"); buildProperties.createFile(); assertEquals(FileType.FILE, buildProperties.getType()); locator.createConfigurationLoader(buildProperties, null, null, new ShimProperties()); } @Test public void loadHadoopConfiguration_with_ignore_classes() throws Exception { HadoopConfigurationLocator locator = new HadoopConfigurationLocator(); FileObject root = VFS.getManager().resolveFile(HADOOP_CONFIGURATIONS_PATH + "/a"); HadoopConfiguration configuration = locator.loadHadoopConfiguration(root); assertNotNull(configuration); } @Test public void createConfigurationLoader() throws Exception { HadoopConfigurationLocator locator = new HadoopConfigurationLocator(); FileObject root = VFS.getManager().resolveFile(HADOOP_CONFIGURATIONS_PATH + "/a"); ClassLoader cl = locator.createConfigurationLoader(root, getClass().getClassLoader(), null, new ShimProperties()); assertNotNull(cl.getResource("config.properties")); } @Test(expected = ConfigurationException.class) public void findHadoopConfigurations_errorLoadingHadoopConfig() throws Exception { FileObject root = VFS.getManager().resolveFile(HADOOP_CONFIGURATIONS_PATH); HadoopConfigurationLocator locator = new HadoopConfigurationLocator() { protected HadoopConfiguration loadHadoopConfiguration(FileObject folder) throws ConfigurationException { throw new ConfigurationException("test"); } }; locator.init(root, new MockActiveHadoopConfigurationLocator("a"), new DefaultFileSystemManager()); } @Test public void parseURLs() throws Exception { HadoopConfigurationLocator locator = new HadoopConfigurationLocator(); FileObject root = VFS.getManager().resolveFile(HADOOP_CONFIGURATIONS_PATH); List<URL> urls = locator.parseURLs(root, "a,b"); assertEquals(2, urls.size()); assertEquals(root.getURL().toURI().resolve("hadoop-configurations/a/"), urls.get(0).toURI()); assertEquals(root.getURL().toURI().resolve("hadoop-configurations/a/a-config.jar"), urls.get(1).toURI()); } }