Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.flink.client; import static org.apache.flink.client.CliFrontendTestUtils.TEST_JAR_CLASSLOADERTEST_CLASS; import static org.apache.flink.client.CliFrontendTestUtils.TEST_JAR_MAIN_CLASS; import static org.apache.flink.client.CliFrontendTestUtils.getNonJarFilePath; import static org.apache.flink.client.CliFrontendTestUtils.getTestJarPath; import static org.apache.flink.client.CliFrontendTestUtils.pipeSystemOutToNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Options; import org.apache.commons.cli.PosixParser; import org.apache.flink.client.CliFrontend; import org.apache.flink.client.program.Client; import org.apache.flink.client.program.PackagedProgram; import org.apache.flink.client.program.ProgramInvocationException; import org.apache.flink.compiler.CompilerException; import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; public class CliFrontendPackageProgramTest { @BeforeClass public static void init() { pipeSystemOutToNull(); } @Test public void testNonExistingJarFile() { try { String[] arguments = { "-j", "/some/none/existing/path", "-a", "plenty", "of", "arguments" }; CommandLine line = new PosixParser().parse(CliFrontend.getProgramSpecificOptions(new Options()), arguments, false); CliFrontend frontend = new CliFrontend(); Object result = frontend.buildProgram(line); assertTrue(result == null); } catch (Exception e) { System.err.println(e.getMessage()); e.printStackTrace(); fail("Program caused an exception: " + e.getMessage()); } } @Test public void testFileNotJarFile() { try { String[] arguments = { "-j", getNonJarFilePath(), "-a", "plenty", "of", "arguments" }; CommandLine line = new PosixParser().parse(CliFrontend.getProgramSpecificOptions(new Options()), arguments, false); CliFrontend frontend = new CliFrontend(); Object result = frontend.buildProgram(line); assertTrue(result == null); } catch (Exception e) { System.err.println(e.getMessage()); e.printStackTrace(); fail("Program caused an exception: " + e.getMessage()); } } @Test public void testVariantWithExplicitJarAndArgumentsOption() { try { String[] parameters = { "-j", getTestJarPath(), "-a", "some", "program", "arguments" }; CommandLine line = new PosixParser().parse(CliFrontend.getProgramSpecificOptions(new Options()), parameters, false); CliFrontend frontend = new CliFrontend(); Object result = frontend.buildProgram(line); assertTrue(result instanceof PackagedProgram); PackagedProgram prog = (PackagedProgram) result; Assert.assertArrayEquals(new String[] { "some", "program", "arguments" }, prog.getArguments()); Assert.assertEquals(TEST_JAR_MAIN_CLASS, prog.getMainClassName()); } catch (Exception e) { System.err.println(e.getMessage()); e.printStackTrace(); fail("Program caused an exception: " + e.getMessage()); } } @Test public void testVariantWithExplicitJarAndNoArgumentsOption() { try { String[] parameters = { "-j", getTestJarPath(), "some", "program", "arguments" }; CommandLine line = new PosixParser().parse(CliFrontend.getProgramSpecificOptions(new Options()), parameters, false); CliFrontend frontend = new CliFrontend(); Object result = frontend.buildProgram(line); assertTrue(result instanceof PackagedProgram); PackagedProgram prog = (PackagedProgram) result; Assert.assertArrayEquals(new String[] { "some", "program", "arguments" }, prog.getArguments()); Assert.assertEquals(TEST_JAR_MAIN_CLASS, prog.getMainClassName()); } catch (Exception e) { System.err.println(e.getMessage()); e.printStackTrace(); fail("Program caused an exception: " + e.getMessage()); } } @Test public void testValidVariantWithNoJarAndNoArgumentsOption() { try { String[] parameters = { getTestJarPath(), "some", "program", "arguments" }; CommandLine line = new PosixParser().parse(CliFrontend.getProgramSpecificOptions(new Options()), parameters, false); CliFrontend frontend = new CliFrontend(); Object result = frontend.buildProgram(line); assertTrue(result instanceof PackagedProgram); PackagedProgram prog = (PackagedProgram) result; Assert.assertArrayEquals(new String[] { "some", "program", "arguments" }, prog.getArguments()); Assert.assertEquals(TEST_JAR_MAIN_CLASS, prog.getMainClassName()); } catch (Exception e) { System.err.println(e.getMessage()); e.printStackTrace(); fail("Program caused an exception: " + e.getMessage()); } } @Test public void testNoJarNoArgumentsAtAll() { try { String[] parameters = {}; CommandLine line = new PosixParser().parse(CliFrontend.getProgramSpecificOptions(new Options()), parameters, false); CliFrontend frontend = new CliFrontend(); Object result = frontend.buildProgram(line); assertTrue(result == null); } catch (Exception e) { System.err.println(e.getMessage()); e.printStackTrace(); fail("Program caused an exception: " + e.getMessage()); } } @Test public void testNonExistingFileWithArguments() { try { String[] parameters = { "/some/none/existing/path", "some", "program", "arguments" }; CommandLine line = new PosixParser().parse(CliFrontend.getProgramSpecificOptions(new Options()), parameters, false); CliFrontend frontend = new CliFrontend(); Object result = frontend.buildProgram(line); assertTrue(result == null); } catch (Exception e) { System.err.println(e.getMessage()); e.printStackTrace(); fail("Program caused an exception: " + e.getMessage()); } } @Test public void testNonExistingFileWithoutArguments() { try { String[] parameters = { "/some/none/existing/path" }; CommandLine line = new PosixParser().parse(CliFrontend.getProgramSpecificOptions(new Options()), parameters, false); CliFrontend frontend = new CliFrontend(); Object result = frontend.buildProgram(line); assertTrue(result == null); } catch (Exception e) { System.err.println(e.getMessage()); e.printStackTrace(); fail("Program caused an exception: " + e.getMessage()); } } /** * Ensure that we will never have the following error. * * The test works as follows: * - Use the CliFrontend to invoke a jar file that loads a class which is only available * in the jarfile itself (via a custom classloader) * - Change the Usercode classloader of the PackagedProgram to a special classloader for this test * - the classloader will accept the special class (and return a String.class) * * org.apache.flink.client.program.ProgramInvocationException: The main method caused an error. at org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:398) at org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:301) at org.apache.flink.client.program.Client.getOptimizedPlan(Client.java:140) at org.apache.flink.client.program.Client.getOptimizedPlanAsJson(Client.java:125) at org.apache.flink.client.CliFrontend.info(CliFrontend.java:439) at org.apache.flink.client.CliFrontend.parseParameters(CliFrontend.java:931) at org.apache.flink.client.CliFrontend.main(CliFrontend.java:951) Caused by: java.io.IOException: java.lang.RuntimeException: java.lang.ClassNotFoundException: org.apache.hadoop.hive.ql.io.RCFileInputFormat at org.apache.hcatalog.mapreduce.HCatInputFormat.setInput(HCatInputFormat.java:102) at org.apache.hcatalog.mapreduce.HCatInputFormat.setInput(HCatInputFormat.java:54) at tlabs.CDR_In_Report.createHCatInputFormat(CDR_In_Report.java:322) at tlabs.CDR_Out_Report.main(CDR_Out_Report.java:380) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:622) at org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:383) */ @Test public void testPlanWithExternalClass() throws CompilerException, ProgramInvocationException { final boolean[] callme = { false }; // create a final object reference, to be able to change its val later try { String[] parameters = { getTestJarPath(), "-c", TEST_JAR_CLASSLOADERTEST_CLASS, "some", "program" }; CommandLine line = new PosixParser().parse(CliFrontend.getProgramSpecificOptions(new Options()), parameters, false); CliFrontend frontend = new CliFrontend(); Object result = frontend.buildProgram(line); assertTrue(result instanceof PackagedProgram); PackagedProgram prog = spy((PackagedProgram) result); ClassLoader testClassLoader = new ClassLoader(prog.getUserCodeClassLoader()) { @Override public Class<?> loadClass(String name) throws ClassNotFoundException { if ("org.apache.hadoop.hive.ql.io.RCFileInputFormat".equals(name)) { callme[0] = true; return String.class; // Intentionally return the wrong class. } else { return super.loadClass(name); } } }; when(prog.getUserCodeClassLoader()).thenReturn(testClassLoader); Assert.assertArrayEquals(new String[] { "some", "program" }, prog.getArguments()); Assert.assertEquals(TEST_JAR_CLASSLOADERTEST_CLASS, prog.getMainClassName()); Configuration c = new Configuration(); c.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, "devil"); Client cli = new Client(c, getClass().getClassLoader()); cli.getOptimizedPlanAsJson(prog, 666); } catch (ProgramInvocationException pie) { assertTrue("Classloader was not called", callme[0]); // class not found exception is expected as some point if (!(pie.getCause() instanceof ClassNotFoundException)) { System.err.println(pie.getMessage()); pie.printStackTrace(); fail("Program caused an exception: " + pie.getMessage()); } } catch (Exception e) { assertTrue("Classloader was not called", callme[0]); System.err.println(e.getMessage()); e.printStackTrace(); fail("Program caused an exception: " + e.getMessage()); } } }