Java tutorial
package net.sourceforge.seqware.pipeline.plugins; import com.google.common.io.Files; import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.io.PrintWriter; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Random; import java.util.concurrent.Callable; import java.util.concurrent.CompletionService; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorCompletionService; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import junit.framework.Assert; import net.sourceforge.seqware.common.module.ReturnValue; import net.sourceforge.seqware.common.util.Log; import net.sourceforge.seqware.common.util.configtools.ConfigTools; import net.sourceforge.seqware.pipeline.runner.PluginRunner; import org.apache.commons.io.FileUtils; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; import org.springframework.util.SerializationUtils; /* * Copyright (C) 2013 SeqWare * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. */ /** * * @author dyuen */ public class PluginRunnerET { private static File tempDir = null; private static Map<String, Integer> installedWorkflows = new HashMap<>(); private static Map<String, File> bundleLocations = new HashMap<>(); private static List<Integer> launchedWorkflowRuns = new ArrayList<>(); private final static boolean DEBUG_SKIP = false; private final static int PARENT = 4707; public static Map<String, Integer> getInstalledWorkflows() { return installedWorkflows; } /** * Returns a map of workflow artifactIds to their bundle path. * @return */ public static Map<String, File> getBundleLocations() { return bundleLocations; } public static List<Integer> getLaunchedWorkflowRuns() { return launchedWorkflowRuns; } @BeforeClass public static void createAndInstallArchetypes() throws IOException { clearStaticVariables(); File bundleFile = new File(System.getProperty("java.io.tmpdir"), "PluginRunnerIT_bundleLocations.bin"); File installedWorkflowsFile = new File(System.getProperty("java.io.tmpdir"), "PluginRunnerIT_installedWorkflows.bin"); if (DEBUG_SKIP) { if (bundleFile.exists() && installedWorkflowsFile.exists()) { byte[] bundleLocationsBinary = Files.toByteArray(bundleFile); byte[] installedWorkflowsBinary = Files.toByteArray(installedWorkflowsFile); bundleLocations = (Map<String, File>) SerializationUtils.deserialize(bundleLocationsBinary); installedWorkflows = (Map<String, Integer>) SerializationUtils .deserialize(installedWorkflowsBinary); return; } } createSharedTempDir(); Log.info("Trying to build and test archetypes at: " + tempDir.getAbsolutePath()); PluginRunner it = new PluginRunner(); String SEQWARE_VERSION = it.getClass().getPackage().getImplementationVersion(); Assert.assertTrue("unable to detect seqware version", SEQWARE_VERSION != null); Log.info("SeqWare version detected as: " + SEQWARE_VERSION); // for all tests, we're going to need to create and install our basic archetypes //String[] archetypes = {"java-workflow", "simplified-ftl-workflow", "legacy-ftl-workflow", "simple-legacy-ftl-workflow"}; // starting with the 1.0.x series we are deprecating the FTL workflows and the Pegasus backend so skip testing them // we are now only testing Java workflows on the Oozie-* backends String[] archetypes = { "java-workflow" }; buildAndInstallArchetypes(archetypes, SEQWARE_VERSION, true, true); Assert.assertTrue("could not locate installed workflows", installedWorkflows.size() == archetypes.length); Assert.assertTrue("could not locate installed workflow paths", installedWorkflows.size() == bundleLocations.size()); if (DEBUG_SKIP) { // dump data to a permanent map just in case we want to re-run tests without waiting byte[] bundleLocationsBinary = SerializationUtils.serialize(bundleLocations); byte[] installedWorkflowsBinary = SerializationUtils.serialize(installedWorkflows); Files.write(bundleLocationsBinary, bundleFile); Files.write(installedWorkflowsBinary, installedWorkflowsFile); } // SEQWARE-1684 - Try to keep workflow bundle size under 8G limit Log.stderr(PluginRunnerET.class.getName() + " Cleaning up " + tempDir.getAbsolutePath()); } @AfterClass public static void cleanup() throws IOException { monitorAndClean(true); } public static void buildAndInstallArchetypes(String[] archetypes, String SEQWARE_VERSION, boolean testListing, boolean deleteBundles) throws IOException, NumberFormatException { for (String archetype : archetypes) { String workflow = "seqware-archetype-" + archetype; String workflowName = workflow.replace("-", ""); // generate and install archetypes to local maven repo String command = "mvn archetype:generate -DarchetypeCatalog=local -Dpackage=com.seqware.github -DgroupId=com.github.seqware -DarchetypeArtifactId=" + workflow + " -Dversion=1.0-SNAPSHOT -DarchetypeGroupId=com.github.seqware -DartifactId=" + workflow + " -Dworkflow-name=" + workflowName + " -B -Dgoals=install"; String genOutput = ITUtility.runArbitraryCommand(command, 0, tempDir); Log.info(genOutput); // install the workflows to the database and record their information File workflowDir = new File(tempDir, workflow); File targetDir = new File(workflowDir, "target"); final String workflow_name = "Workflow_Bundle_" + workflowName + "_1.0-SNAPSHOT_SeqWare_" + SEQWARE_VERSION; File bundleDir = new File(targetDir, workflow_name); bundleLocations.put(workflow, bundleDir); // zip up the bundles first if we want the bundle locations to survive String zipCommand = "--plugin net.sourceforge.seqware.pipeline.plugins.BundleManager -verbose -- --path-to-package " + bundleDir.getAbsolutePath() + " --bundle " + tempDir.getAbsolutePath(); String zipOutput = ITUtility.runSeqWareJar(zipCommand, ReturnValue.SUCCESS, null); Log.info(zipOutput); String installCommand = "-p net.sourceforge.seqware.pipeline.plugins.BundleManager -verbose -- -i -b " + tempDir.getAbsolutePath() + File.separatorChar + workflow_name + ".zip"; String installOutput = ITUtility.runSeqWareJar(installCommand, ReturnValue.SUCCESS, null); Log.info(installOutput); int accession = ITUtility.extractSwid(installOutput); installedWorkflows.put(workflow, accession); Log.info("Found workflow " + workflow + " with accession " + accession); if (testListing) { Log.info("Attempting to list " + workflow); String listCommand = "-p net.sourceforge.seqware.pipeline.plugins.BundleManager -- -l -b " + bundleDir.getAbsolutePath(); String listOutput = ITUtility.runSeqWareJar(listCommand, ReturnValue.SUCCESS, null); Log.info(listOutput); } if (deleteBundles) { Log.info("Attempting to delete bundle after install " + workflow); FileUtils.deleteDirectory(bundleDir); } } } public static void clearStaticVariables() { //clean-up static variables installedWorkflows.clear(); bundleLocations.clear(); launchedWorkflowRuns.clear(); createSharedTempDir(); } public static void monitorAndClean(boolean monitor) throws IOException { // testing monitoring one more time for (int launchedWorkflowRun : launchedWorkflowRuns) { Log.info("Attempting to monitor " + launchedWorkflowRun); String listCommand = "-p net.sourceforge.seqware.pipeline.plugins.WorkflowStatusChecker -- --workflow-run-accession " + launchedWorkflowRun; String listOutput = ITUtility.runSeqWareJar(listCommand, ReturnValue.SUCCESS, null); Log.info(listOutput); } if (!DEBUG_SKIP) { FileUtils.deleteDirectory(tempDir); } clearStaticVariables(); } private static void createSharedTempDir() { // need to create in a shared location for seqware installs with multiple nodes //tempDir = Files.createTempDir(); String parentDir = ConfigTools.getSettings().get("OOZIE_WORK_DIR"); tempDir = new File(parentDir, String.valueOf(Math.abs(new Random().nextInt()))); tempDir.mkdir(); } @Test public void testExportParameters() throws IOException { Map<String, File> iniParams = exportWorkflowInis(); Assert.assertTrue("Loaded correct number of ini files", iniParams.size() == installedWorkflows.size()); } @Test public void testScheduleAndLaunch() throws IOException { Map<String, File> iniParams = exportWorkflowInis(); String localhost = ITUtility.getLocalhost(); Log.info("Attempting to schedule on host: " + localhost); Map<String, Integer> wr_accessions = new HashMap<>(); for (Entry<String, Integer> e : installedWorkflows.entrySet()) { Log.info("Attempting to schedule " + e.getKey()); String workflowPath = iniParams.get(e.getKey()).getAbsolutePath(); String accession = Integer.toString(installedWorkflows.get(e.getKey())); String listCommand = "-p net.sourceforge.seqware.pipeline.plugins.WorkflowLauncher -- --ini-files " + workflowPath + " --workflow-accession " + accession + " --schedule --parent-accessions " + PARENT + " --host " + localhost; String listOutput = ITUtility.runSeqWareJar(listCommand, ReturnValue.SUCCESS, null); Log.info(listOutput); int wr_accession = ITUtility.extractSwid(listOutput); wr_accessions.put(e.getKey(), wr_accession); launchedWorkflowRuns.add(wr_accession); Log.info("Scheduled workflow " + e.getKey() + " with accession " + wr_accession); } // launch-scheduled String schedCommand = "-p net.sourceforge.seqware.pipeline.plugins.WorkflowLauncher -- --launch-scheduled"; String schedOutput = ITUtility.runSeqWareJar(schedCommand, ReturnValue.SUCCESS, null); Log.info(schedOutput); try { Log.info("Wait for launches to settle "); Thread.sleep(5000); } catch (InterruptedException ex) { } // testing monitoring for (Entry<String, Integer> e : wr_accessions.entrySet()) { Log.info("Attempting to monitor " + e.getKey()); String listCommand = "-p net.sourceforge.seqware.pipeline.plugins.WorkflowStatusChecker -- --workflow-run-accession " + e.getValue(); String listOutput = ITUtility.runSeqWareJar(listCommand, ReturnValue.SUCCESS, null); Log.info(listOutput); } } @Test public void testLaunchingWithoutWait() throws IOException { Map<String, File> iniParams = exportWorkflowInis(); String localhost = ITUtility.getLocalhost(); Log.info("Attempting to launch without wait on host: " + localhost); Map<String, Integer> wr_accessions = new HashMap<>(); for (Entry<String, Integer> e : installedWorkflows.entrySet()) { Log.info("Attempting to launch " + e.getKey()); String workflowPath = iniParams.get(e.getKey()).getAbsolutePath(); String accession = Integer.toString(installedWorkflows.get(e.getKey())); String listCommand = "-p net.sourceforge.seqware.pipeline.plugins.WorkflowLauncher -- --ini-files " + workflowPath + " --workflow-accession " + accession + " --parent-accessions " + PARENT + " --host " + localhost; String listOutput = ITUtility.runSeqWareJar(listCommand, ReturnValue.SUCCESS, null); Log.info(listOutput); } } @Test public void testLaunchingWithWait() throws IOException { Map<String, File> iniParams = exportWorkflowInis(); String localhost = ITUtility.getLocalhost(); Log.info("Attempting to launch with wait on host: " + localhost); Map<String, Integer> wr_accessions = new HashMap<>(); for (Entry<String, Integer> e : installedWorkflows.entrySet()) { Log.info("Attempting to launch " + e.getKey()); String workflowPath = iniParams.get(e.getKey()).getAbsolutePath(); String accession = Integer.toString(installedWorkflows.get(e.getKey())); String listCommand = "-p net.sourceforge.seqware.pipeline.plugins.WorkflowLauncher -- --ini-files " + workflowPath + " --workflow-accession " + accession + " --parent-accessions " + PARENT + " --wait --host " + localhost; String listOutput = ITUtility.runSeqWareJar(listCommand, ReturnValue.SUCCESS, null); Log.info(listOutput); } } @Test public void testLaunchingWithWaitAndNoMetadata() throws IOException { Map<String, File> iniParams = exportWorkflowInis(); String localhost = ITUtility.getLocalhost(); Log.info("Attempting to launch with wait on host: " + localhost); Map<String, Integer> wr_accessions = new HashMap<>(); for (Entry<String, Integer> e : installedWorkflows.entrySet()) { Log.info("Attempting to launch " + e.getKey()); String workflowPath = iniParams.get(e.getKey()).getAbsolutePath(); String accession = Integer.toString(e.getValue()); String listCommand = "-p net.sourceforge.seqware.pipeline.plugins.WorkflowLauncher -- --ini-files " + workflowPath + " --workflow-accession " + accession + " --no-metadata --parent-accessions " + PARENT + " --wait --host " + localhost; String listOutput = ITUtility.runSeqWareJar(listCommand, ReturnValue.SUCCESS, null); Log.info(listOutput); } } public static void main(String[] args) throws IOException { PluginRunnerET it = new PluginRunnerET(); List<Integer> list = new ArrayList<>(); for (String acc : args) { try { Integer accInt = Integer.valueOf(acc); list.add(accInt); } catch (NumberFormatException e) { Log.stdout("NumberFormatException, skipped acc"); } } it.testLatestWorkflowsInternal(list); } @Test public void testLatestWorkflows() throws IOException { testLatestWorkflowsInternal(new ArrayList<Integer>()); } private Map<String, File> exportWorkflowInis() throws IOException { Map<String, File> iniParams = new HashMap<>(); for (Entry<String, Integer> e : installedWorkflows.entrySet()) { File workflowIni = exportINIFile(e.getKey(), e.getValue(), false); iniParams.put(e.getKey(), workflowIni); } return iniParams; } public void testLatestWorkflowsInternal(List<Integer> accessions) throws IOException { String output = ITUtility.runSeqWareJar( "-p net.sourceforge.seqware.pipeline.plugins.BundleManager -- --list-installed", ReturnValue.SUCCESS, null); Assert.assertTrue("output should include installed workflows", output.contains("INSTALLED WORKFLOWS")); Map<String, WorkflowInfo> latestWorkflows = new HashMap<>(); String[] lines = output.split(System.getProperty("line.separator")); for (String line : lines) { String[] lineParts = line.split("\t"); try { int workflow_accession = Integer.valueOf(lineParts[3]); String workflowName = lineParts[0]; String path = lineParts[lineParts.length - 2]; if (path.equals("null")) { continue; } WorkflowInfo wi = new WorkflowInfo(workflow_accession, path, workflowName, lineParts[1]); //TODO: check that the permanent workflow actually exists, if not warn and skip File fileAtPath = new File(path); if (!fileAtPath.exists()) { Log.warn("Skipping " + workflowName + ":" + workflow_accession + " , bundle path does not exist at " + path); continue; } if (!latestWorkflows.containsKey(workflowName)) { latestWorkflows.put(workflowName, wi); } else { // contained int old = latestWorkflows.get(workflowName).sw_accession; if (workflow_accession > old) { latestWorkflows.put(workflowName, wi); } } } catch (Exception e) { /** * do nothing and skip this line of the BundleManager output */ } } // setup thread pool ExecutorService threadPool = Executors.newFixedThreadPool(latestWorkflows.size()); CompletionService<String> pool = new ExecutorCompletionService<>(threadPool); for (Entry<String, WorkflowInfo> e : latestWorkflows.entrySet()) { System.out.println("Testing " + e.getKey() + " " + e.getValue().sw_accession); // if we have an accession list, skip accessions that are not in it if (accessions.size() > 0) { Integer acc = e.getValue().sw_accession; if (!accessions.contains(acc)) { System.out.println( "Skipping " + e.getKey() + " " + e.getValue().sw_accession + " due to accession list"); continue; } } StringBuilder params = new StringBuilder(); params.append("--bundle ").append(e.getValue().path).append(" "); params.append("--version ").append(e.getValue().version).append(" "); params.append("--test "); File tempFile = File.createTempFile(e.getValue().name, ".out"); pool.submit(new TestingThread(params.toString(), tempFile)); } for (Entry<String, WorkflowInfo> e : latestWorkflows.entrySet()) { try { pool.take().get(); } catch (InterruptedException ex) { Log.error(ex); } catch (ExecutionException ex) { Log.error(ex); } } threadPool.shutdown(); } public File exportINIFile(String name, Integer accession, boolean newCLI) throws IOException { String listOutput; if (newCLI) { Log.info("Attempting to export parameters for " + name); File workflowIni = File.createTempFile("workflow", "ini"); String listCommand = " workflow ini --accession " + accession + " --out " + workflowIni.getAbsolutePath(); ITUtility.runSeqwareCLI(listCommand, ReturnValue.SUCCESS, null); // new command line does not go to stdout listOutput = FileUtils.readFileToString(workflowIni); } else { Log.info("Attempting to export parameters for " + name); String listCommand = "-p net.sourceforge.seqware.pipeline.plugins.BundleManager -- --list-workflow-params --workflow-accession " + accession; listOutput = ITUtility.runSeqWareJar(listCommand, ReturnValue.SUCCESS, null); Log.info(listOutput); } // go through output and dump out the workflow.ini File workflowIni = File.createTempFile("workflow", "ini"); String[] lines = listOutput.split(System.getProperty("line.separator")); PrintWriter out = new PrintWriter(new FileWriter(workflowIni)); for (String line : lines) { if (line.startsWith("-") || line.startsWith("=") || line.startsWith("$") || line.startsWith("Running Plugin") || line.startsWith("Setting Up Plugin")) { continue; } out.println(line); } out.close(); return workflowIni; } public class WorkflowInfo { public int sw_accession; public String path; public String name; public String version; public WorkflowInfo(int sw_accession, String path, String name, String version) { this.sw_accession = sw_accession; this.path = path; this.name = name; this.version = version; } } private final class TestingThread implements Callable<String> { private final String command; private final File output; protected TestingThread(String command, File output) { this.command = command; this.output = output; } @Override public String call() { try { String tOutput = ITUtility.runSeqWareJar( "-p net.sourceforge.seqware.pipeline.plugins.BundleManager -- " + command.toString(), ReturnValue.SUCCESS, null); Log.error(command + " completed, writing output to " + output.getAbsolutePath()); FileUtils.write(output, tOutput); return tOutput; } catch (IOException ex) { Log.error("IOException while running " + command, ex); return null; } } } }