Java tutorial
/* Copyright 2013 KU Leuven Research and Development - iMinds - Distrinet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Administrative Contact: dnet-project-office@cs.kuleuven.be Technical Contact: bart.vanbrabant@cs.kuleuven.be */ package drm.taskworker.workers; import static drm.taskworker.workers.Constants.PRM_START_FILE; import static drm.taskworker.workers.Constants.PRM_START_METHOD; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.FileReader; import java.io.FilenameFilter; import java.io.IOException; import java.io.OutputStream; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; import org.apache.commons.codec.binary.Base64; import org.apache.commons.io.IOUtils; import drm.taskworker.Worker; import drm.taskworker.tasks.ParameterFoundException; import drm.taskworker.tasks.Task; import drm.taskworker.tasks.TaskResult; import drm.taskworker.tasks.ValueRef; /** * The optimus worker! * * */ public class OptimusWorker extends Worker { /** * Creates a new work with the name blob-to-cache */ public OptimusWorker(String workerName) { super(workerName); } private Map<UUID, Process> handles = new HashMap<>(); /** * Archive the result of the previous task */ public TaskResult work(Task task) { try { File workdir = new File(task.getJobId().toString()); if (!workdir.exists()) { workdir.mkdir(); return start(workdir, task); } else { return step(workdir, task); } } catch (Exception e) { TaskResult out = new TaskResult(); out.setException(e); out.fail(); return out; } } private TaskResult start(File workdir, Task task) throws IOException, ParameterFoundException { String obj = (String) task.getParam(PRM_START_FILE); // FIXME only works with the adapted taskworker-client script (not committed) byte[] file = Base64.decodeBase64(obj); String method = (String) task.getParam(PRM_START_METHOD); File f = new File(workdir, "input.zip"); OutputStream out = new FileOutputStream(f); out.write(file); out.close(); ProcessBuilder pb = new ProcessBuilder("python", task.getJobOption("optimus.exe"), "input.zip", method); pb.directory(workdir); Process handle = pb.start(); (new VoidStreamPump(handle.getInputStream())).start(); (new VoidStreamPump(handle.getErrorStream())).start(); handles.put(task.getJobId(), handle); // start looking for flag file File goFile = new File(workdir, task.getJobOption("optimus.flagfile.go")); while (!goFile.exists()) { try { Thread.sleep(1000); } catch (InterruptedException e) { throw new Error("should not occur", e); } } goFile.delete(); return split(workdir, task); } private TaskResult split(File dir, Task task) throws IOException { TaskResult out = new TaskResult(); File workdir = dir; if (workdir == null) workdir = new File(task.getJobId().toString()); File jobsFile = new File(workdir, task.getJobOption("optimus.jobs")); BufferedReader br = new BufferedReader(new FileReader(jobsFile)); String line = br.readLine(); while (line != null) { out.addNextTask(makeTask(task, workdir, line)); line = br.readLine(); } br.close(); jobsFile.delete(); out.setSplit(); out.setResult(TaskResult.Result.SUCCESS); return out; } private Task makeTask(Task task, File workdir, String line) throws IOException, IllegalArgumentException { String[] parts = line.split(" "); String jobType = parts[0]; String command; if (jobType.equals("static")) command = "/home/ec2-user/optimus64/10.13/bin/LINUX-x86_64/truss_stat.exe"; else if (jobType.equals("dynamic")) command = "/home/ec2-user/optimus64/10.13/bin/LINUX-x86_64/truss_dyn.exe"; else throw new IllegalArgumentException("Unknown job type"); String inputfile = parts[1]; String outputfile = parts[2]; String fullPathCurrentJob = parts[3]; // String currentExperiment = parts[4]; // String totalExperiments = parts[5]; FileInputStream fis = new FileInputStream(new File(fullPathCurrentJob, inputfile)); byte[] inputData = IOUtils.toByteArray(fis); Task out = new Task(task, "execute"); out.addParam("command", command); out.addParam("inputData", inputData); out.addParam("fullOutputName", fullPathCurrentJob + "/" + outputfile); return out; } @SuppressWarnings("unchecked") private TaskResult step(File workdir, Task task) throws IOException, ParameterFoundException { // write outputs List<ValueRef> results = (List<ValueRef>) task.getParam("outputData"); List<ValueRef> fileNames = (List<ValueRef>) task.getParam("fullOutputName"); for (int i = 0; i < results.size(); i++) output((byte[]) results.get(i).getValue(), (String) fileNames.get(i).getValue()); // activate Optimus File optimusFile = new File(workdir, task.getJobOption("optimus.flagfile.step")); optimusFile.createNewFile(); File stopFile = new File(workdir, task.getJobOption("optimus.flagfile.stop")); while (!stopFile.exists()) { File goFile = new File(workdir, task.getJobOption("optimus.flagfile.go")); if (goFile.exists()) { goFile.delete(); return split(workdir, task); } else { try { Thread.sleep(1000); } catch (InterruptedException e) { throw new Error("should not occur", e); } } } stopFile.delete(); return stop(task, locateSummaryFile(workdir)); } private void output(byte[] value, String fullName) throws IOException { OutputStream os = new FileOutputStream(new File(fullName)); os.write(value); os.close(); } private TaskResult stop(Task task, File summary) throws IOException { //don't wait for kill: kill Optimus handles.remove(task.getJobId()).destroy(); FileInputStream fis = new FileInputStream(summary); byte[] summaryData = IOUtils.toByteArray(fis); Task newTask = new Task(task, this.getNextWorker(task.getJobId())); newTask.addParam("arg0", summaryData); TaskResult result = new TaskResult(); result.addNextTask(newTask); result.setResult(TaskResult.Result.SUCCESS); return result; } private File locateSummaryFile(File workdir) { FilenameFilter filter = new SummaryFileFilter(); File[] summaries = workdir.listFiles(filter); if (summaries != null && summaries.length > 0 && summaries[0].exists()) { return summaries[0]; } else { File[] fList = workdir.listFiles(); for (File file : fList) { try { if (file.isDirectory()) return locateSummaryFile(file); } catch (IllegalStateException e) { // try next file } } } throw new IllegalStateException("No summary file"); } public class SummaryFileFilter implements FilenameFilter { @Override public boolean accept(File dir, String fileName) { return fileName.endsWith(".summary"); } } }