Java tutorial
/******************************************************************************* * Copyright (c) 2006-2010 Vienna University of Technology, * Department of Software Technology and Interactive Systems * * All rights reserved. This program and the accompanying * materials are made available under the terms of the * Apache License, Version 2.0 which accompanies * this distribution, and is available at * http://www.apache.org/licenses/LICENSE-2.0 *******************************************************************************/ package eu.planets_project.pp.plato.massmigration; import java.io.BufferedOutputStream; import java.io.BufferedWriter; import java.io.File; import java.io.FileFilter; import java.io.FileOutputStream; import java.io.FileWriter; import java.io.IOException; import java.io.OutputStream; import java.io.Serializable; import java.text.NumberFormat; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; import org.apache.commons.logging.Log; import eu.planets_project.pp.plato.model.DetailedExperimentInfo; import eu.planets_project.pp.plato.model.PreservationActionDefinition; import eu.planets_project.pp.plato.model.SampleObject; import eu.planets_project.pp.plato.model.ToolExperience; import eu.planets_project.pp.plato.model.measurement.MeasurableProperty; import eu.planets_project.pp.plato.model.measurement.Measurement; import eu.planets_project.pp.plato.model.scales.BooleanScale; import eu.planets_project.pp.plato.model.scales.FreeStringScale; import eu.planets_project.pp.plato.model.values.BooleanValue; import eu.planets_project.pp.plato.model.values.FreeStringValue; import eu.planets_project.pp.plato.model.values.INumericValue; import eu.planets_project.pp.plato.services.action.IMigrationAction; import eu.planets_project.pp.plato.services.action.MigrationResult; import eu.planets_project.pp.plato.services.action.PreservationActionServiceFactory; import eu.planets_project.pp.plato.util.FileUtils; import eu.planets_project.pp.plato.util.MeasurementStatistics; import eu.planets_project.pp.plato.util.PlatoLogger; public class MassMigrationRunner extends Thread implements Serializable { private static final long serialVersionUID = -2687062223716715348L; private static final String CSV_SEPARATOR = ";"; private Log log = PlatoLogger.getLogger(MassMigrationRunner.class); private File sourceDir; private File resultDir; private String[] inputFiles; MassMigrationSetup setup; public MassMigrationRunner(MassMigrationSetup setup) { this.setup = setup; } private class FileDirectoryFilter implements FileFilter { public boolean accept(java.io.File f) { return (f.isFile() && !f.isHidden()); } } public void run() { sourceDir = new File(setup.getSourcePath()); resultDir = new File(setup.getResultPath()); if (!sourceDir.exists() || !sourceDir.isDirectory() || !sourceDir.canRead()) { setup.getStatus().setStatus(MassMigrationStatus.FAILED); throw new IllegalArgumentException("Cannot read from source directory: " + setup.getSourcePath()); } File currentResultdir = new File(resultDir, setup.getName() + "-" + System.nanoTime()); setup.setLastResultPath(currentResultdir.getAbsolutePath()); if (!currentResultdir.mkdirs()) { setup.getStatus().setStatus(MassMigrationStatus.FAILED); throw new IllegalArgumentException("Cannot write to result directory: " + setup.getSourcePath()); } try { File[] list = sourceDir.listFiles(new FileDirectoryFilter()); List<String> array = new ArrayList<String>(); for (int i = 0; i < list.length; i++) { array.add(list[i].getName()); } inputFiles = array.toArray(new String[] {}); java.util.Arrays.sort(inputFiles, String.CASE_INSENSITIVE_ORDER); Map<String, String> alternativesPaths = new HashMap<String, String>(); // create directories for each alternative for (MassMigrationExperiment exp : setup.getExperiments()) { String altPath = FileUtils.makeFilename(makeUniqueActionName(exp.getAction())); File dir = new File(currentResultdir, altPath); if (dir.mkdir()) { // add this path only if it can be created alternativesPaths.put(exp.getAction().getShortname(), dir.getAbsolutePath()); } } setup.getStatus().setNumOfSamples(inputFiles.length); setup.getStatus().setNumOfTools(setup.getExperiments().size()); int current = 0; for (String filename : inputFiles) { File file = new File(sourceDir + File.separator + filename); current++; int currentTool = 0; setup.getStatus().setCurrentSample(current); try { byte[] data = FileUtils.getBytesFromFile(file); /* * migrate this file with every migration service, so we have to load it only once */ for (MassMigrationExperiment exp : setup.getExperiments()) { currentTool++; setup.getStatus().setCurrentTool(currentTool); MigrationResult result = runExperiment(exp, file.getName(), data); if ((result != null) && (result.isSuccessful())) { // store migration result String altPath = alternativesPaths.get(exp.getAction().getShortname()); if (altPath != null) { File mResult = new File(altPath, file.getName() + "." + result.getTargetFormat().getDefaultExtension()); OutputStream out = new BufferedOutputStream(new FileOutputStream(mResult)); out.write(result.getMigratedObject().getData().getData()); out.close(); } } } } catch (IOException ioe) { log.error("Could not load file: " + file.getAbsolutePath(), ioe); } catch (Exception e) { log.error( "Exception while running experiment on file " + file.getAbsolutePath() + e.getMessage(), e); } } /* * Calculation finished - collect result values and export them for further calculations */ Locale locale = Locale.getDefault(); NumberFormat format = NumberFormat.getNumberInstance(locale); //new DecimalFormat("##########.##"); ToolExperience toolExp = null; List<String> allProperties = new ArrayList<String>(); Map<String, DetailedExperimentInfo> accumulatedAvg = new HashMap<String, DetailedExperimentInfo>(); for (MassMigrationExperiment e : setup.getExperiments()) { /* calculate average per experiment */ /* put measurements of sample files to toolExp */ toolExp = MeasurementStatistics.generateToolExperience(e.getResult()); /* get calculated average per property */ DetailedExperimentInfo average = MeasurementStatistics.getAverage(toolExp); accumulatedAvg.put(e.getAction().getShortname(), average); e.getAverages().clear(); e.getAverages().put(average); /* a list of all properties to iterate over the values */ allProperties.clear(); allProperties.addAll(toolExp.getMeasurements().keySet()); Collections.sort(allProperties); /* * write all measurements of this experiment to a file */ String statistics = FileUtils.makeFilename(makeUniqueActionName(e.getAction())); File statisticsFile = new File(currentResultdir, statistics + ".csv"); try { BufferedWriter out = new BufferedWriter(new FileWriter(statisticsFile)); StringBuffer header = new StringBuffer(); header.append("sample"); for (String key : allProperties) { header.append(CSV_SEPARATOR).append(key); } /* write header */ out.append(header); out.newLine(); List<String> keySet = new ArrayList<String>(e.getResult().keySet()); String[] toSort = keySet.toArray(new String[] {}); java.util.Arrays.sort(toSort, String.CASE_INSENSITIVE_ORDER); /* write measured values for all samples */ for (int i = 0; i < toSort.length; i++) { String sample = toSort[i]; /* 1. column: sample name */ out.append(sample); /* followed by all properties */ DetailedExperimentInfo info = e.getResult().get(sample); for (String prop : allProperties) { out.append(CSV_SEPARATOR); Measurement m = info.getMeasurements().get(prop); if (m != null) { if (m.getValue() instanceof INumericValue) { /* */ double value = ((INumericValue) m.getValue()).value(); out.append(format.format(value)); } else out.append(m.getValue().toString()); } } out.newLine(); } /* write header again */ out.append(header); out.newLine(); /* and write calculated average */ out.append("average"); for (String key : allProperties) { out.append(CSV_SEPARATOR); Measurement m = e.getAverages().getMeasurements().get(key); if (m != null) { if (m.getValue() instanceof INumericValue) { double value = ((INumericValue) m.getValue()).value(); out.append(format.format(value)); } else out.append(m.getValue().toString()); } } out.newLine(); out.append("startupTime"); out.append(CSV_SEPARATOR); try { out.append(Double.toString(toolExp.getStartupTime())); } catch (Exception ex) { log.error("Error in calculating the startup time (linear regression): " + ex.getMessage()); out.append("Err"); } out.newLine(); out.close(); } catch (IOException e1) { log.error("Could not write statistics for: " + statistics, e1); } } /* * and write accumulated values */ File statisticsFile = new File(currentResultdir, "accumulated.csv"); allProperties.clear(); allProperties.add(MigrationResult.MIGRES_ELAPSED_TIME_PER_MB); allProperties.add(MigrationResult.MIGRES_USED_TIME_PER_MB); allProperties.add(MigrationResult.MIGRES_ELAPSED_TIME); allProperties.add(MigrationResult.MIGRES_USED_TIME); //... try { BufferedWriter out = new BufferedWriter(new FileWriter(statisticsFile)); /* write machine info */ if (toolExp != null) { // use machine info of last experiment! for (String prop : toolExp.getMeasurements().keySet()) { if (prop.startsWith("machine:")) { out.append(prop).append(CSV_SEPARATOR).append(toolExp.getMeasurements().get(prop) .getList().get(0).getValue().getFormattedValue()); out.newLine(); } } out.newLine(); } /* write header */ out.append("tool"); for (String key : allProperties) { out.append(CSV_SEPARATOR).append(key); } out.newLine(); /* write averaged values for all actions */ for (String action : accumulatedAvg.keySet()) { /* 1. column: action name */ out.append(action); /* followed by all properties */ DetailedExperimentInfo average = accumulatedAvg.get(action); for (String prop : allProperties) { out.append(CSV_SEPARATOR); Measurement m = average.getMeasurements().get(prop); if (m != null) { if (m.getValue() instanceof INumericValue) { /* */ double value = ((INumericValue) m.getValue()).value(); out.append(format.format(value)); } else out.append(m.getValue().toString()); } } out.newLine(); } out.newLine(); out.close(); } catch (IOException e1) { log.error("Could not write accumulated statistics.", e1); } setup.getStatus().setStatus(MassMigrationStatus.FINISHED); } catch (RuntimeException e) { setup.getStatus().setStatus(MassMigrationStatus.FAILED); log.error("Massmigration failed.", e); } } /** * Migrates data with service defined in experiment <param>e</param>s action, and stores the result * in experiment's results with key <param>filename</param> * * @param e * @param filename * @param data */ private MigrationResult runExperiment(MassMigrationExperiment e, String filename, byte[] data) { IMigrationAction service = getService(e.getAction()); DetailedExperimentInfo eInfo = e.getResult().get(filename); if (eInfo == null) { eInfo = new DetailedExperimentInfo(); e.getResult().put(filename, eInfo); } // remove old results eInfo.getMeasurements().clear(); // why does this expect an instance of SampleObject ??!! SampleObject r = new SampleObject(); r.getData().setData(data); r.setFullname(filename); Measurement success = new Measurement(); success.setProperty(new MeasurableProperty(new BooleanScale(), MigrationResult.MIGRES_SUCCESS)); success.setValue(success.getProperty().getScale().createValue()); Measurement report = new Measurement(); report.setProperty(new MeasurableProperty(new FreeStringScale(), MigrationResult.MIGRES_REPORT)); report.setValue(report.getProperty().getScale().createValue()); try { MigrationResult result = service.migrate(e.getAction(), r); if (result.isSuccessful()) { /* put all info to toolExperience */ eInfo.getMeasurements().putAll(result.getMeasurements()); ((BooleanValue) success.getValue()).setValue("true"); ((FreeStringValue) report.getValue()).setValue(result.getReport()); } else { ((BooleanValue) success.getValue()).setValue("false"); ((FreeStringValue) report.getValue()).setValue(result.getReport()); } // and return result, (including the migrated object) return result; } catch (Exception e1) { ((BooleanValue) success.getValue()).setValue("false"); ((FreeStringValue) report.getValue()).setValue(e1.getMessage()); log.error("Migration with service " + e.getAction().getShortname() + " failed."); } return null; } /** * returns a matching service stub * could be use to cache already used stubs... */ private IMigrationAction getService(PreservationActionDefinition a) { return (IMigrationAction) PreservationActionServiceFactory.getPreservationAction(a); } private static String makeUniqueActionName(PreservationActionDefinition action) { String key = "minimee/"; String actionIdentifier = action.getUrl().substring(action.getUrl().indexOf(key) + key.length()); return action.getShortname() + "_" + actionIdentifier; } }