Java tutorial
/* * iBIOMES - Integrated Biomolecular Simulations * Copyright (C) 2014 Julien Thibault, University of Utah * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. */ package edu.utah.bmi.ibiomes.lite; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.nio.file.Files; import java.nio.file.LinkOption; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.nio.file.attribute.PosixFilePermission; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Properties; import java.util.Set; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.OutputKeys; import javax.xml.transform.Transformer; import javax.xml.transform.TransformerException; import javax.xml.transform.TransformerFactory; import javax.xml.transform.dom.DOMSource; import javax.xml.transform.stream.StreamResult; import javax.xml.xpath.XPathConstants; import javax.xml.xpath.XPathExpressionException; import org.jfree.chart.JFreeChart; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.Node; import org.w3c.dom.NodeList; import org.xml.sax.SAXException; import edu.utah.bmi.Utils; import edu.utah.bmi.ibiomes.catalog.MetadataLookup; import edu.utah.bmi.ibiomes.conf.DirectoryStructureDescriptor; import edu.utah.bmi.ibiomes.conf.IBIOMESConfiguration; import edu.utah.bmi.ibiomes.experiment.XMLConverter; import edu.utah.bmi.ibiomes.graphics.plot.ColumnDataFile; import edu.utah.bmi.ibiomes.graphics.plot.PlotGenerator; import edu.utah.bmi.ibiomes.io.Locker; import edu.utah.bmi.ibiomes.lite.search.IBIOMESLiteSearchRecordSet; import edu.utah.bmi.ibiomes.metadata.FileMetadata; import edu.utah.bmi.ibiomes.metadata.IBIOMESFileGroup; import edu.utah.bmi.ibiomes.metadata.MetadataAVU; import edu.utah.bmi.ibiomes.metadata.MetadataAVUList; import edu.utah.bmi.ibiomes.parse.CSVFile; import edu.utah.bmi.ibiomes.parse.DirectoryParsingProgressBar; import edu.utah.bmi.ibiomes.parse.IBIOMESListener; import edu.utah.bmi.ibiomes.parse.LocalFile; import edu.utah.bmi.ibiomes.parse.chem.ExperimentFactory; import edu.utah.bmi.ibiomes.parse.chem.ExperimentFolder; import edu.utah.bmi.ibiomes.xml.XPathReader; /** * Utility class to find iBIOMES descriptors in the local resource, * and copy the necessary files (xml, pictures) to the public html folder for web display. * * @author Julien Thibault, University of Utah * */ public class IBIOMESLiteManager { public final static String PATH_FOLDER_SEPARATOR = (Utils.isWindows() ? "\\" : "/"); public final static String PATH_FOLDER_SEPARATOR_REGEX = (Utils.isWindows() ? "\\\\" : "/"); public final static String IBIOMES_LITE_VERSION = "11-12-2013"; private final static String IBIOMES_EXPERIMENT_SUMMARY_XSL_PATH = "style" + PATH_FOLDER_SEPARATOR + "experiment_summary.xsl"; private final static String IBIOMES_EXPERIMENT_WORKFLOW_XSL_PATH = "style" + PATH_FOLDER_SEPARATOR + "experiment_workflow.xsl"; private final static String IBIOMES_EXPERIMENT_FILE_TREE_XSL_PATH = "style" + PATH_FOLDER_SEPARATOR + "experiment_files.xsl"; private final static String IBIOMES_EXPERIMENT_RUNS_XSL_PATH = "style" + PATH_FOLDER_SEPARATOR + "experiment_runs.xsl"; private final static String IBIOMES_EXPERIMENT_SET_XSL_PATH = "style" + PATH_FOLDER_SEPARATOR + "experiment_list.xsl"; private final static String IBIOMES_LITE_EXPERIMENT_DIR = "experiments"; private final static String IBIOMES_LITE_XML_INDEX = "ibiomes.compiled.xml"; private final static String IBIOMES_LITE_HTML_INDEX = "index.html"; private final static String METADATA_LOOKUP_INDEX_PATH = "data" + PATH_FOLDER_SEPARATOR + "metadata-attr"; public static final String IBIOMES_DESC_FILE_TREE_FILE_NAME = ".ibiomes.xml"; public static final String IBIOMES_DESC_WORKFLOW_FILE_NAME = ".ibiomes-details.xml"; public static final String IBIOMES_PARSE_CONFIG_FILE_NAME = ".ibiomes-parse.config"; public static final String IBIOMES_LITE_DATA_DIR = "data"; public static final String IBIOMES_TEMP_DIR = "temp"; public static final String PARSER_CONFIG_PROPERTY_RULE_SET_FILE = "parsingRuleSetFilePath"; public static final String PARSER_CONFIG_PROPERTY_SOFTWARE_CONTEXT = "softwareContext"; private static final String PARSER_CONFIG_PROPERTY_DEPTH_INDEPENDENCE = "depthForIndependentGroups"; private final static int PLOT_WIDTH = 600; private final static int PLOT_HEIGHT = 480; /** * Environment variable that specifies the path to the web folder. */ public final static String IBIOMES_LITE_WEBDIR_ENV_VAR = "IBIOMES_LITE_WEBDIR"; /** * Environment variable that sets the path to the iBIOMES libraries. */ public final static String IBIOMES_HOME_ENV_VAR = "IBIOMES_HOME"; private String publicHtmlFolder; private String ibiomesHomeFolder; private XmlExperimentListFile experimentListXml; private static IBIOMESLiteManager ibiomesLite = null; private boolean outputToConsole = true; /** * Create instance of iBIOMES Lite manager by specifying the location of the target web folder. * @param dirPath Path to the iBIOMES Lite web folder * @throws Exception */ private IBIOMESLiteManager(String webDirPath) throws Exception { //set default location for web directory if necessary if (webDirPath == null || webDirPath.length() == 0) { webDirPath = System.getenv(IBIOMES_LITE_WEBDIR_ENV_VAR); if (webDirPath == null || webDirPath.length() == 0) throw new Exception("Environment variable '" + IBIOMES_LITE_WEBDIR_ENV_VAR + "' was not set"); } File dir = new File(webDirPath); if (webDirPath != null && dir.exists()) { publicHtmlFolder = dir.getCanonicalPath(); experimentListXml = new XmlExperimentListFile( publicHtmlFolder + PATH_FOLDER_SEPARATOR + IBIOMES_LITE_XML_INDEX); } else throw new FileNotFoundException( "The iBIOMES Lite web folder could not be located ('" + webDirPath + "'?)"); //check that $IBIOMES_HOME is set ibiomesHomeFolder = System.getenv(IBIOMES_HOME_ENV_VAR); if (ibiomesHomeFolder == null || ibiomesHomeFolder.length() == 0) { throw new FileNotFoundException("Environment variable '" + IBIOMES_HOME_ENV_VAR + "' was not set"); } //load iBIOMES configuration IBIOMESConfiguration ibiomesConfig = IBIOMESConfiguration.getInstance(); outputToConsole = ibiomesConfig.isOutputToConsole(); } /** * Get new iBIOMES Lite manager with default settings or current instance if it already exists. * @throws Exception */ public static IBIOMESLiteManager getInstance() throws Exception { if (ibiomesLite == null) ibiomesLite = new IBIOMESLiteManager(null); return ibiomesLite; } /** * Get new iBIOMES Lite manager with default settings or current instance if forceLoad flag is set to false. * @param forceReload Flag used to force the creation of a new instance * @throws Exception */ public static IBIOMESLiteManager getInstance(boolean forceReload) throws Exception { if ((ibiomesLite == null) || forceReload) { ibiomesLite = new IBIOMESLiteManager(null); } return ibiomesLite; } /** * Create instance of iBIOMES Lite manager by specifying the location of the web directory. * @param webDirPath Path to the iBIOMES Lite web folder * @throws Exception */ public static IBIOMESLiteManager getInstance(String webDirPath, boolean forceReload) throws Exception { if ((ibiomesLite == null) || forceReload) { ibiomesLite = new IBIOMESLiteManager(webDirPath); } return ibiomesLite; } /** * Get path to iBIOMES Lite web folder * @return path to iBIOMES Lite web folder */ public String getWebDirLocation() { return publicHtmlFolder; } /** * Publish experiment. Parse corresponding directory if descriptor files does not exist yet. * @param experimentDirPath Path to experiment directory * @throws Exception */ public void publishExperiment(String experimentDirPath) throws Exception { publishExperiment(experimentDirPath, null, null, false, 0); } /** * Publish experiment. Parse corresponding directory if descriptor files does not exist yet. * @param experimentDirPath Path to experiment directory * @param software Software context * @param xmlDescPath Path to XML file containing metadata generation rules * @throws Exception */ public void publishExperiment(String experimentDirPath, String software, String xmlDescPath, boolean isForceDescUpdate) throws Exception { this.publishExperiment(experimentDirPath, software, xmlDescPath, isForceDescUpdate, 0); } /** * Publish experiment. Parse corresponding directory if descriptor files does not exist yet. * @param experimentDirPath Path to experiment directory * @param software Software context * @param xmlDescPath Path to XML file containing metadata generation rules * @param depth * @throws Exception */ public void publishExperiment(String experimentDirPath, String software, String xmlDescPath, boolean isForceDescUpdate, int depth) throws Exception { Locker experimentLocker = null; Locker webdirLocker = null; try { //lock directory being parsed experimentLocker = new Locker(experimentDirPath); webdirLocker = new Locker(publicHtmlFolder); //check lock ... boolean isLocked = experimentLocker.lock(); if (isLocked) { File experimentDesc = new File( experimentDirPath + PATH_FOLDER_SEPARATOR + IBIOMES_DESC_FILE_TREE_FILE_NAME); if (!experimentDesc.exists() || isForceDescUpdate) { //parse directory this.parse(experimentDirPath, software, xmlDescPath, depth); } else { if (outputToConsole) System.out.println("Using existing descriptors (use -f option to force parsing)..."); } //lock web directory for updates if (outputToConsole) System.out.println("Wating for other users to finish publishing..."); isLocked = webdirLocker.waitAndLock(); //update XML file representing experiment list XmlExperimentFile experimentXml = new XmlExperimentFile(experimentDesc.getAbsolutePath()); int id = experimentListXml.addNewExperiment(experimentXml); experimentListXml.saveXml(); //create new directory for this experiment String liteDirPath = publicHtmlFolder + PATH_FOLDER_SEPARATOR + IBIOMES_LITE_EXPERIMENT_DIR + PATH_FOLDER_SEPARATOR + id; if (Files.exists(Paths.get(liteDirPath))) Utils.removeDirectoryRecursive(Paths.get(liteDirPath)); Files.createDirectory(Paths.get(liteDirPath)); //copy experiment XML descriptors String newFileTreeXmlPath = liteDirPath + "/index.xml"; String newWorkflowXmlPath = liteDirPath + "/index-details.xml"; Files.copy(Paths.get(experimentDesc.getAbsolutePath()), Paths.get(newFileTreeXmlPath), StandardCopyOption.REPLACE_EXISTING); Files.copy(Paths.get(experimentDirPath + PATH_FOLDER_SEPARATOR + IBIOMES_DESC_WORKFLOW_FILE_NAME), Paths.get(newWorkflowXmlPath), StandardCopyOption.REPLACE_EXISTING); //pull data files (csv, pdb, and images) String dataDirPath = liteDirPath + PATH_FOLDER_SEPARATOR + IBIOMES_LITE_DATA_DIR; Files.createDirectory(Paths.get(dataDirPath)); this.pullDataFilesForExperiment(newFileTreeXmlPath, newWorkflowXmlPath, liteDirPath + PATH_FOLDER_SEPARATOR + IBIOMES_LITE_DATA_DIR); //generate HTML pages if (outputToConsole) System.out.println("Generating HTML..."); //experiment summary this.transformXmlToHtml(newWorkflowXmlPath, publicHtmlFolder + PATH_FOLDER_SEPARATOR + IBIOMES_LITE_EXPERIMENT_DIR + PATH_FOLDER_SEPARATOR + id + PATH_FOLDER_SEPARATOR + "index.html", publicHtmlFolder + PATH_FOLDER_SEPARATOR + IBIOMES_EXPERIMENT_SUMMARY_XSL_PATH); //experiment workflow (tree view) this.transformXmlToHtml(newWorkflowXmlPath, publicHtmlFolder + PATH_FOLDER_SEPARATOR + IBIOMES_LITE_EXPERIMENT_DIR + PATH_FOLDER_SEPARATOR + id + PATH_FOLDER_SEPARATOR + "details.html", publicHtmlFolder + PATH_FOLDER_SEPARATOR + IBIOMES_EXPERIMENT_WORKFLOW_XSL_PATH); //experiment runs (timings, resources) this.transformXmlToHtml(newWorkflowXmlPath, publicHtmlFolder + PATH_FOLDER_SEPARATOR + IBIOMES_LITE_EXPERIMENT_DIR + PATH_FOLDER_SEPARATOR + id + PATH_FOLDER_SEPARATOR + "runs.html", publicHtmlFolder + PATH_FOLDER_SEPARATOR + IBIOMES_EXPERIMENT_RUNS_XSL_PATH); //experiment file browser this.transformXmlToHtml(newFileTreeXmlPath, publicHtmlFolder + PATH_FOLDER_SEPARATOR + IBIOMES_LITE_EXPERIMENT_DIR + PATH_FOLDER_SEPARATOR + id + PATH_FOLDER_SEPARATOR + "files.html", publicHtmlFolder + PATH_FOLDER_SEPARATOR + IBIOMES_EXPERIMENT_FILE_TREE_XSL_PATH); //list of experiments experimentListXml.saveToHTML(publicHtmlFolder + PATH_FOLDER_SEPARATOR + IBIOMES_LITE_HTML_INDEX, publicHtmlFolder + PATH_FOLDER_SEPARATOR + IBIOMES_EXPERIMENT_SET_XSL_PATH); } else { System.out.println("Could not lock " + experimentDirPath + " for parsing."); System.out.println( "Another user might be publishing this experiment right now. If you believe this is not the case, delete the " + Locker.LOCK_FILE_NAME + " file in the experiment directory and re-try."); } //unlock directories experimentLocker.unlock(); webdirLocker.unlock(); } catch (Exception e) { if (experimentLocker != null) experimentLocker.unlock(); if (webdirLocker != null) webdirLocker.unlock(); throw e; } } /** * Parse experiment directory and generate XML file * @param experimentDirPath Path to experiment directory * @param software Software context * @param xmlDescPath Path to XML file containing metadata generation rules * @param depth * @throws Exception */ public ExperimentFolder parse(String experimentDirPath, String software, String xmlDescPath, int depth) throws Exception { //read descriptor file if exists DirectoryStructureDescriptor desc = null; if (xmlDescPath != null && xmlDescPath.length() != 0) { if (outputToConsole) System.out.println("Loading parsing rules from XML descriptor..."); desc = new DirectoryStructureDescriptor(xmlDescPath); } ExperimentFolder experimentFolder = null; File dir = new File(experimentDirPath); int nFiles = Utils.countNumberOfFilesInDirectory(dir.getAbsolutePath()); //System.out.println("Parsing directory ("+nFiles+" files)..."); experimentDirPath = dir.getCanonicalPath(); ExperimentFactory expFactory = new ExperimentFactory(experimentDirPath, depth); //create listeners for progress bar List<IBIOMESListener> listeners = null; if (outputToConsole) { listeners = new ArrayList<IBIOMESListener>(); listeners.add((IBIOMESListener) new DirectoryParsingProgressBar(nFiles, "Parsing file")); } //parse directory experimentFolder = expFactory.parseDirectoryForExperimentWorkflowAndMetadata(software, desc, listeners); //create XML descriptor if (outputToConsole) System.out.println("Saving experiment file tree descriptor (" + IBIOMESLiteManager.IBIOMES_DESC_FILE_TREE_FILE_NAME + ")..."); experimentFolder.getFileDirectory().storeToXML( experimentDirPath + PATH_FOLDER_SEPARATOR + IBIOMESLiteManager.IBIOMES_DESC_FILE_TREE_FILE_NAME); //create detailed XML description of experiment if (outputToConsole) System.out.println( "Saving experiment workflow (" + IBIOMESLiteManager.IBIOMES_DESC_WORKFLOW_FILE_NAME + ")..."); this.generateDetailedXML(experimentFolder, experimentDirPath + PATH_FOLDER_SEPARATOR + IBIOMESLiteManager.IBIOMES_DESC_WORKFLOW_FILE_NAME); //create property file to store the parsing configuration if (outputToConsole) System.out.println( "Saving parser configuration (" + IBIOMESLiteManager.IBIOMES_PARSE_CONFIG_FILE_NAME + ")..."); this.createParsingConfigFile( experimentDirPath + PATH_FOLDER_SEPARATOR + IBIOMESLiteManager.IBIOMES_PARSE_CONFIG_FILE_NAME, xmlDescPath, software, depth); return experimentFolder; } /** * Pull data files (pdb and images) for a given experiment * @param fileTreeXmlPath Path to XML file representing the project file tree * @param workflowXmlPath Path to XML file representing the experiment workflow * @param dataDirPath Path to directory used to store data files * @throws SAXException * @throws IOException * @throws XPathExpressionException * @throws ParserConfigurationException * @throws TransformerException */ private void pullDataFilesForExperiment(String fileTreeXmlPath, String workflowXmlPath, String dataDirPath) throws SAXException, IOException, XPathExpressionException, ParserConfigurationException, TransformerException { if (outputToConsole) System.out.println("Copying analysis data files..."); DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance(); DocumentBuilder docBuilder = docFactory.newDocumentBuilder(); Document fileTreeDoc = docBuilder.parse(fileTreeXmlPath); fileTreeDoc = Utils.normalizeXmlDoc(fileTreeDoc); Element fileTreeRootElt = (Element) fileTreeDoc.getDocumentElement().getChildNodes().item(0); String dirPath = fileTreeRootElt.getAttribute("absolutePath"); XPathReader xreader = new XPathReader(fileTreeDoc); //load XML representation of experiment workflow Document docWorkflow = docBuilder.parse(workflowXmlPath); docWorkflow = Utils.normalizeXmlDoc(docWorkflow); Element workflowRootElt = (Element) docWorkflow.getDocumentElement(); //find main structure for display in Jmol Element jmolElt = pullJmolFile(fileTreeDoc, fileTreeRootElt, xreader, dataDirPath, dirPath); if (jmolElt != null) workflowRootElt.appendChild(docWorkflow.importNode(jmolElt, true)); //find analysis data NodeList matchingFiles = (NodeList) xreader.read("//file[AVUs/AVU[@id='" + FileMetadata.FILE_CLASS + "' and text()='" + FileMetadata.FILE_CLASS_ANALYSIS.toUpperCase() + "']]", XPathConstants.NODESET); //add publication information //Element dirNode = (Element)fileTreeDoc.getDocumentElement().getFirstChild(); //dirNode.setAttribute("publisher", workflowRootElt.getAttribute("publisher")); //dirNode.setAttribute("publicationDate", workflowRootElt.getAttribute("publicationDate")); //analysis data if (matchingFiles != null && matchingFiles.getLength() > 0) { Element dataElt = docWorkflow.createElement("analysis"); workflowRootElt.appendChild(dataElt); Element imgElt = docWorkflow.createElement("images"); Element pdbElt = docWorkflow.createElement("structures"); Element csvElts = docWorkflow.createElement("spreadsheets"); Element otherDataElts = docWorkflow.createElement("unknowns"); dataElt.appendChild(imgElt); dataElt.appendChild(csvElts); dataElt.appendChild(pdbElt); dataElt.appendChild(otherDataElts); PlotGenerator plotTool = new PlotGenerator(); for (int f = 0; f < matchingFiles.getLength(); f++) { Element fileNode = (Element) matchingFiles.item(f); String dataFilePath = fileNode.getAttribute("absolutePath"); //copy file String dataFileNewName = dataFilePath.substring(dirPath.length() + 1) .replaceAll(PATH_FOLDER_SEPARATOR_REGEX, "_"); String dataFileDestPath = dataDirPath + PATH_FOLDER_SEPARATOR + dataFileNewName; Files.copy(Paths.get(dataFilePath), Paths.get(dataFileDestPath), StandardCopyOption.REPLACE_EXISTING); //set read permissions if (!Utils.isWindows()) { HashSet<PosixFilePermission> permissions = new HashSet<PosixFilePermission>(); permissions.add(PosixFilePermission.OWNER_READ); permissions.add(PosixFilePermission.OWNER_WRITE); permissions.add(PosixFilePermission.OWNER_EXECUTE); permissions.add(PosixFilePermission.GROUP_READ); permissions.add(PosixFilePermission.OTHERS_READ); Files.setPosixFilePermissions(Paths.get(dataFileDestPath), permissions); } //read file AVUs NodeList avuNodes = (NodeList) xreader.read("//file[@absolutePath='" + dataFilePath + "']/AVUs/AVU", XPathConstants.NODESET); MetadataAVUList avuList = new MetadataAVUList(); if (avuNodes != null) { for (int a = 0; a < avuNodes.getLength(); a++) { Element avuNode = (Element) avuNodes.item(a); avuList.add(new MetadataAVU(avuNode.getAttribute("id").toUpperCase(), avuNode.getFirstChild().getNodeValue())); } } //add reference in XML doc String description = avuList.getValue(FileMetadata.FILE_DESCRIPTION); String format = fileNode.getAttribute("format"); if (IBIOMESFileGroup.isJmolFile(format)) { Element jmolFileElt = docWorkflow.createElement("structure"); jmolFileElt.setAttribute("path", dataFileNewName); if (description != null && description.length() > 0) jmolFileElt.setAttribute("description", description); pdbElt.appendChild(jmolFileElt); } else if (format.equals(LocalFile.FORMAT_CSV)) { Element csvElt = docWorkflow.createElement("spreadsheet"); csvElt.setAttribute("path", dataFileNewName); if (description != null && description.length() > 0) csvElt.setAttribute("description", description); csvElts.appendChild(csvElt); //try to generate plot and save image try { String imgPath = dataFileNewName + "_plot.png"; String plotType = generatePlotForCSV(plotTool, dataFileDestPath, avuList, dataFileDestPath + "_plot", "png"); csvElt.setAttribute("plotPath", imgPath); if (outputToConsole) { if (plotType == null) plotType = ""; else plotType += " "; System.out.println("\t" + plotType + "plot generated for " + dataFileNewName); } } catch (Exception e) { if (outputToConsole) System.out.println( "Warning: Plot for '" + dataFileDestPath + "' could not be generated."); try { if (IBIOMESConfiguration.getInstance().isOutputErrorStackToConsole()) e.printStackTrace(); } catch (Exception e1) { } } } else if (IBIOMESFileGroup.isImageFile(format)) { Element imgFileElt = docWorkflow.createElement("image"); imgFileElt.setAttribute("path", dataFileNewName); if (description != null && description.length() > 0) imgFileElt.setAttribute("description", description); imgElt.appendChild(imgFileElt); } else { Element otherFileElt = docWorkflow.createElement("unknown"); otherFileElt.setAttribute("path", dataFileNewName); if (description != null && description.length() > 0) otherFileElt.setAttribute("description", description); imgElt.appendChild(otherDataElts); } } } //update XML files File outputXmlAvusFile = new File(fileTreeXmlPath); if (outputXmlAvusFile.exists()) outputXmlAvusFile.delete(); File outputXmlWorkflowFile = new File(workflowXmlPath); if (outputXmlWorkflowFile.exists()) outputXmlWorkflowFile.delete(); TransformerFactory transformerFactory = TransformerFactory.newInstance(); Transformer transformer = transformerFactory.newTransformer(); transformer.setOutputProperty(OutputKeys.INDENT, "yes"); transformer.setOutputProperty(OutputKeys.ENCODING, "ISO-8859-1"); transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2"); DOMSource source = new DOMSource(fileTreeDoc); StreamResult result = null; result = new StreamResult(fileTreeXmlPath); transformer.transform(source, result); source = new DOMSource(docWorkflow); result = null; result = new StreamResult(outputXmlWorkflowFile); transformer.transform(source, result); } /** * Copy file that will displayed in Jmol * @param doc XML document * @param rootElt Root element * @param xreader XPath reader for the document * @param dataDirPath Path to directory that contains analysis data * @param dirPath Path to experiment directory * @return XML element for Jmol data * @throws IOException */ private Element pullJmolFile(Document doc, Node rootElt, XPathReader xreader, String dataDirPath, String dirPath) throws IOException { Element jmolElt = doc.createElement("jmol"); String mainStructureRelPath = (String) xreader .read("ibiomes/directory/AVUs/AVU[@id='MAIN_3D_STRUCTURE_FILE']", XPathConstants.STRING); if (mainStructureRelPath != null && mainStructureRelPath.length() > 0) { String dataFileNewName = mainStructureRelPath.replaceAll(PATH_FOLDER_SEPARATOR_REGEX, "_"); String dataFileDestPath = dataDirPath + PATH_FOLDER_SEPARATOR + dataFileNewName; Files.copy(Paths.get(dirPath + PATH_FOLDER_SEPARATOR + mainStructureRelPath), Paths.get(dataFileDestPath), StandardCopyOption.REPLACE_EXISTING); //set read permissions if (!Utils.isWindows()) { Set<PosixFilePermission> permissions = new HashSet<PosixFilePermission>(); permissions.add(PosixFilePermission.OWNER_READ); permissions.add(PosixFilePermission.OWNER_WRITE); permissions.add(PosixFilePermission.OWNER_EXECUTE); permissions.add(PosixFilePermission.GROUP_READ); permissions.add(PosixFilePermission.OTHERS_READ); Files.setPosixFilePermissions(Paths.get(dataFileDestPath), permissions); } jmolElt.setAttribute("path", dataFileNewName); jmolElt.setAttribute("name", mainStructureRelPath); NodeList avuNodes = (NodeList) xreader.read("//file[@absolutePath='" + dirPath + PATH_FOLDER_SEPARATOR + mainStructureRelPath + "']/AVUs/AVU", XPathConstants.NODESET); MetadataAVUList avuList = parseMetadata(avuNodes); String description = avuList.getValue(FileMetadata.FILE_DESCRIPTION); if (description != null && description.length() > 0) jmolElt.setAttribute("description", description); rootElt.appendChild(jmolElt); return jmolElt; } else return null; } private MetadataAVUList parseMetadata(NodeList avuNodes) { MetadataAVUList avuList = new MetadataAVUList(); if (avuNodes != null) { for (int a = 0; a < avuNodes.getLength(); a++) { Element avuNode = (Element) avuNodes.item(a); avuList.add(new MetadataAVU(avuNode.getAttribute("id").toUpperCase(), avuNode.getFirstChild().getNodeValue())); } } return avuList; } /** * Generate plot image for given CSV file * @param plotTool Plot generator * @param csvPath Path to CSV file to plot * @param avuList List of metadata for the file * @param imagePath Path to the output image * @param format Format of the image * @return type of the plot * @throws Exception */ public static String generatePlotForCSV(PlotGenerator plotTool, String csvPath, MetadataAVUList avuList, String imagePath, String format) throws Exception { String selectedChartType = avuList.getValue(CSVFile.PREFERRED_PLOT_TYPE); String title = avuList.getValue(FileMetadata.FILE_DESCRIPTION); String units = avuList.getValue(CSVFile.DATA_UNITS); String axisLabels = avuList.getValue(CSVFile.DATA_LABELS); String logScaleXStr = avuList.getValue(CSVFile.SCALE_LOG_X).trim().toLowerCase(); String logScaleYStr = avuList.getValue(CSVFile.SCALE_LOG_Y).trim().toLowerCase(); boolean logScaleX = logScaleXStr.matches("(true)|(yes)"); boolean logScaleY = logScaleYStr.matches("(true)|(yes)"); String seriesLabelList = avuList.getValue(CSVFile.SERIES_LABELS); String[] seriesLabels = null; if (seriesLabelList != null && seriesLabelList.length() > 0) { seriesLabels = seriesLabelList.split("\\,"); } //parse axis labels String[] labelValues = null; String[] unitsValues = null; String xTitle = null, yTitle = null, zTitle = null; String xUnit = null, yUnit = null, zUnit = null; if (axisLabels != null) { labelValues = axisLabels.split("\\,"); if (labelValues.length > 0) xTitle = labelValues[0]; if (labelValues.length > 1) yTitle = labelValues[1]; if (labelValues.length > 2) zTitle = labelValues[2]; } if (xTitle == null) xTitle = "x"; if (yTitle == null) yTitle = "y"; if (zTitle == null) zTitle = "z"; //parse units if (units != null) { unitsValues = units.split(","); if (unitsValues.length > 0) xUnit = unitsValues[0]; if (unitsValues.length > 1) yUnit = unitsValues[1]; if (unitsValues.length > 2) zUnit = unitsValues[2]; } if (xUnit != null && xUnit.length() > 0) xTitle = xTitle + " (" + xUnit + ")"; if (yUnit != null && yUnit.length() > 0) yTitle = yTitle + " (" + yUnit + ")"; if (zUnit != null && zUnit.length() > 0) zTitle = zTitle + " (" + zUnit + ")"; ColumnDataFile csv = new ColumnDataFile(new File(csvPath)); //create plot JFreeChart chart = plotTool.createPlot(csv, selectedChartType, title, xTitle, yTitle, zTitle, seriesLabels, logScaleX, logScaleY); //save as image plotTool.createImage(chart, PLOT_WIDTH, PLOT_HEIGHT, imagePath, format); return selectedChartType; } /** * Remove existing XML and HTML files * @throws Exception */ public int cleanData() throws Exception { int nDelete = 0; if (this.outputToConsole) System.out.println("Removing experiments..."); //remove main XML file Path indexXmlPath = Paths.get(publicHtmlFolder + PATH_FOLDER_SEPARATOR + IBIOMES_LITE_XML_INDEX); if (Files.exists(indexXmlPath, LinkOption.NOFOLLOW_LINKS)) Files.delete(indexXmlPath); this.experimentListXml = new XmlExperimentListFile( publicHtmlFolder + PATH_FOLDER_SEPARATOR + IBIOMES_LITE_XML_INDEX); //remove main HTML file Path indexHtmlPath = Paths.get(publicHtmlFolder + PATH_FOLDER_SEPARATOR + IBIOMES_LITE_HTML_INDEX); if (Files.exists(indexHtmlPath, LinkOption.NOFOLLOW_LINKS)) Files.delete(indexHtmlPath); //remove experiments data (XML, HTML, files) File htmlDir = new File(publicHtmlFolder + PATH_FOLDER_SEPARATOR + IBIOMES_LITE_EXPERIMENT_DIR); File[] htmlFiles = htmlDir.listFiles(); for (int f = 0; f < htmlFiles.length; f++) { Utils.removeDirectoryRecursive(Paths.get(htmlFiles[f].getCanonicalPath())); nDelete++; } return nDelete; } /** * Remove XML and HTML files for a given experiment * @throws TransformerException * @throws IOException * @throws SAXException * @throws ParserConfigurationException * @throws XPathExpressionException */ public int removeExperiment(String experimentPath) throws TransformerException, IOException, XPathExpressionException, ParserConfigurationException, SAXException { File experimentDir = new File(experimentPath); if (!experimentDir.exists()) return -1; experimentPath = experimentDir.getCanonicalPath(); int experimentId = this.experimentListXml.removeExperiment(experimentPath); if (experimentId != -1) { //update XML list and regenerate HTML this.experimentListXml.saveXml(); this.experimentListXml.saveToHTML(publicHtmlFolder + PATH_FOLDER_SEPARATOR + IBIOMES_LITE_HTML_INDEX, publicHtmlFolder + PATH_FOLDER_SEPARATOR + IBIOMES_EXPERIMENT_SET_XSL_PATH); //delete experiment folder in iBIOMES Lite Path expWebDirPath = Paths.get(publicHtmlFolder + PATH_FOLDER_SEPARATOR + IBIOMES_LITE_EXPERIMENT_DIR + PATH_FOLDER_SEPARATOR + experimentId); if (Files.exists(expWebDirPath, LinkOption.NOFOLLOW_LINKS)) Utils.removeDirectoryRecursive(expWebDirPath); } return experimentId; } /** * Regenerate descriptors for the experiments contained in the given file tree. * @return List of descriptor paths * @throws Exception * @throws IOException */ public List<String> regeneratePublishedDescriptors() throws Exception { String descriptorPath = null; int depth = 0; List<String> descriptorsPath = experimentListXml.getExperimentList(); List<ExperimentFolder> experiments = experimentListXml.readExperiments(this.outputToConsole); for (ExperimentFolder experiment : experiments) { try { //load parsing configuration previously used String parsingRuleSetFilePath = null; String softwareContext = null; Properties parsingConfig = loadParsingConfigFile(experiment.getFileDirectory().getAbsolutePath() + PATH_FOLDER_SEPARATOR + IBIOMES_PARSE_CONFIG_FILE_NAME); if (parsingConfig != null) { parsingRuleSetFilePath = parsingConfig.getProperty(PARSER_CONFIG_PROPERTY_RULE_SET_FILE); softwareContext = parsingConfig.getProperty(PARSER_CONFIG_PROPERTY_SOFTWARE_CONTEXT); try { depth = Integer .parseInt(parsingConfig.getProperty(PARSER_CONFIG_PROPERTY_SOFTWARE_CONTEXT)); } catch (NumberFormatException e) { //TODO } } //parse descriptorPath = experiment.getFileDirectory().getAbsolutePath() + PATH_FOLDER_SEPARATOR + IBIOMES_DESC_FILE_TREE_FILE_NAME; this.parse(experiment.getFileDirectory().getAbsolutePath(), softwareContext, parsingRuleSetFilePath, depth); experiment.getFileDirectory().storeToXML(descriptorPath); } catch (Exception e) { //logger System.err .println("[ERROR] Error when updating '" + descriptorPath + "':" + e.getLocalizedMessage()); } } return descriptorsPath; } /** * Update web content based on descriptors that have already been published. * @return List of updated experiments * @throws Exception */ public List<String> updateWebContent() throws Exception { //retrieve list of experiments already published List<String> experimentsPath = experimentListXml.getExperimentList(); //clean this.cleanData(); //retrieve descriptors (or parse if necessary) and data for (String experimentPath : experimentsPath) { this.publishExperiment(experimentPath); } return experimentsPath; } /** * Generate PDF document with experiment metadata and available images * @param outputPath */ public void generatePdf(String outputPath, String experimentDirPath, String software, String xmlDescPath, boolean isForceDescUpdate, int depth) throws Exception { File experimentDesc = new File( experimentDirPath + PATH_FOLDER_SEPARATOR + IBIOMES_DESC_FILE_TREE_FILE_NAME); /*if (!experimentDesc.exists() || isForceDescUpdate){ //parse directory this.parse(experimentDirPath, software, xmlDescPath, depth); }*/ //load experiment descriptor XmlExperimentFile experimentXml = new XmlExperimentFile(experimentDesc.getAbsolutePath()); ExperimentFolder experiment = experimentXml.readExperiment(); //store to PDF document MetadataLookup lookupdIndex = new MetadataLookup( ibiomesHomeFolder + PATH_FOLDER_SEPARATOR + METADATA_LOOKUP_INDEX_PATH); ExperimentTransformer transformer = new ExperimentTransformer(lookupdIndex); transformer.storeExperimentAsPDF(experiment, outputPath); } /** * Store experiment details as XML file * @param outputFilePath Path to XML file * @throws Exception * @throws IllegalAccessException * @throws IllegalArgumentException */ public void generateDetailedXML(ExperimentFolder experimentFolder, String outputFilePath) throws IllegalArgumentException, IllegalAccessException, Exception { XMLConverter converter = new XMLConverter(); Document doc = converter.convertExperiment(experimentFolder); DOMSource source = new DOMSource(doc); TransformerFactory transformerFactory = TransformerFactory.newInstance(); Transformer transformer = transformerFactory.newTransformer(); transformer.setOutputProperty(OutputKeys.INDENT, "no"); //transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2"); File outputXMLFile = new File(outputFilePath); StreamResult result = new StreamResult(outputXMLFile); transformer.transform(source, result); } /** * Store experiment details as HTML file * @param outputFilePath Path to HTML file * @throws Exception * @throws IllegalAccessException * @throws IllegalArgumentException */ public void transformXmlToHtml(String experimentDescPath, String outputFilePath, String xslUrl) throws IllegalArgumentException, IllegalAccessException, Exception { //load XML DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); DocumentBuilder db = dbf.newDocumentBuilder(); Document doc = db.parse(experimentDescPath); DOMSource source = new DOMSource(doc); //transform XML to HTML System.setProperty("javax.xml.transform.TransformerFactory", "net.sf.saxon.TransformerFactoryImpl"); TransformerFactory tFactory = TransformerFactory.newInstance(); Transformer transformer = tFactory.newTransformer(new javax.xml.transform.stream.StreamSource(xslUrl)); //transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes"); transformer.setOutputProperty(OutputKeys.INDENT, "no"); File outputHTMLFile = new File(outputFilePath); StreamResult result = new StreamResult(outputHTMLFile); transformer.transform(source, result); } private void createParsingConfigFile(String configFilePath, String parsingRuleSetFilePath, String softwareContext, int depth) { Properties prop = new Properties(); try { //set the properties value if (parsingRuleSetFilePath != null && parsingRuleSetFilePath.length() > 0) prop.setProperty(PARSER_CONFIG_PROPERTY_RULE_SET_FILE, parsingRuleSetFilePath); if (softwareContext != null && softwareContext.length() > 0) prop.setProperty(PARSER_CONFIG_PROPERTY_SOFTWARE_CONTEXT, softwareContext); if (softwareContext != null && softwareContext.length() > 0) prop.setProperty(PARSER_CONFIG_PROPERTY_DEPTH_INDEPENDENCE, String.valueOf(depth)); //save properties to project root folder prop.store(new FileOutputStream(configFilePath), null); } catch (IOException ex) { ex.printStackTrace(); } } private Properties loadParsingConfigFile(String configFilePath) { Properties prop = new Properties(); try { //load a properties file prop.load(new FileInputStream(configFilePath)); return prop; } catch (IOException ex) { ex.printStackTrace(); return null; } } /** * Search experiments by keywords * @param keywords Keywords * @return List of matching experiments * @throws SAXException * @throws ParserConfigurationException * @throws IOException * @throws XPathExpressionException */ public IBIOMESLiteSearchRecordSet search(String[] keywords) throws XPathExpressionException, IOException, ParserConfigurationException, SAXException { XmlExperimentListFile xmlList = new XmlExperimentListFile( publicHtmlFolder + PATH_FOLDER_SEPARATOR + IBIOMES_LITE_XML_INDEX); return xmlList.search(keywords); } /** * get version of iBIOMES Lite * @return version */ public static String getVersion() { return IBIOMES_LITE_VERSION + " [" + Utils.getOperatingSystem() + "]"; } }