Java tutorial
/************************************************************************* * * * This file is part of the 20n/act project. * * 20n/act enables DNA prediction for synthetic biology/bioengineering. * * Copyright (C) 2017 20n Labs, Inc. * * * * Please direct all queries to act@20n.com. * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation, either version 3 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see <http://www.gnu.org/licenses/>. * * * *************************************************************************/ package act.installer.pubchem; import act.server.MongoDB; import act.shared.Chemical; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.DefaultParser; import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.jaxen.JaxenException; import org.jaxen.dom.DOMXPath; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.Node; import org.w3c.dom.Text; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import javax.xml.stream.XMLEventReader; import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLStreamConstants; import javax.xml.stream.XMLStreamException; import javax.xml.stream.events.Characters; import javax.xml.stream.events.XMLEvent; import javax.xml.xpath.XPathExpressionException; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import java.util.zip.GZIPInputStream; public class PubchemParser { private static final Logger LOGGER = LogManager.getFormatterLogger(PubchemParser.class); private static final String OPTION_DATA_DIRECTORY = "i"; private static final String OPTION_DB = "o"; private static final String GZIP_FILE_EXT = ".gz"; private static final String COMPOUND_DOC_TAG = "PC-Compound"; private static final int GZIP_BUFFER_SIZE = 1 << 27; // ~128MB of buffer space to help GZip really move. private static final boolean ENABLE_XML_STREAM_TEXT_COALESCING = true; public static final HelpFormatter HELP_FORMATTER = new HelpFormatter(); static { HELP_FORMATTER.setWidth(100); } public static final String HELP_MESSAGE = "This class is used for parsing xml files and storing them in a db"; public static final List<Option.Builder> OPTION_BUILDERS = new ArrayList<Option.Builder>() { { add(Option.builder(OPTION_DATA_DIRECTORY).argName("OPTION_DATA_DIRECTORY") .desc("The data directory where the pubchem files live").hasArg().required() .type(String.class)); add(Option.builder(OPTION_DB).argName("OPTION_DB").desc("The db to save the results in").hasArg() .required().type(String.class)); add(Option.builder("h").argName("help").desc("Prints this help message").longOpt("help")); } }; /** * Key terminologies for this file: * * a) An example of how an inchi is packaged in the XML document * <PC-InfoData> <PC-InfoData_urn> <PC-Urn> <PC-Urn_label>InChI</PC-Urn_label> <PC-Urn_name>Standard</PC-Urn_name> <PC-Urn_datatype> <PC-UrnDataType value="string">1</PC-UrnDataType> </PC-Urn_datatype> <PC-Urn_version>1.0.4</PC-Urn_version> <PC-Urn_software>InChI</PC-Urn_software> <PC-Urn_source>iupac.org</PC-Urn_source> <PC-Urn_release>2012.11.26</PC-Urn_release> </PC-Urn> </PC-InfoData_urn> <PC-InfoData_value> <PC-InfoData_value_sval>InChI=1S/C12H17FO/c1-12(2,3)10-6-4-9(5-7-10)11(13)8-14/h4-7,11,14H,8H2,1-3H3</PC-InfoData_value_sval> </PC-InfoData_value> </PC-InfoData> * * In order to parse the inchi, we first detect the element <PC-Urn_label>InChI</PC-Urn_label>, since it tells us this * node is an inchi node. The element PC-Urn_label is called a "ResourceName". The term "inchi", a value of the resource * name node, is called a "ResourceValue". For element <PC-Urn_label>InChI</PC-Urn_label>, two xml events occurs in sequence: * 1) START ELEMENT 2) CHARACTERS. We identify a ResourceName in (1) and ResourceValue in (2). In this case, we have two * nodes of interest: * a) <PC-Urn_label>InChI</PC-Urn_label> * b) <PC-InfoData_value_sval>InChI=1S/C12H17FO/c1-12(2,3)10-6-4-9(5-7-10)11(13)8-14/h4-7,11,14H,8H2,1-3H3</PC-InfoData_value_sval> * * Once we detect the node as an inchi node from a), we can easily detect the ResourceName "PC-InfoData_value_sval" and * store it's value for b). We call PC-Urn_label "PUBCHEM_KEY" since it is a key to the value of the inchi, whereas * PC-InfoData_value_sval is the "PUBCHEM_VALUE" since it is the value to the inchi. */ private enum PC_XPATHS { /* Structure: <feature name>_<level>_[_<sub-feature or structure>]_<type> * [IUPAC_NAME]_[L1]_[NODES]: nodes in the original document (L1) that correspond to IUPAC name entries. * [IUPAC_NAME]_[L2]_[VALUE]_[TEXT]: textual names in the IUPAC name sub-tree (L2). */ IUPAC_NAME_L1_NODES( "/PC-Compound/PC-Compound_props/PC-InfoData[./PC-InfoData_urn/PC-Urn/PC-Urn_label/text()=\"IUPAC Name\"]"), IUPAC_NAME_L2_TYPE_TEXT( "/PC-InfoData/PC-InfoData_urn/PC-Urn/PC-Urn_name/text()"), IUPAC_NAME_L2_VALUE_TEXT( "/PC-InfoData/PC-InfoData_value/PC-InfoData_value_sval/text()"), // TODO: ensure there is exactly one id_cid per compound. PC_ID_L1_TEXT( "/PC-Compound/PC-Compound_id/PC-CompoundType/PC-CompoundType_id/PC-CompoundType_id_cid/text()"), INCHI_L1_NODES( "/PC-Compound/PC-Compound_props/PC-InfoData[./PC-InfoData_urn/PC-Urn/PC-Urn_label/text()=\"InChI\"]"), /* We could just use //PC-InfoData[./PC-InfoData_urn//PC-Urn_label/text()="InChI"]//PC-InfoData_value_sval/text() * but we split the InChI parsing into two pieces in case there are multiple InChI entries (which would be insane). */ INCHI_L2_TEXT("/PC-InfoData/PC-InfoData_value/PC-InfoData_value_sval/text()"), SMILES_L1_NODES("//PC-InfoData[./PC-InfoData_urn//PC-Urn_label/text()=\"SMILES\"]"), SMILES_L2_TEXT( "//PC-InfoData_value_sval/text()"),; private String path; PC_XPATHS(String path) { this.path = path; } public String getPath() { return path; } DOMXPath compile() throws JaxenException { return new DOMXPath(this.getPath()); } } private static final Set<String> TAGS_TO_INCLUDE = Collections.unmodifiableSet(new HashSet<String>() { { add("PC-Compound_id"); add("PC-Compound_props"); } }); private final Map<PC_XPATHS, DOMXPath> xpaths = new HashMap<>(PC_XPATHS.values().length); private MongoDB db; private DocumentBuilder documentBuilder; private XMLInputFactory xmlInputFactory; public PubchemParser(MongoDB db) { this.db = db; } /** * Initializes a PubchemParser. Must be called before the PubchemParser can be used. * @throws XPathExpressionException * @throws ParserConfigurationException */ public void init() throws ParserConfigurationException, JaxenException { // Would rather do this in its own block, but have to handle the XPath exception. :( for (PC_XPATHS x : PC_XPATHS.values()) { xpaths.put(x, x.compile()); } DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); documentBuilder = factory.newDocumentBuilder(); xmlInputFactory = XMLInputFactory.newInstance(); /* Configure the XMLInputFactory to return event streams that coalesce consecutive character events. Without this * we can end up with malformed names and InChIs, as XPath will only fetch the first text node if there are several * text children under one parent. */ xmlInputFactory.setProperty(XMLInputFactory.IS_COALESCING, ENABLE_XML_STREAM_TEXT_COALESCING); if ((Boolean) xmlInputFactory.getProperty(XMLInputFactory.IS_COALESCING)) { LOGGER.info("Successfully configured XML stream to coalesce character elements."); } else { LOGGER.error("Unable to configure XML stream to coalesce character elements."); } } /** * This function writes a chemical record to the DB. * @param chemical Chemical to be written to the DB. */ private void writeChemicalToDB(Chemical chemical) { Long id = db.getNextAvailableChemicalDBid(); db.submitToActChemicalDB(chemical, id); } /** * Extracts compound features from a sub-document/sub-tree containing one PC-Compound element. Nodes that contain * interesting features are found and their text extracted using XPath. * * @param d The document from which to extract features. * @return A PubchemEntry object corresponding to features from one PC-Compound document. * @throws XPathExpressionException */ private PubchemEntry extractPCCompoundFeatures(Document d) throws JaxenException { Long id = Long.valueOf(xpaths.get(PC_XPATHS.PC_ID_L1_TEXT).stringValueOf(d)); PubchemEntry entry = new PubchemEntry(id); // Jaxen's API is from a pre-generics age! List<Node> nodes = (List<Node>) xpaths.get(PC_XPATHS.IUPAC_NAME_L1_NODES).selectNodes(d); if (nodes.size() == 0) { LOGGER.warn("No names available for compound %d", id); } for (Node n : nodes) { /* In order to run XPath on a sub-document, we have to Extract the relevant nodes into their own document object. * If we try to run evaluate on `n` instead of this new document, we'll get matching paths for the original * document `d` but not for the nodes we're looking at right now. Very weird. * TODO: remember this way of running XPath on documents the next time we need to write an XML parser. */ Document iupacNameDoc = documentBuilder.newDocument(); iupacNameDoc.adoptNode(n); iupacNameDoc.appendChild(n); String type = xpaths.get(PC_XPATHS.IUPAC_NAME_L2_TYPE_TEXT).stringValueOf(iupacNameDoc); String value = xpaths.get(PC_XPATHS.IUPAC_NAME_L2_VALUE_TEXT).stringValueOf(iupacNameDoc); entry.setNameByType(type, value); } // We really need an InChI for a chemical to make sense, so log errors if we can't find one. boolean hasInChI = false; nodes = xpaths.get(PC_XPATHS.INCHI_L1_NODES).selectNodes(d); if (nodes.size() == 0) { LOGGER.warn("Found chemical (%d) with no InChIs, hoping for SMILES instead", id); } else if (nodes.size() > 1) { LOGGER.error("Assumption violation: found chemical with multiple InChIs (%d), skipping", id); return null; } else { hasInChI = true; Node n = nodes.get(0); Document inchiDoc = documentBuilder.newDocument(); inchiDoc.adoptNode(n); inchiDoc.appendChild(n); String value = xpaths.get(PC_XPATHS.INCHI_L2_TEXT).stringValueOf(inchiDoc); entry.setInchi(value); } nodes = xpaths.get(PC_XPATHS.SMILES_L1_NODES).selectNodes(d); if (nodes.size() == 0) { if (hasInChI) { LOGGER.warn("Found chemical (%d) with no SMILES, using only InChI"); } else { LOGGER.warn("Found chemical (%d) with no InChI or SMILES, skipping"); return null; } } else { for (Node n : nodes) { Document smilesDoc = documentBuilder.newDocument(); smilesDoc.adoptNode(n); smilesDoc.appendChild(n); String smiles = xpaths.get(PC_XPATHS.SMILES_L2_TEXT).stringValueOf(smilesDoc); entry.appendSmiles(smiles); } } return entry; } /** * Incrementally parses a stream of XML events from a PubChem file, extracting the next available PC-Compound entry * as a Chemical object. * @param eventReader The xml event reader we are parsing the XML from * @return The constructed chemical * @throws XMLStreamException * @throws XPathExpressionException */ public Chemical extractNextChemicalFromXMLStream(XMLEventReader eventReader) throws XMLStreamException, JaxenException { Document bufferDoc = null; Element currentElement = null; StringBuilder textBuffer = null; /* With help from * http://stackoverflow.com/questions/7998733/loading-local-chunks-in-dom-while-parsing-a-large-xml-file-in-sax-java */ while (eventReader.hasNext()) { XMLEvent event = eventReader.nextEvent(); switch (event.getEventType()) { case XMLStreamConstants.START_ELEMENT: String eventName = event.asStartElement().getName().getLocalPart(); if (COMPOUND_DOC_TAG.equals(eventName)) { // Create a new document if we've found the start of a compound object. bufferDoc = documentBuilder.newDocument(); currentElement = bufferDoc.createElement(eventName); bufferDoc.appendChild(currentElement); } else if (currentElement != null) { // Wait until we've found a compound entry to start slurping up data. // Create a new child element and push down the current pointer when we find a new node. Element newElement = bufferDoc.createElement(eventName); currentElement.appendChild(newElement); currentElement = newElement; } // If we aren't in a PC-Compound tree, we just let the elements pass by. break; case XMLStreamConstants.CHARACTERS: if (currentElement == null) { // Ignore this event if we're not in a PC-Compound tree. continue; } Characters chars = event.asCharacters(); // Ignore only whitespace strings, which just inflate the size of the DOM. Text coalescing makes this safe. if (chars.isWhiteSpace()) { continue; } // Rely on the XMLEventStream to coalesce consecutive text events. Text textNode = bufferDoc.createTextNode(chars.getData()); currentElement.appendChild(textNode); break; case XMLStreamConstants.END_ELEMENT: if (currentElement == null) { // Ignore this event if we're not in a PC-Compound tree. continue; } eventName = event.asEndElement().getName().getLocalPart(); Node parentNode = currentElement.getParentNode(); if (parentNode instanceof Element) { currentElement = (Element) parentNode; } else if (parentNode instanceof Document && eventName.equals(COMPOUND_DOC_TAG)) { // We're back at the top of the node stack! Convert the buffered document into a Chemical. PubchemEntry entry = extractPCCompoundFeatures(bufferDoc); if (entry != null) { return entry.asChemical(); } else { // Skip this entry if we can't process it correctly by resetting the world and continuing on. bufferDoc = null; currentElement = null; } } else { // This should not happen, but is here as a sanity check. throw new RuntimeException(String.format("Parent of XML element %s is of type %d, not Element", currentElement.getTagName(), parentNode.getNodeType())); } break; // TODO: do we care about attributes or other XML structures? } } // Return null when we run out of chemicals, just like readLine(). return null; } /** * This function reads a given gzipped XML file, passes the xml event stream to a function to parse out the chemical, * and writes the chemical to the db. * @param file The input gzipped file that is being processed. * @throws XMLStreamException * @throws IOException */ public void openCompressedXMLFileAndWriteChemicals(File file) throws XMLStreamException, JaxenException, IOException { XMLEventReader eventReader = xmlInputFactory .createXMLEventReader(new GZIPInputStream(new FileInputStream(file), GZIP_BUFFER_SIZE)); Chemical result; while ((result = extractNextChemicalFromXMLStream(eventReader)) != null) { writeChemicalToDB(result); } } /** * This function is the main driver of the class, which processes all the xml files of the pubchem dump. * @throws XMLStreamException * @throws IOException */ private void run(List<File> filesToProcess) throws XMLStreamException, JaxenException, IOException { int counter = 1; for (File file : filesToProcess) { LOGGER.info("Processing file %d of %d", counter, filesToProcess.size()); LOGGER.info("File name is %s", file.getPath()); openCompressedXMLFileAndWriteChemicals(file); counter++; } } /** * This function extracts gzipped xml files from a file directory. * @param dataDirectory The directory of interest. * @return A list of files of gzipped xml files. * @throws XMLStreamException * @throws IOException */ private static List<File> findGZippedFilesInDirectory(String dataDirectory) throws XMLStreamException, IOException { File folder = new File(dataDirectory); if (!folder.exists()) { String msg = String.format("The folder %s does not exists", folder.getAbsolutePath()); LOGGER.error(msg); throw new RuntimeException(msg); } List<File> result = Arrays.stream(folder.listFiles()).filter(f -> f.getName().endsWith(GZIP_FILE_EXT)) .collect(Collectors.toList()); // Sort files lexicographically for installer stability. Collections.sort(result); return result; } public static void main(String[] args) throws Exception { // Parse the command line options Options opts = new Options(); for (Option.Builder b : OPTION_BUILDERS) { opts.addOption(b.build()); } CommandLine cl = null; try { CommandLineParser parser = new DefaultParser(); cl = parser.parse(opts, args); } catch (ParseException e) { LOGGER.error("Argument parsing failed: %s\n", e.getMessage()); HELP_FORMATTER.printHelp(PubchemParser.class.getCanonicalName(), HELP_MESSAGE, opts, null, true); System.exit(1); } if (cl.hasOption("help")) { HELP_FORMATTER.printHelp(PubchemParser.class.getCanonicalName(), HELP_MESSAGE, opts, null, true); return; } String dataDir = cl.getOptionValue(OPTION_DATA_DIRECTORY); String dbName = cl.getOptionValue(OPTION_DB); MongoDB db = new MongoDB("localhost", 27017, dbName); PubchemParser pubchemParser = new PubchemParser(db); pubchemParser.init(); pubchemParser.run(findGZippedFilesInDirectory(dataDir)); } }