Java tutorial
/************************************************************************* * * * This file is part of the 20n/act project. * * 20n/act enables DNA prediction for synthetic biology/bioengineering. * * Copyright (C) 2017 20n Labs, Inc. * * * * Please direct all queries to act@20n.com. * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation, either version 3 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see <http://www.gnu.org/licenses/>. * * * *************************************************************************/ package com.act.lcms; import org.apache.commons.io.input.ReaderInputStream; import org.apache.commons.lang3.tuple.Pair; import org.w3c.dom.Document; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import javax.xml.stream.XMLEventReader; import javax.xml.stream.XMLEventWriter; import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLOutputFactory; import javax.xml.stream.XMLStreamException; import javax.xml.stream.events.XMLEvent; import javax.xml.xpath.XPath; import javax.xml.xpath.XPathConstants; import javax.xml.xpath.XPathException; import javax.xml.xpath.XPathFactory; import java.io.FileInputStream; import java.io.StringReader; import java.io.StringWriter; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.ArrayList; import java.util.Base64; import java.util.Iterator; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.io.IOException; public abstract class MzMLParser<S> { public static final String SPECTRUM_OBJECT_TAG = "spectrum"; public static final String XML_PREAMBLE = "<?xml version=\"1.0\" encoding=\"utf-8\"?>"; // Paths for data extraction. public static final String SPECTRUM_PATH_INDEX = "/spectrum/@index"; public static final String SPECTRUM_PATH_ID = "/spectrum/@id"; public static final String SPECTRUM_PATH_BASE_PEAK_MZ = "/spectrum/cvParam[@name='base peak m/z']/@value"; public static final String SPECTRUM_PATH_BASE_PEAK_INTENSITY = "/spectrum/cvParam[@name='base peak intensity']/@value"; public static final String SPECTRUM_PATH_SCAN_START_TIME = "/spectrum/scanList/scan/cvParam[@name='scan start time']/@value"; public static final String SPECTRUM_PATH_SCAN_START_TIME_UNIT = "/spectrum/scanList/scan/cvParam[@name='scan start time']/@unitName"; public static final String SPECTRUM_PATH_MZ_BINARY_DATA = "/spectrum/binaryDataArrayList/binaryDataArray[./cvParam/@name='m/z array']/binary/text()"; public static final String SPECTRUM_PATH_INTENSITY_BINARY_DATA = "/spectrum/binaryDataArrayList/binaryDataArray[./cvParam/@name='intensity array']/binary/text()"; public static final Pattern SPECTRUM_EXTRACTION_REGEX = Pattern .compile("function=(\\d+) *process=(\\d+) scan=(\\d+)"); // XPathFactory is known to be non-thread-safe. protected static final ThreadLocal<XPathFactory> XPATH_FACTORY = new ThreadLocal<XPathFactory>() { @Override protected XPathFactory initialValue() { return XPathFactory.newInstance(); } }; /** * Helper function: builds an XML DocumentBuilderFactory that can be used repeatedly in this class. * <p> * TODO: move this to an XML utility class, as I'm sure we'll use it again some day. * * @return An XML DocumentBuilderFactory. * @throws ParserConfigurationException */ public static DocumentBuilderFactory mkDocBuilderFactory() throws ParserConfigurationException { /* This factory must be configured within the context of a method call for exception handling. * TODO: can we work around this w/ dependency injection? */ // from http://stackoverflow.com/questions/155101/make-documentbuilder-parse-ignore-dtd-references DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance(); docFactory.setValidating(false); docFactory.setNamespaceAware(true); docFactory.setFeature("http://xml.org/sax/features/namespaces", false); docFactory.setFeature("http://xml.org/sax/features/validation", false); docFactory.setFeature("http://apache.org/xml/features/nonvalidating/load-dtd-grammar", false); docFactory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false); return docFactory; } protected static List<Double> base64ToDoubleList(String b64) { byte[] decodedBytes = Base64.getDecoder().decode(b64); ByteBuffer buf = ByteBuffer.wrap(decodedBytes).order(ByteOrder.LITTLE_ENDIAN); List<Double> values = new ArrayList<>(decodedBytes.length / 8); while (buf.hasRemaining()) { values.add(buf.getDouble()); } return values; } // TODO: isn't there some library method for this? The Interwebs seem to say there isn't... protected static <K, V> List<Pair<K, V>> zipLists(List<K> keys, List<V> vals) { if (keys.size() != vals.size()) { throw new RuntimeException(String.format("Mismatched list sizes: %d vs %d", keys.size(), vals.size())); } List<Pair<K, V>> res = new ArrayList<>(keys.size()); Iterator<K> ki = keys.listIterator(); Iterator<V> vi = vals.listIterator(); while (ki.hasNext() && vi.hasNext()) { // Length check should ensure these are exhausted simultaneously. K k = ki.next(); V v = vi.next(); res.add(Pair.of(k, v)); } return res; } public MzMLParser() { } protected XPathFactory getXPathFactory() { return XPATH_FACTORY.get(); } public Iterator<S> getIterator(String inputFile) throws ParserConfigurationException, IOException, XMLStreamException { DocumentBuilderFactory docFactory = mkDocBuilderFactory(); DocumentBuilder docBuilder = docFactory.newDocumentBuilder(); final XMLInputFactory xmlInputFactory = XMLInputFactory.newInstance(); final XMLOutputFactory xmlOutputFactory = XMLOutputFactory.newInstance(); return new Iterator<S>() { boolean inEntry = false; XMLEventReader xr = xmlInputFactory.createXMLEventReader(new FileInputStream(inputFile), "utf-8"); // TODO: is the use of the XML version/encoding tag definitely necessary? StringWriter w = new StringWriter().append(XML_PREAMBLE).append("\n"); XMLEventWriter xw = xmlOutputFactory.createXMLEventWriter(w); S next = null; /* Because we're handling the XML as a stream, we can only determine whether we have another Spectrum to return * by attempting to parse the next one. `this.next()` reads */ private S getNextSpectrum() { S spectrum = null; if (xr == null || !xr.hasNext()) { return null; } try { while (xr.hasNext()) { XMLEvent e = xr.nextEvent(); if (!inEntry && e.isStartElement() && e.asStartElement().getName().getLocalPart().equals((SPECTRUM_OBJECT_TAG))) { xw.add(e); inEntry = true; } else if (e.isEndElement() && e.asEndElement().getName().getLocalPart().equals(SPECTRUM_OBJECT_TAG)) { xw.add(e); xw.flush(); /* TODO: the XMLOutputFactory docs don't make it clear if/how events can be written directly into a new * document structure, so we incur the cost of extracting each spectrum entry, serializing it, and * re-reading it into its own document so it can be handled by XPath. Master this strange corner of the * Java ecosystem and get rid of <></>his doc -> string -> doc conversion. */ Document doc = docBuilder.parse(new ReaderInputStream(new StringReader(w.toString()))); spectrum = handleSpectrumEntry(doc); xw.close(); /* Note: this can also be accomplished with `w.getBuffer().setLength(0);`, but using a new event writer * seems safer. */ w = new StringWriter(); w.append(XML_PREAMBLE).append("\n"); xw = xmlOutputFactory.createXMLEventWriter(w); inEntry = false; // Don't stop parsing if handleSpectrumEntry didn't like this spectrum document. if (spectrum != null) { break; } } else if (inEntry) { // Add this element if we're in an entry xw.add(e); } } // We've reached the end of the document; close the reader to show that we're done. if (!xr.hasNext()) { xr.close(); xr = null; } } catch (Exception e) { // TODO: do better. We seem to run into this sort of thing with Iterators a lot... throw new RuntimeException(e); } return spectrum; } private S tryParseNext() { // Fail the attempt if the reader is closed. if (xr == null || !xr.hasNext()) { return null; } // No checks on whether we already have a spectrum stored: we expect the callers to do that. return getNextSpectrum(); } @Override public boolean hasNext() { // Prime the pump if the iterator doesn't have a value stored yet. if (this.next == null) { this.next = tryParseNext(); } // If we have an entry waiting, return true; otherwise read the next entry and return true if successful. return this.next != null; } @Override public S next() { // Prime the pump like we do in hasNext(). if (this.next == null) { this.next = tryParseNext(); } // Take available spectrum and return it. S res = this.next; /* Advance to the next element immediately, making next() do the heavy lifting most of the time. Otherwise, * the parsing will resume on hasNext(), which seems like it ought to be a light-weight operation. */ this.next = tryParseNext(); return res; } }; } public List<S> parse(String inputFile) throws ParserConfigurationException, IOException, XMLStreamException { List<S> spectra = new ArrayList<>(); Iterator<S> iter = this.getIterator(inputFile); while (iter.hasNext()) { spectra.add(iter.next()); } return spectra; } protected abstract S handleSpectrumEntry(Document doc) throws XPathException; }