Java tutorial
/************************************************************************* * * * This file is part of the 20n/act project. * * 20n/act enables DNA prediction for synthetic biology/bioengineering. * * Copyright (C) 2017 20n Labs, Inc. * * * * Please direct all queries to act@20n.com. * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation, either version 3 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see <http://www.gnu.org/licenses/>. * * * *************************************************************************/ package com.act.biointerpretation.metadata; import act.server.NoSQLAPI; import act.shared.Reaction; import org.apache.commons.io.FileUtils; import org.json.JSONArray; import org.json.JSONObject; import java.io.File; import java.util.Map; import java.util.HashMap; import java.util.Set; import java.util.HashSet; import java.util.List; import java.util.ArrayList; import java.util.Iterator; /** * Requires newer database: * mongod --dbpath /Users/jca20n/Downloads/2016-11-22-full-run-w-validation & disown %1 * <p> * This one has the modification and cloning data also included * mongod --dbpath /Users/jca20n/Downloads/2016-12-08-actv01_vijay_proteins_only & disown %1 * Created by jca20n on 12/1/16. */ public class ProteinMetadataFactory { private Set<String> dataList = new HashSet<>(); private Map<String, Integer> dataMap = new HashMap<>(); //Data used in algorithms private Set<String> modificationTermsTrue; private Set<String> modificationTermsFalse; private Map<String, Map<Host, Integer>> clonedtermToScore; private Map<String, Map<Host, Localization>> termToHostLocalization; private ProteinMetadataFactory() { } public static ProteinMetadataFactory initiate() throws Exception { //Construct the Sets to hold modification data terms Set<String> modTrue = new HashSet<>(); Set<String> modFalse = new HashSet<>(); // TODO: Move this to resources directory? File termfile = new File("data/ProteinMetadata/2016_12_07-modification_terms.txt"); String data = FileUtils.readFileToString(termfile); String[] lines = data.split("\\r|\\r?\\n"); for (int i = 1; i < lines.length; i++) { String line = lines[i]; String[] tabs = line.split("\t"); String term = tabs[0].toLowerCase(); boolean val = Boolean.parseBoolean(tabs[1]); if (val == true) { modTrue.add(term); } else { modFalse.add(term); } } //Construct term data for 'cloned' Map<String, String> termToGenus = new HashMap<>(); // TODO: Move this to resources directory? termfile = new File("data/ProteinMetadata/2016_12_07-cloned_term_to_genus.txt"); data = FileUtils.readFileToString(termfile); lines = data.split("\\r|\\r?\\n"); for (int i = 1; i < lines.length; i++) { String line = lines[i]; String[] tabs = line.split("\t"); if (tabs[1].isEmpty() || tabs[0].isEmpty()) { continue; } termToGenus.put(tabs[0], tabs[1]); } //Pre-Compute distance to hosts for all terms for "cloned" Map<String, Genus> nameToGenus = Genus.parseGenuses(); Map<String, Map<Host, Integer>> termToScore = new HashMap<>(); for (String term : termToGenus.keySet()) { String tgenus = termToGenus.get(term); Genus ggenus = nameToGenus.get(tgenus); Map<Host, Integer> hostToScore = new HashMap<>(); for (Host host : Host.values()) { Genus hostgenus = nameToGenus.get(host.toString()); Integer score = Genus.similarity(ggenus, hostgenus); hostToScore.put(host, score); } termToScore.put(term, hostToScore); } //Construct the data to handle localization Map<String, Map<Host, Localization>> locMap = new HashMap<>(); // TODO: Move this to resources directory? termfile = new File("data/ProteinMetadata/2016_12_06-localization.txt"); data = FileUtils.readFileToString(termfile); lines = data.split("\\r|\\r?\\n"); for (int i = 1; i < lines.length; i++) { //Create the host map for each term and initially assume all values are 'questionable' Map<Host, Localization> hostToLoc = new HashMap<>(); for (Host host : Host.values()) { hostToLoc.put(host, Localization.questionable); } //Parse the line and put in replacement values per host String line = lines[i]; String[] tabs = line.split("\t"); String term = tabs[0]; for (int x = 1; x < tabs.length; x++) { try { String sloc = tabs[x]; Localization loc = Localization.valueOf(sloc); //Ecoli Bsubtilis Cglutamicum Scerevisiae Ppasteuris Aniger Hsapiens Sfrugiperda if (x == 1) { hostToLoc.put(Host.Ecoli, loc); } else if (x == 2) { hostToLoc.put(Host.Bsubtilis, loc); } else if (x == 3) { hostToLoc.put(Host.Cglutamicum, loc); } else if (x == 4) { hostToLoc.put(Host.Scerevisiae, loc); } else if (x == 5) { hostToLoc.put(Host.Ppasteuris, loc); } else if (x == 6) { hostToLoc.put(Host.Aniger, loc); } else if (x == 7) { hostToLoc.put(Host.Hsapiens, loc); } else if (x == 8) { hostToLoc.put(Host.Sfrugiperda, loc); } } catch (Exception err) { err.printStackTrace(); } } locMap.put(term, hostToLoc); } //Create the factory and put in data ProteinMetadataFactory factory = new ProteinMetadataFactory(); factory.modificationTermsTrue = modTrue; factory.modificationTermsFalse = modFalse; factory.clonedtermToScore = termToScore; factory.termToHostLocalization = locMap; return factory; } public ProteinMetadata create(JSONObject json) throws Exception { Double kcatkm = handleKcatKm(json); Double specificActivity = handleSpecificActivity(json); Boolean heteroSubunits = handlesubunits(json); Boolean modifications = handleModifications(json); Map<Host, Integer> cloning = handleCloned(json); Map<Host, Localization> localization = handleLocalization(json); List<Long> seqIds = handleSequences(json); ProteinMetadata out = new ProteinMetadata(); out.kcatkm = kcatkm; out.specificActivity = specificActivity; out.heteroSubunits = heteroSubunits; out.modifications = modifications; out.cloned = cloning; out.localization = localization; out.sequences = seqIds; return out; } private List<Long> handleSequences(JSONObject json) { //Try to pull the data JSONArray jarray = null; List<Long> seqIds = new ArrayList<>(); try { jarray = json.getJSONArray("sequences"); } catch (Exception err) { return seqIds; } //If there is no data, the value is undefined if (jarray.length() == 0) { return seqIds; } for (int i = 0; i < jarray.length(); i++) { Long data = (Long) jarray.get(i); seqIds.add(data); } return seqIds; } private Double handleKcatKm(JSONObject json) { //Try to pull the data JSONArray jarray = null; try { jarray = json.getJSONArray("kcat/km"); } catch (Exception err) { return null; } //If there is no data, the value is undefined if (jarray.length() == 0) { return null; } //If many observations are given for the protein, return the highest value observed /*TODO: the name of chemical in the substrate field could be matched to the substrate of the reaction Currently I am just picking the highest value, but it would be best to pick the highest value for the substrate being examined. This complicates the wiring, so I didn't try to implement */ if (jarray.length() > 1) { Double highest = -1.0; for (int i = 0; i < jarray.length(); i++) { try { JSONObject obj = jarray.getJSONObject(i); Double dval = obj.getDouble("val"); if (dval > highest) { highest = dval; } } catch (Exception err) { } } if (highest > 0) { return highest; } else { return null; } } //Otherwise return the one value try { JSONObject obj = jarray.getJSONObject(0); Double val = obj.getDouble("val"); return val; } catch (Exception err) { return null; } } private Double handleSpecificActivity(JSONObject json) { //Try to pull the data JSONArray jarray = null; try { jarray = json.getJSONArray("specific_activity"); } catch (Exception err) { return null; } //If there is no data, the value is undefined if (jarray.length() == 0) { return null; } //If many observations are given for the protein, return the highest value observed if (jarray.length() > 1) { Double highest = -1.0; for (int i = 0; i < jarray.length(); i++) { try { JSONObject obj = jarray.getJSONObject(i); Double dval = obj.getDouble("val"); if (dval > highest) { highest = dval; } } catch (Exception err) { } } if (highest > 0) { return highest; } else { return null; } } //Otherwise return the one value try { JSONObject obj = jarray.getJSONObject(0); Double val = obj.getDouble("val"); return val; } catch (Exception err) { return null; } } private Boolean handlesubunits(JSONObject json) { //Try to pull the data JSONArray jarray = null; try { jarray = json.getJSONArray("subunits"); } catch (Exception err) { return null; } //If there is no data, the value is undefined if (jarray.length() == 0) { return null; } //If many observations are given for the protein, return the consensus with a true value dominating if (jarray.length() > 1) { Boolean out = null; for (int i = 0; i < jarray.length(); i++) { try { JSONObject obj = jarray.getJSONObject(i); Boolean bval = assignSubunitHelper(obj); if (bval == true) { return true; } else if (bval == false) { out = false; } } catch (Exception err) { } } return out; } //Otherwise return the one value try { JSONObject obj = jarray.getJSONObject(0); Boolean bval = assignSubunitHelper(obj); return bval; } catch (Exception err) { return null; } } private Boolean assignSubunitHelper(JSONObject json) { try { String val = json.getString("val"); String comment = json.getString("comment"); if (val.equals("?")) { return null; } if (val.contains("hetero")) { return true; } if (comment.contains("hetero")) { return true; } if (val.contains("monomer")) { return false; } if (val.startsWith("homo")) { return false; } if (comment.contains("alpha") && comment.contains("beta")) { return true; } return false; } catch (Exception err) { return null; } } /** * This is a unit test for handleSubunits * @throws Exception */ private boolean testHandlesubunits() throws Exception { // TODO: Move this to resources directory? File testfile = new File("data/ProteinMetadata/2016_12_07-subunit_testset.txt"); String data = FileUtils.readFileToString(testfile); data = data.replaceAll("\"\"", "\""); String[] lines = data.split("\\r|\\r?\\n"); for (int i = 1; i < lines.length; i++) { String line = lines[i]; String[] tabs = line.split("\t"); //Pull out the json for each test and re-wrap it String arrayStr = tabs[0]; if (arrayStr.startsWith("\"")) { arrayStr = arrayStr.substring(1); } if (arrayStr.endsWith("\"")) { arrayStr = arrayStr.substring(0, arrayStr.length() - 1); } String jsonstr = "{\"subunits\":" + arrayStr + "}"; JSONObject json = new JSONObject(jsonstr); //Run the test and compare result to expected Boolean result = this.handlesubunits(json); Boolean expected = null; if (!tabs[1].equals("null")) { expected = Boolean.parseBoolean(tabs[1]); } if (result != expected) { System.err.println("Subunit testing error:\n" + json.toString() + " " + tabs[1]); System.err.println("Expect: " + expected + " Found: " + result); return false; } } return true; } private Boolean handleModifications(JSONObject json) { //Try to pull the data JSONArray jarray = null; try { jarray = json.getJSONArray("post_translational_modification"); } catch (Exception err) { return null; } //If there is no data, nothing is known if (jarray.length() == 0) { return null; } //If many observations are given for the protein, return the consensus with a true value dominating if (jarray.length() > 1) { Boolean out = null; for (int i = 0; i < jarray.length(); i++) { try { JSONObject obj = jarray.getJSONObject(i); Boolean bval = assignModificationHelper(obj); if (bval == true) { return true; } else if (bval == false) { out = false; } } catch (Exception err) { } } return out; } //Otherwise return the one value try { JSONObject obj = jarray.getJSONObject(0); Boolean bval = assignModificationHelper(obj); return bval; } catch (Exception err) { return null; } } private Boolean assignModificationHelper(JSONObject json) { try { String ptm = json.getString("post_translational_modification"); if (this.modificationTermsTrue.contains(ptm)) { return true; } if (this.modificationTermsFalse.contains(ptm)) { return false; } return null; } catch (Exception err) { return null; } } private Map<Host, Integer> handleCloned(JSONObject json) { //Populate output with null for all hosts (no prediction) Map<Host, Integer> out = new HashMap<>(); for (Host host : Host.values()) { out.put(host, null); } //Read in any data from JSON and interpret prediction based on phylogenetic distance to host try { JSONArray jarray = json.getJSONArray("cloned"); for (int i = 0; i < jarray.length(); i++) { JSONObject obj = jarray.getJSONObject(i); String comment = obj.getString("comment"); String[] words = comment.toLowerCase().split("[\\s,;]+"); for (String word : words) { Map<Host, Integer> hosttoint = this.clonedtermToScore.get(word); if (hosttoint == null) { continue; } for (Host host : hosttoint.keySet()) { Integer currval = out.get(host); if (currval == null) { currval = -99999; } Integer newval = hosttoint.get(host); if (newval > currval) { out.put(host, newval); } } } } } catch (Exception err) { } return out; } // Dead code: This was being called from within `handleCloned` but not now. private void printoutHosts(JSONObject json) { try { JSONArray jarray = json.getJSONArray("cloned"); for (int i = 0; i < jarray.length(); i++) { JSONObject obj = jarray.getJSONObject(i); String comment = obj.getString("comment"); int index = comment.indexOf("in "); if (index < 0) { continue; } String[] words = comment.substring(index).toLowerCase().split("[\\s,;]+"); String datapoint = ""; int limit = 1; int counter = 0; for (String word : words) { if (counter > limit) { break; } counter++; datapoint += word + "\t"; } Integer currval = dataMap.get(datapoint); if (currval == null) { currval = 0; } currval++; dataMap.put(datapoint, currval); } } catch (Exception err) { } } private Map<Host, Localization> handleLocalization(JSONObject json) { //Construct the output with unknown for all Hosts Map<Host, Localization> out = new HashMap<>(); for (Host host : Host.values()) { out.put(host, Localization.unknown); } //Parse out localization information from json and populate the output map with predictions JSONArray jarray = null; try { jarray = json.getJSONArray("localization"); } catch (Exception err) { err.printStackTrace(); } //If there is no metadata, all values should be "unknown" if (jarray.length() == 0) { return out; } //Scan through each observation for (int i = 0; i < jarray.length(); i++) { try { JSONObject obj = jarray.getJSONObject(i); String term = obj.getString("val"); Map<Host, Localization> hostToLoc = this.termToHostLocalization.get(term); if (hostToLoc == null) { System.out.println("missing term: " + term); dataList.add(term); continue; } for (Host host : hostToLoc.keySet()) { Localization currval = out.get(host); // Whatever is currently in the Map Localization newval = hostToLoc.get(host); // The potential new value //If the current value is "unknown", replace that value the one with the term if (currval == Localization.unknown) { out.put(host, newval); } //If the current value is "questionable", stay with questionable else if (currval == Localization.questionable) { out.put(host, Localization.questionable); } //If currval and newval have non-identical, but aren't unknown/questionable, then //There must be 2 non-identical predictions, in which case this becomes questionable else if (currval != newval) { out.put(host, Localization.questionable); } } } catch (Exception err) { err.printStackTrace(); } } return out; } public static void main(String[] args) throws Exception { // TODO: This is referencing a temporary collection. Change it! // TODO: FIX THIS BEFORE MERGE! NoSQLAPI api = new NoSQLAPI("actv01_vijay_proteins", "actv01_vijay_proteins"); Iterator<Reaction> iterator = api.readRxnsFromInKnowledgeGraph(); //Create a single instance of the factory method to use for all json ProteinMetadataFactory factory = ProteinMetadataFactory.initiate(); //Run some tests try { if (factory.testHandlesubunits() == true) { System.out.println("Subunit test OK"); } } catch (Exception err) { System.err.println("Failed to test subunits"); } //Create a list to aggregate the results of the database scan List<ProteinMetadata> agg = new ArrayList<>(); //Scan the database and store ProteinMetadata objects while (iterator.hasNext()) { Reaction rxn = iterator.next(); Reaction.RxnDataSource source = rxn.getDataSource(); if (!source.equals(Reaction.RxnDataSource.BRENDA)) { continue; } Set<JSONObject> jsons = rxn.getProteinData(); for (JSONObject json : jsons) { ProteinMetadata meta = factory.create(json); agg.add(meta); } } //Write out any messages to file StringBuilder sb = new StringBuilder(); for (String aline : factory.dataList) { sb.append(aline).append("\n"); } File outfile = new File("output/ProteinMetadata/Factory_output.txt"); if (outfile.exists()) { outfile.delete(); } FileUtils.writeStringToFile(outfile, sb.toString()); sb = new StringBuilder(); for (String key : factory.dataMap.keySet()) { int value = factory.dataMap.get(key); sb.append(key + "\t" + value + "\n"); } outfile = new File("output/ProteinMetadata/Factory_output_map.txt"); if (outfile.exists()) { outfile.delete(); } FileUtils.writeStringToFile(outfile, sb.toString()); //Count up the results of modifications to get statistics int falsecount = 0; int truecount = 0; int nullcount = 0; for (ProteinMetadata datum : agg) { if (datum == null) { System.err.println("null datum"); continue; } if (datum.modifications == null) { nullcount++; } else if (datum.modifications == false) { falsecount++; } else if (datum.modifications == true) { truecount++; } } System.out.println("Total # protein metadata: " + agg.size()); System.out.println(); System.out.println("modification true count: " + truecount); System.out.println("modification false count: " + falsecount); System.out.println("modification null count: " + nullcount); System.out.println(); //Get some statistics for cloned nullcount = 0; int emptycount = 0; int colicount = 0; int humancount = 0; int bothcount = 0; for (ProteinMetadata datum : agg) { if (datum == null) { System.err.println("null datum"); continue; } if (datum.cloned == null) { nullcount++; continue; } if (datum.cloned.isEmpty()) { emptycount++; continue; } Integer human = datum.cloned.get(Host.Hsapiens); if (human != null && human > 0) { humancount++; } Integer coli = datum.cloned.get(Host.Ecoli); if (coli != null && coli > 0) { colicount++; if (human != null && human > 0) { bothcount++; } } } System.out.println("cloned null count: " + nullcount); System.out.println("cloned empty count: " + emptycount); System.out.println("cloned coli count: " + colicount); System.out.println("cloned human count: " + humancount); System.out.println("cloned both count: " + bothcount); System.out.println(); } }