Java tutorial
/************************************************************************* * * * This file is part of the 20n/act project. * * 20n/act enables DNA prediction for synthetic biology/bioengineering. * * Copyright (C) 2017 20n Labs, Inc. * * * * Please direct all queries to act@20n.com. * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation, either version 3 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see <http://www.gnu.org/licenses/>. * * * *************************************************************************/ package act.installer.bing; import java.io.IOException; import java.util.ArrayList; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Set; import java.util.stream.Collectors; import com.act.utils.CLIUtil; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import com.mongodb.BasicDBObject; import act.server.MongoDB; /** * This class contains the main logic for installing Bing Search results in the Installer DB */ public class BingSearcher { private static final Logger LOGGER = LogManager.getFormatterLogger(BingSearcher.class); private static final String USAGE_TERMS_FILENAME = "usage_terms.txt"; public static final String HELP_MESSAGE = "This class contains the main logic for installing Bing Search results in the Installer DB."; public static final String OPTION_DB_NAME = "n"; public static final String OPTION_DB_PORT = "p"; public static final String OPTION_DB_HOST = "h"; public static final String OPTION_CACHE_ONLY = "c"; public static final String DEFAULT_HOST = "localhost"; public static final String DEFAULT_PORT = "27017"; public static final String DEFAULT_DATABASE = "actv01"; public static final List<Option.Builder> OPTION_BUILDERS = new ArrayList<Option.Builder>() { { add(Option.builder(OPTION_DB_NAME).argName("db name") .desc(String.format("The name of the database from which to fetch chemicals inchis and " + "write Bing cross-references (default: %s). It needs to contains a 'chemicals' collection.", DEFAULT_DATABASE)) .hasArg().longOpt("db").type(String.class)); add(Option.builder(OPTION_DB_HOST).argName("DB host") .desc(String.format("The database host to which to connect (default: %s)", DEFAULT_HOST)) .hasArg().longOpt("db-host")); add(Option.builder(OPTION_DB_PORT).argName("DB port") .desc(String.format("The port on which to connect to the database (default: %s)", DEFAULT_PORT)) .hasArg().longOpt("db-port")); add(Option.builder(OPTION_CACHE_ONLY).argName("CACHE_ONLY").desc( "Use the cache only. If that option is used, they will not be any queries against the Bing search API.") .longOpt("cache-only").type(String.class)); } }; private MongoDB db; private BingSearchResults bingSearchResults; private Set<String> usageTerms; private boolean forceUpdate; private boolean cacheOnly; public static void main(String args[]) { CLIUtil cliUtil = new CLIUtil(BingSearcher.class, HELP_MESSAGE, OPTION_BUILDERS); CommandLine cl = cliUtil.parseCommandLine(args); MongoDB db = new MongoDB(cl.getOptionValue(OPTION_DB_HOST, DEFAULT_HOST), Integer.parseInt(cl.getOptionValue(OPTION_DB_PORT, DEFAULT_PORT)), cl.getOptionValue(OPTION_DB_NAME, DEFAULT_DATABASE)); BingSearcher bingSearcher = new BingSearcher(db, true, cl.hasOption(OPTION_CACHE_ONLY)); bingSearcher.addBingSearchResultsForEntireDatabase(); } public BingSearcher(MongoDB db) { this(db, false, false); } public BingSearcher(MongoDB db, boolean forceUpdate, boolean cacheOnly) { this.db = db; this.forceUpdate = forceUpdate; this.cacheOnly = cacheOnly; this.bingSearchResults = new BingSearchResults(true); // Get the usage terms LOGGER.debug("Getting usage terms corpus."); UsageTermsCorpus usageTermsCorpus = new UsageTermsCorpus(USAGE_TERMS_FILENAME); try { usageTermsCorpus.buildCorpus(); } catch (IOException e) { LOGGER.error("Usage term corpus source file not found in class resources: %s", USAGE_TERMS_FILENAME); System.exit(1); } this.usageTerms = usageTermsCorpus.getUsageTerms(); } private void addBingSearchResultsForEntireDatabase() { Iterator<String> it = db.getIteratorOverInchis(new BasicDBObject()); while (it.hasNext()) { String inchi = it.next(); if (inchi.contains("FAKE")) { continue; } try { addBingSearchResultsForInChI(inchi); } catch (IOException e) { LOGGER.error("Bing Search results could not be added for: %s", inchi); } } } private void addBingSearchResultsForInChI(String inchi) throws IOException { LOGGER.debug("Processing InChI " + inchi); // Fetches the names (Brenda, Metacyc, Chebi, Drugbank) NamesOfMolecule namesOfMolecule = db.fetchNamesFromInchi(inchi); if (namesOfMolecule == null) { LOGGER.debug("Molecule corresponding to %s was not found in the database. Skipping.", inchi); return; } // Chooses the best name according to Bing search results String bestName = bingSearchResults.findBestMoleculeName(namesOfMolecule); if (bestName.equals("")) { return; } // Get the total number of hits and the top search results Long totalCountSearchResults; Set<SearchResult> topSearchResults; if (cacheOnly) { totalCountSearchResults = bingSearchResults.getTotalCountSearchResultsFromCache(bestName); topSearchResults = bingSearchResults.getTopSearchResultsFromCache(bestName); } else { totalCountSearchResults = bingSearchResults.getAndCacheTotalCountSearchResults(bestName); topSearchResults = bingSearchResults.getAndCacheTopSearchResults(bestName); } NameSearchResults nameSearchResults = new NameSearchResults(bestName); nameSearchResults.setTotalCountSearchResults(totalCountSearchResults); nameSearchResults.setTopSearchResults(topSearchResults); // Intersect usage names with search results Set<UsageTermUrlSet> moleculeUsageTerms = new HashSet<>(); for (String usageTerm : usageTerms) { UsageTermUrlSet usageTermUrlSet = new UsageTermUrlSet(usageTerm); usageTermUrlSet.populateUrlsFromNameSearchResults(nameSearchResults); if (usageTermUrlSet.getUrlSet().size() > 0) { moleculeUsageTerms.add(usageTermUrlSet); } } // Annotate the chemical with Bing Search Results BasicDBObject doc = db.createBingMetadataDoc(moleculeUsageTerms, totalCountSearchResults, bestName); db.updateChemicalWithBingSearchResults(inchi, bestName, doc); } public void addBingSearchResultsForInchiSet(Set<String> inchis) { Set<String> filteredInchis = inchis.stream().filter(inchi -> !inchi.contains("FAKE")) .collect(Collectors.toSet()); LOGGER.info("Annotating %d chemicals with Bing Search results and usage terms.", filteredInchis.size()); int counter = 0; for (String inchi : filteredInchis) { if (!forceUpdate && db.hasBingSearchResultsFromInchi(inchi)) { LOGGER.debug("Existing Bing search results found for %s. Skipping.", inchi); continue; } try { addBingSearchResultsForInChI(inchi); } catch (IOException e) { LOGGER.error("Could not add bing results for %s. Skipping.", inchi); } if (++counter % 100 == 0) { LOGGER.info("Added Bing Search results for %d chemicals (total %d)", counter, filteredInchis.size()); } } } }