List of usage examples for java.lang String trim
public String trim()
From source file:gov.noaa.ncdc.iosp.avhrr.util.AvhrrLevel1B2Netcdf.java
/** * @param args//from w ww .ja v a2 s .co m * the command line arguments */ public static void main(String args[]) throws Exception { // create the Options object Options options = new Options(); options.addOption(OPTION_NOGUI, false, "command line mode"); options.addOption(OPTION_ALLVAR, false, "Process all channels"); options.addOption(OPTION_CH1, false, "Process channel 1"); options.addOption(OPTION_CH2, false, "Process channel 2"); options.addOption(OPTION_CH3, false, "Process channel 3"); options.addOption(OPTION_CH4, false, "Process channel 4"); options.addOption(OPTION_CH5, false, "Process channel 5"); options.addOption(OPTION_DEFAULT, false, "Output All Channels, Brightness Temperature and Latitude and Longitude"); options.addOption(OPTION_TEMP, false, "Output calibrated temperatures/albedos"); options.addOption(OPTION_RAW, false, "Output raw values"); options.addOption(OPTION_RADIANCE, false, "Output radiance values"); options.addOption(OPTION_LATLON, false, "Output latitude and longitude"); options.addOption(OPTION_ALLVAR, false, "Output all variables)"); options.addOption(OPTION_CLOBBER, false, "Overwrite existing netCDF file."); options.addOption(OPTION_CALIBRATION, false, "Output calibration variables"); options.addOption(OPTION_QUALITY, false, "Output quality variables."); options.addOption(OPTION_METADATA, false, "Output metadata variables."); options.addOption("memory", false, "java heap space."); Option logOption = new Option("logFile", true, "Send output to file"); options.addOption(logOption); Option outdirOption = new Option("outputdir", true, "Output Directory."); options.addOption(outdirOption); // 2nd way is by creating an Option and then adding it Option timeOption = new Option(OPTION_AVHRR, true, "Process the Level1B file"); timeOption.setRequired(false); options.addOption(timeOption); // now lets parse the input CommandLineParser parser = new BasicParser(); CommandLine cmd; try { cmd = parser.parse(options, args); } catch (ParseException pe) { usage(options); return; } // now lets interrogate the options and execute the relevant parts if (cmd.hasOption(OPTION_NOGUI)) { // process command line args - need to refactor this (doing this // twice--BAD!!!!!) List<String> list = processCommandLineArgs(cmd); String file = cmd.getOptionValue(OPTION_AVHRR); if (null == file) { System.out.println("You must specify a file to convert!! missing -avhrr argument"); usage(options); return; } String outdir = cmd.getOptionValue("outputdir"); if (null == outdir || outdir.trim().length() < 1) { File f = new File("."); outdir = f.getAbsolutePath(); } // try { Avhrr2Netcdf.convert(file, list, outdir); // } catch (Exception e) { // System.out.println("this blew up"); // TODO Auto-generated catch block // e.printStackTrace(); // } } else { java.awt.EventQueue.invokeLater(new Runnable() { public void run() { UIManager.LookAndFeelInfo[] info = UIManager.getInstalledLookAndFeels(); for (int i = 0; i < info.length; i++) { // Get the name of the look and feel that is suitable // for // display to the user String className = info[i].getClassName(); } String javaLF = UIManager.getSystemLookAndFeelClassName(); try { // UIManager.setLookAndFeel("com.sun.java.swing.plaf.motif.MotifLookAndFeel"); UIManager.setLookAndFeel(javaLF); } catch (ClassNotFoundException e) { // log.error(e.getMessage()); } catch (InstantiationException e) { // log.error(e.getMessage()); } catch (IllegalAccessException e) { // log.error(e.getMessage()); } catch (UnsupportedLookAndFeelException e) { // log.error(e.getMessage()); } mFrame = new AvhrrLevel1B2Netcdf(); mFrame.setVisible(true); } }); } }
From source file:jhc.redsniff.generation.PackageScanningGenerator.java
public static void main(String[] args) throws Exception { if (args.length != 5) { System.err.println("Args: source-dir package-class-filter parent-class generated-class output-dir"); System.err.println(""); System.err.println(" source-dir : Path to Java source containing matchers to generate sugar for."); System.err.println(" May contain multiple paths, separated by commas."); System.err.println(" e.g. src/java,src/more-java"); System.err.println(// w w w . j a va 2s . c om " package-class-filter : base of package to look for classes with methods, eg jhc.selenium.matchers"); System.err.println(""); System.err.println( "parent-class : Full name of parent class type to examine - eg org.hamcrest.Matcher, jhc.selenium.seeker.Seeker"); System.err.println(" e.g. org.myproject.MyMatchers"); System.err.println("generated-class : Full name of class to generate."); System.err.println(" e.g. org.myproject.MyMatchers"); System.err.println(""); System.err.println(" output-dir : Where to output generated code (package subdirs will be"); System.err.println(" automatically created)."); System.err.println(" e.g. build/generated-code"); System.exit(-1); } String srcDirs = args[0]; String package_name_prefix = args[1]; String parentClassName = args[2]; String fullClassName = args[3]; File outputDir = new File(args[4]); String fileName = fullClassName.replace('.', File.separatorChar) + ".java"; int dotIndex = fullClassName.lastIndexOf("."); String packageName = dotIndex == -1 ? "" : fullClassName.substring(0, dotIndex); String shortClassName = fullClassName.substring(dotIndex + 1); if (!outputDir.isDirectory()) { System.err.println("Output directory not found : " + outputDir.getAbsolutePath()); System.exit(-1); } Class<?> parentClass = Class.forName(parentClassName); File outputFile = new File(outputDir, fileName); outputFile.getParentFile().mkdirs(); File tmpFile = new File(outputDir, fileName + ".tmp"); SugarGenerator sugarGenerator = new SugarGenerator(); try { sugarGenerator .addWriter(new SeleniumFactoryWriter(packageName, shortClassName, new FileWriter(tmpFile))); sugarGenerator.addWriter(new QuickReferenceWriter(System.out)); PackageScanningGenerator pkgScanningGenerator = new PackageScanningGenerator(sugarGenerator, PackageScanningGenerator.class.getClassLoader(), parentClass); if (srcDirs.trim().length() > 0) { for (String srcDir : srcDirs.split(",")) { pkgScanningGenerator.addSourceDir(new File(srcDir)); } } // could add use of xml just to list filter expressions // pkgScanningGenerator.load(new InputSource(configFile)); pkgScanningGenerator.addClasses(package_name_prefix); System.out.println("Generating " + fullClassName); sugarGenerator.generate(); sugarGenerator.close(); outputFile.delete(); FileUtils.moveFile(tmpFile, outputFile); } finally { tmpFile.delete(); sugarGenerator.close(); } }
From source file:com.genentech.chemistry.openEye.apps.QTorsionProfileGenerator.java
public static void main(String... args) throws IOException { // create command line Options object Options options = new Options(); Option opt = new Option(OPT_INFILE, true, "input file oe-supported Use .sdf|.smi to specify the file type."); opt.setRequired(true);/* w ww . j av a 2 s.com*/ options.addOption(opt); opt = new Option(OPT_SDFFILE, true, "file to write onformers for debugging (oe-supported Use .sdf|.smi to specify the file type)."); options.addOption(opt); opt = new Option(OPT_WORKDIR, true, "Write files into this directory!"); opt.setRequired(false); options.addOption(opt); opt = new Option(OPT_OUTPREFIX, true, "Prefix for output file."); opt.setRequired(false); options.addOption(opt); opt = new Option(OPT_OUTNAMETAG, true, "TagName of field containing outFilePrefix. (Use TITLE for mol title)."); opt.setRequired(false); options.addOption(opt); opt = new Option(OPT_TEMPLATE, true, "Template file for quantum program containing #XYZ# place holder line. A #FName# placeholder can be used fro chk files and the like."); opt.setRequired(true); options.addOption(opt); opt = new Option(OPT_SPIN_MULTIPLICITY, true, "Spin Multiplicity of the input molecules (default 1)"); opt.setRequired(false); options.addOption(opt); opt = new Option(OPT_NCPU, true, "Overwrite nprocshared parameter in guassian input file if given"); opt.setRequired(false); options.addOption(opt); opt = new Option(OPT_MEM, true, "Memory for gaussian default='10GB' replaces #mem# in tempalte"); opt.setRequired(false); options.addOption(opt); opt = new Option(OPT_MINIMIZE, false, "minimize conformer at each step using MMFFs"); opt.setRequired(false); options.addOption(opt); opt = new Option(OPT_CONSTRIANT, true, "one of strong (90),medium (45), weak(20), none or a floating point number" + " specifying the strength of tethered constrains for -doMinimize (def=strong)"); opt.setRequired(false); options.addOption(opt); opt = new Option(OPT_BONDFILE, true, "Structure file containing 4 atoms defining the torsion. " + "In each input molecule the atoms colses these atoms are used to define the torsion."); opt.setRequired(true); options.addOption(opt); opt = new Option(OPT_STARTTorsion, true, "The torsion in your inMol will be rotated by this value for the first job"); opt.setRequired(false); options.addOption(opt); opt = new Option(OPT_TORSIONIncrement, true, "Incremnt each subsequent conformation by this step size"); opt.setRequired(true); options.addOption(opt); opt = new Option(OPT_NSTEPS, true, "Number of conformations to create"); opt.setRequired(true); options.addOption(opt); opt = new Option(OPT_MAXCONFS_PER_STEP, true, "While holding the torsion fixed, maximum number of conformations of free atoms to generate. default=1"); opt.setRequired(false); options.addOption(opt); opt = new Option(OPT_COREFILE, true, "Outputfile to store guessed core."); opt.setRequired(false); options.addOption(opt); opt = new Option(OPT_DEBUG, false, "Produce more debug output."); opt.setRequired(false); options.addOption(opt); CommandLineParser parser = new PosixParser(); CommandLine cmd = null; try { cmd = parser.parse(options, args); } catch (Exception e) { System.err.println(e.getMessage()); exitWithHelp(options); } args = cmd.getArgs(); if (args.length != 0) { System.err.println("Unknown arguments" + args); exitWithHelp(options); } if (cmd.hasOption("d")) { System.err.println("Start debugger and press return:"); new BufferedReader(new InputStreamReader(System.in)).readLine(); } String inFile = cmd.getOptionValue(OPT_INFILE); String sdfFile = cmd.getOptionValue(OPT_SDFFILE); String outPrefix = cmd.getOptionValue(OPT_OUTPREFIX); String outNameTag = cmd.getOptionValue(OPT_OUTNAMETAG); String workDir = cmd.getOptionValue(OPT_WORKDIR); String coreFile = cmd.getOptionValue(OPT_COREFILE); String tempalte = cmd.getOptionValue(OPT_TEMPLATE); String bondFile = cmd.getOptionValue(OPT_BONDFILE); String nCPU = cmd.getOptionValue(OPT_NCPU); String memStr = cmd.getOptionValue(OPT_MEM); String maxConfsStr = cmd.getOptionValue(OPT_MAXCONFS_PER_STEP); String spinMult = cmd.getOptionValue(OPT_SPIN_MULTIPLICITY); boolean doMinimize = cmd.hasOption(OPT_MINIMIZE); String constraintStrength = cmd.getOptionValue(OPT_CONSTRIANT); int nStep = Integer.parseInt(cmd.getOptionValue(OPT_NSTEPS)); if (memStr == null || memStr.length() == 0) memStr = "10GB"; if (spinMult == null || spinMult.length() == 0) spinMult = "1"; if ((outPrefix == null && outNameTag == null) || (outPrefix != null && outNameTag != null)) { System.err.println("Exactly one of -outPrefix or outNameTag must be given!"); exitWithHelp(options); } if (workDir == null || workDir.trim().length() == 0) workDir = "."; double startTorsion = Double.NaN; if (cmd.hasOption(OPT_STARTTorsion)) startTorsion = Double.parseDouble(cmd.getOptionValue(OPT_STARTTorsion)); double torInc = Double.parseDouble(cmd.getOptionValue(OPT_TORSIONIncrement)); int maxStepConfs = maxConfsStr == null ? 1 : Integer.parseInt(maxConfsStr); QTorsionProfileGenerator calculator = new QTorsionProfileGenerator(tempalte, workDir, outPrefix, outNameTag, bondFile, spinMult, startTorsion, torInc, nStep, maxStepConfs, doMinimize, constraintStrength, cmd.hasOption(OPT_DEBUG)); calculator.run(inFile, sdfFile, coreFile, nCPU, memStr); }
From source file:com.sds.acube.ndisc.xadmin.XNDiscAdminUtil.java
public static void main(String args[]) { if (args.length == 0) { XNDiscAdminUtil.printAdminUsage(null, null); // System.exit(0); }/*from w ww. j a v a2 s. c o m*/ XNDiscAdminFile file = new XNDiscAdminFile(printlog, out, logger); XNDiscAdminMedia media = new XNDiscAdminMedia(printlog, out, logger); XNDiscAdminVolume volume = new XNDiscAdminVolume(printlog, out, logger); XNDiscAdminEnDecrypt endecrypt = new XNDiscAdminEnDecrypt(printlog, out, logger); Scanner in = new Scanner(System.in); String input = ""; List<String> options = null; Scanner part = null; do { input = in.nextLine(); part = new Scanner(input).useDelimiter(" "); options = new ArrayList<String>(); while (part.hasNext()) { String val = part.next().trim(); if (StringUtils.isEmpty(val)) { continue; } options.add(val); } if (options.size() < 2 || input.equalsIgnoreCase("q")) { if (input != null && input.trim().equalsIgnoreCase("q")) { System.out.println(LINE_SEPERATOR + "XNDiscAdminUtil quit!!!" + LINE_SEPERATOR); } else { System.out .println(LINE_SEPERATOR + "XNDiscAdminUtil are invalid parameters!!!" + LINE_SEPERATOR); } System.exit(0); } String main_op = (StringUtils.isEmpty(options.get(0))) ? "" : options.get(0); String process_op = (StringUtils.isEmpty(options.get(1))) ? "" : options.get(1); if (main_op.equals("clear")) { if (process_op.equals("screen")) { clearConsoleOutput(); } } else if (main_op.equals("file")) { if (process_op.equals("ls")) { String fileid = ""; if (options.size() > 2) { fileid = options.get(2); } if (StringUtils.isEmpty(fileid)) { file.selectFileList(); } else { file.selectFileById(fileid); } } else if (process_op.equals("reg")) { String host = ""; int port = -1; String filepath = ""; int vol = -1; if (options.size() == 4) { host = XNDiscAdminConfig.getString(XNDiscAdminConfig.HOST); port = XNDiscAdminConfig.getInt(XNDiscAdminConfig.PORT); filepath = options.get(2); if (isInteger(options.get(3))) { vol = Integer.parseInt(options.get(3)); } } else if (options.size() > 5) { host = options.get(2); port = Integer.parseInt(options.get(3)); filepath = options.get(4); if (isInteger(options.get(5))) { vol = Integer.parseInt(options.get(5)); } } if (!StringUtils.isEmpty(host) && !StringUtils.isEmpty(filepath) && port > 0 && vol > 0) { file.regFile(host, port, filepath, vol, "0"); } else { XNDiscAdminUtil.printAdminUsage(main_op, process_op); } } else if (process_op.equals("get")) { String host = ""; int port = -1; String fileid = ""; String filepath = ""; if (options.size() == 4) { host = XNDiscAdminConfig.getString(XNDiscAdminConfig.HOST); port = XNDiscAdminConfig.getInt(XNDiscAdminConfig.PORT); fileid = options.get(2); filepath = options.get(3); } else if (options.size() > 5) { host = options.get(2); if (isInteger(options.get(3))) { port = Integer.parseInt(options.get(3)); } fileid = options.get(4); filepath = options.get(5); } if (!StringUtils.isEmpty(host) && !StringUtils.isEmpty(fileid) && !StringUtils.isEmpty(filepath) && port > 0) { file.getFile(host, port, fileid, filepath); } else { XNDiscAdminUtil.printAdminUsage(main_op, process_op); } } else if (process_op.equals("wh")) { String fileid = ""; if (options.size() > 2) { fileid = options.get(2); } if (!StringUtils.isEmpty(fileid)) { file.getFilePathByFileId(fileid); } else { XNDiscAdminUtil.printAdminUsage(main_op, process_op); } } else if (process_op.equals("rm")) { String host = ""; int port = -1; String fileid = ""; if (options.size() == 3) { host = XNDiscAdminConfig.getString(XNDiscAdminConfig.HOST); port = XNDiscAdminConfig.getInt(XNDiscAdminConfig.PORT); fileid = options.get(2); } else if (options.size() > 4) { host = options.get(2); if (isInteger(options.get(3))) { port = Integer.parseInt(options.get(3)); } fileid = options.get(4); } if (!StringUtils.isEmpty(host) && !StringUtils.isEmpty(fileid) && port > 0) { file.removeFile(host, port, fileid); } else { XNDiscAdminUtil.printAdminUsage(main_op, process_op); } } } else if (main_op.equals("media")) { if (process_op.equals("mk")) { String host = ""; int port = -1; String name = ""; int type = 0; String path = ""; String desc = " "; int maxsize = -1; int vol = -1; if (options.size() == 7) { host = XNDiscAdminConfig.getString(XNDiscAdminConfig.HOST); port = XNDiscAdminConfig.getInt(XNDiscAdminConfig.PORT); name = options.get(2); if (isInteger(options.get(3))) { type = Integer.parseInt(options.get(3)); } path = options.get(4); if (isInteger(options.get(5))) { maxsize = Integer.parseInt(options.get(5)); } if (isInteger(options.get(6))) { vol = Integer.parseInt(options.get(6)); } } else if (options.size() == 8) { host = XNDiscAdminConfig.getString(XNDiscAdminConfig.HOST); port = XNDiscAdminConfig.getInt(XNDiscAdminConfig.PORT); name = options.get(2); if (isInteger(options.get(3))) { type = Integer.parseInt(options.get(3)); } path = options.get(4); desc = options.get(5) + " "; if (isInteger(options.get(6))) { maxsize = Integer.parseInt(options.get(6)); } if (isInteger(options.get(7))) { vol = Integer.parseInt(options.get(7)); } } else if (options.size() == 9) { host = options.get(2); if (isInteger(options.get(3))) { port = Integer.parseInt(options.get(3)); } name = options.get(4); if (isInteger(options.get(5))) { type = Integer.parseInt(options.get(5)); } path = options.get(6); if (isInteger(options.get(7))) { maxsize = Integer.parseInt(options.get(7)); } if (isInteger(options.get(8))) { vol = Integer.parseInt(options.get(8)); } } else if (options.size() > 9) { host = options.get(2); if (isInteger(options.get(3))) { port = Integer.parseInt(options.get(3)); } name = options.get(4); if (isInteger(options.get(5))) { type = Integer.parseInt(options.get(5)); } path = options.get(6); desc = options.get(7) + " "; if (isInteger(options.get(8))) { maxsize = Integer.parseInt(options.get(8)); } if (isInteger(options.get(9))) { vol = Integer.parseInt(options.get(9)); } } if (!StringUtils.isEmpty(host) && !StringUtils.isEmpty(name) && !StringUtils.isEmpty(path) && port > 0 && vol > 0 && maxsize > 0) { media.makeMedia(host, port, name, type, path, desc, maxsize, vol); } else { XNDiscAdminUtil.printAdminUsage(main_op, process_op); } } else if (process_op.equals("ls")) { String mediaid = "-1"; if (options.size() > 2) { mediaid = options.get(2); } if (StringUtils.isEmpty(mediaid) || mediaid.equals("-1")) { media.selectMediaList(); } else { int id = -1; if (isInteger(mediaid)) { id = Integer.parseInt(mediaid); } if (id > 0) { media.selectMediaById(id); } else { XNDiscAdminUtil.printAdminUsage(main_op, process_op); } } } else if (process_op.equals("rm")) { String mediaid = "-1"; if (options.size() > 2) { mediaid = options.get(2); } if (!StringUtils.isEmpty(mediaid)) { int id = -1; if (isInteger(mediaid)) { id = Integer.parseInt(options.get(2)); } if (id > 0) { media.removeMedia(id); } else { XNDiscAdminUtil.printAdminUsage(main_op, process_op); } } } else if (process_op.equals("ch")) { int mediaid = -1; String name = ""; int type = 0; String path = ""; String desc = ""; int maxsize = -1; int vol = -1; if (options.size() == 8) { if (isInteger(options.get(2))) { mediaid = Integer.parseInt(options.get(2)); } name = options.get(3); if (isInteger(options.get(4))) { type = Integer.parseInt(options.get(4)); } path = options.get(5); if (isInteger(options.get(6))) { maxsize = Integer.parseInt(options.get(6)); } if (isInteger(options.get(7))) { vol = Integer.parseInt(options.get(7)); } } else if (options.size() > 8) { if (isInteger(options.get(2))) { mediaid = Integer.parseInt(options.get(2)); } name = options.get(3); if (isInteger(options.get(4))) { type = Integer.parseInt(options.get(4)); } path = options.get(5); desc = options.get(6); if (isInteger(options.get(7))) { maxsize = Integer.parseInt(options.get(7)); } if (isInteger(options.get(8))) { vol = Integer.parseInt(options.get(8)); } } if (!StringUtils.isEmpty(name) && !StringUtils.isEmpty(path) && mediaid > 0 && maxsize > 0 && vol > 0) { media.changeMedia(mediaid, name, type, path, desc, maxsize, vol); } else { XNDiscAdminUtil.printAdminUsage(main_op, process_op); } } } else if (main_op.equals("vol")) { if (process_op.equals("mk")) { String name = ""; String access = ""; String desc = " "; if (options.size() == 4) { name = options.get(2); access = options.get(3); } else if (options.size() > 4) { name = options.get(2); access = options.get(3); desc = options.get(4) + " "; } if (!StringUtils.isEmpty(name) && !StringUtils.isEmpty(access)) { volume.makeVolume(name, access, desc); } else { XNDiscAdminUtil.printAdminUsage(main_op, process_op); } } else if (process_op.equals("ls")) { String volid = "-1"; if (options.size() > 2) { volid = options.get(2); } if (StringUtils.isEmpty(volid) || volid.equals("-1")) { volume.selectVolumeList(); } else { int volumeid = -1; if (isInteger(volid)) { volumeid = Integer.parseInt(volid); } if (volumeid > 0) { volume.selectVolumeById(volumeid); } else { XNDiscAdminUtil.printAdminUsage(main_op, process_op); } } } else if (process_op.equals("rm")) { String volid = "-1"; if (options.size() > 2) { volid = options.get(2); } if (!StringUtils.isEmpty(volid)) { int volumeid = -1; if (isInteger(volid)) { volumeid = Integer.parseInt(volid); } if (volumeid > 0) { volume.removeVolume(volumeid); } } else { XNDiscAdminUtil.printAdminUsage(main_op, process_op); } } else if (process_op.equals("ch")) { int volumeid = -1; String name = ""; String access = ""; String desc = ""; if (options.size() == 5) { if (isInteger(options.get(2))) { volumeid = Integer.parseInt(options.get(2)); } name = options.get(3); access = options.get(4); } else if (options.size() > 5) { if (isInteger(options.get(2))) { volumeid = Integer.parseInt(options.get(2)); } name = options.get(3); access = options.get(4); desc = options.get(5) + " "; } if (!StringUtils.isEmpty(name) && !StringUtils.isEmpty(access) && volumeid > 0) { volume.changeVolume(volumeid, name, access, desc); } else { XNDiscAdminUtil.printAdminUsage(main_op, process_op); } } } else if (main_op.equals("id")) { String id = ""; if (options.size() > 2) { id = options.get(2); } if (process_op.equals("enc")) { if (!StringUtils.isEmpty(id)) { endecrypt.encrypt(id); } else { XNDiscAdminUtil.printAdminUsage(main_op, process_op); } } else if (process_op.equals("dec")) { if (!StringUtils.isEmpty(id)) { endecrypt.decrypt(id); } else { XNDiscAdminUtil.printAdminUsage(main_op, process_op); } } } part.close(); } while (!input.equalsIgnoreCase("q")); in.close(); }
From source file:DIA_Umpire_Quant.DIA_Umpire_ProtQuant.java
/** * @param args the command line arguments */// ww w .j a v a2 s .c o m public static void main(String[] args) throws FileNotFoundException, IOException, Exception { System.out.println( "================================================================================================="); System.out.println( "DIA-Umpire protein quantitation module (version: " + UmpireInfo.GetInstance().Version + ")"); if (args.length != 1) { System.out.println( "command format error, the correct format should be: java -jar -Xmx10G DIA_Umpire_PortQuant.jar diaumpire_module.params"); return; } try { ConsoleLogger.SetConsoleLogger(Level.INFO); ConsoleLogger.SetFileLogger(Level.DEBUG, FilenameUtils.getFullPath(args[0]) + "diaumpire_orotquant.log"); } catch (Exception e) { } Logger.getRootLogger().info("Version: " + UmpireInfo.GetInstance().Version); Logger.getRootLogger().info("Parameter file:" + args[0]); BufferedReader reader = new BufferedReader(new FileReader(args[0])); String line = ""; String WorkFolder = ""; int NoCPUs = 2; String Combined_Prot = ""; boolean DefaultProtFiltering = true; float Freq = 0f; int TopNPep = 6; int TopNFrag = 6; String FilterWeight = "GW"; float MinWeight = 0.9f; TandemParam tandemPara = new TandemParam(DBSearchParam.SearchInstrumentType.TOF5600); HashMap<String, File> AssignFiles = new HashMap<>(); boolean ExportSaint = false; boolean SAINT_MS1 = false; boolean SAINT_MS2 = true; HashMap<String, String[]> BaitList = new HashMap<>(); HashMap<String, String> BaitName = new HashMap<>(); HashMap<String, String[]> ControlList = new HashMap<>(); HashMap<String, String> ControlName = new HashMap<>(); //<editor-fold defaultstate="collapsed" desc="Reading parameter file"> while ((line = reader.readLine()) != null) { line = line.trim(); Logger.getRootLogger().info(line); if (!"".equals(line) && !line.startsWith("#")) { //System.out.println(line); if (line.equals("==File list begin")) { do { line = reader.readLine(); line = line.trim(); if (line.equals("==File list end")) { continue; } else if (!"".equals(line)) { File newfile = new File(line); if (newfile.exists()) { AssignFiles.put(newfile.getAbsolutePath(), newfile); } else { Logger.getRootLogger().info("File: " + newfile + " does not exist."); } } } while (!line.equals("==File list end")); } if (line.split("=").length < 2) { continue; } String type = line.split("=")[0].trim(); String value = line.split("=")[1].trim(); switch (type) { case "Path": { WorkFolder = value; break; } case "path": { WorkFolder = value; break; } case "Thread": { NoCPUs = Integer.parseInt(value); break; } case "Fasta": { tandemPara.FastaPath = value; break; } case "Combined_Prot": { Combined_Prot = value; break; } case "DefaultProtFiltering": { DefaultProtFiltering = Boolean.parseBoolean(value); break; } case "DecoyPrefix": { if (!"".equals(value)) { tandemPara.DecoyPrefix = value; } break; } case "ProteinFDR": { tandemPara.ProtFDR = Float.parseFloat(value); break; } case "FilterWeight": { FilterWeight = value; break; } case "MinWeight": { MinWeight = Float.parseFloat(value); break; } case "TopNFrag": { TopNFrag = Integer.parseInt(value); break; } case "TopNPep": { TopNPep = Integer.parseInt(value); break; } case "Freq": { Freq = Float.parseFloat(value); break; } //<editor-fold defaultstate="collapsed" desc="SaintOutput"> case "ExportSaintInput": { ExportSaint = Boolean.parseBoolean(value); break; } case "QuantitationType": { switch (value) { case "MS1": { SAINT_MS1 = true; SAINT_MS2 = false; break; } case "MS2": { SAINT_MS1 = false; SAINT_MS2 = true; break; } case "BOTH": { SAINT_MS1 = true; SAINT_MS2 = true; break; } } break; } // case "BaitInputFile": { // SaintBaitFile = value; // break; // } // case "PreyInputFile": { // SaintPreyFile = value; // break; // } // case "InterationInputFile": { // SaintInteractionFile = value; // break; // } default: { if (type.startsWith("BaitName_")) { BaitName.put(type.substring(9), value); } if (type.startsWith("BaitFile_")) { BaitList.put(type.substring(9), value.split("\t")); } if (type.startsWith("ControlName_")) { ControlName.put(type.substring(12), value); } if (type.startsWith("ControlFile_")) { ControlList.put(type.substring(12), value.split("\t")); } break; } //</editor-fold> } } } //</editor-fold> //Initialize PTM manager using compomics library PTMManager.GetInstance(); //Check if the fasta file can be found if (!new File(tandemPara.FastaPath).exists()) { Logger.getRootLogger().info("Fasta file :" + tandemPara.FastaPath + " cannot be found, the process will be terminated, please check."); System.exit(1); } //Check if the prot.xml file can be found if (!new File(Combined_Prot).exists()) { Logger.getRootLogger().info("ProtXML file: " + Combined_Prot + " cannot be found, the export protein summary table will be empty."); } LCMSID protID = null; //Parse prot.xml and generate protein master list given an FDR if (Combined_Prot != null && !Combined_Prot.equals("")) { protID = LCMSID.ReadLCMSIDSerialization(Combined_Prot); if (!"".equals(Combined_Prot) && protID == null) { protID = new LCMSID(Combined_Prot, tandemPara.DecoyPrefix, tandemPara.FastaPath); ProtXMLParser protxmlparser = new ProtXMLParser(protID, Combined_Prot, 0f); //Use DIA-Umpire default protein FDR calculation if (DefaultProtFiltering) { protID.RemoveLowLocalPWProtein(0.8f); protID.RemoveLowMaxIniProbProtein(0.9f); protID.FilterByProteinDecoyFDRUsingMaxIniProb(tandemPara.DecoyPrefix, tandemPara.ProtFDR); } //Get protein FDR calculation without other filtering else { protID.FilterByProteinDecoyFDRUsingLocalPW(tandemPara.DecoyPrefix, tandemPara.ProtFDR); } protID.LoadSequence(); protID.WriteLCMSIDSerialization(Combined_Prot); } Logger.getRootLogger().info("Protein No.:" + protID.ProteinList.size()); } HashMap<String, HashMap<String, FragmentPeak>> IDSummaryFragments = new HashMap<>(); //Generate DIA file list ArrayList<DIAPack> FileList = new ArrayList<>(); try { File folder = new File(WorkFolder); if (!folder.exists()) { Logger.getRootLogger().info("The path : " + WorkFolder + " cannot be found."); System.exit(1); } for (final File fileEntry : folder.listFiles()) { if (fileEntry.isFile() && (fileEntry.getAbsolutePath().toLowerCase().endsWith(".mzxml") | fileEntry.getAbsolutePath().toLowerCase().endsWith(".mzml")) && !fileEntry.getAbsolutePath().toLowerCase().endsWith("q1.mzxml") && !fileEntry.getAbsolutePath().toLowerCase().endsWith("q2.mzxml") && !fileEntry.getAbsolutePath().toLowerCase().endsWith("q3.mzxml")) { AssignFiles.put(fileEntry.getAbsolutePath(), fileEntry); } if (fileEntry.isDirectory()) { for (final File fileEntry2 : fileEntry.listFiles()) { if (fileEntry2.isFile() && (fileEntry2.getAbsolutePath().toLowerCase().endsWith(".mzxml") | fileEntry2.getAbsolutePath().toLowerCase().endsWith(".mzml")) && !fileEntry2.getAbsolutePath().toLowerCase().endsWith("q1.mzxml") && !fileEntry2.getAbsolutePath().toLowerCase().endsWith("q2.mzxml") && !fileEntry2.getAbsolutePath().toLowerCase().endsWith("q3.mzxml")) { AssignFiles.put(fileEntry2.getAbsolutePath(), fileEntry2); } } } } Logger.getRootLogger().info("No. of files assigned :" + AssignFiles.size()); for (File fileEntry : AssignFiles.values()) { Logger.getRootLogger().info(fileEntry.getAbsolutePath()); } for (File fileEntry : AssignFiles.values()) { String mzXMLFile = fileEntry.getAbsolutePath(); if (mzXMLFile.toLowerCase().endsWith(".mzxml") | mzXMLFile.toLowerCase().endsWith(".mzml")) { DIAPack DiaFile = new DIAPack(mzXMLFile, NoCPUs); Logger.getRootLogger().info( "================================================================================================="); Logger.getRootLogger().info("Processing " + mzXMLFile); if (!DiaFile.LoadDIASetting()) { Logger.getRootLogger().info("Loading DIA setting failed, job is incomplete"); System.exit(1); } if (!DiaFile.LoadParams()) { Logger.getRootLogger().info("Loading parameters failed, job is incomplete"); System.exit(1); } Logger.getRootLogger().info("Loading identification results " + mzXMLFile + "...."); //If the serialization file for ID file existed if (DiaFile.ReadSerializedLCMSID()) { DiaFile.IDsummary.ReduceMemoryUsage(); DiaFile.IDsummary.ClearAssignPeakCluster(); FileList.add(DiaFile); HashMap<String, FragmentPeak> FragMap = new HashMap<>(); IDSummaryFragments.put(FilenameUtils.getBaseName(mzXMLFile), FragMap); } } } //<editor-fold defaultstate="collapsed" desc="Peptide and fragment selection"> Logger.getRootLogger().info("Peptide and fragment selection across the whole dataset"); ArrayList<LCMSID> SummaryList = new ArrayList<>(); for (DIAPack diafile : FileList) { if (protID != null) { //Generate protein list according to mapping of peptide ions for each DIA file to the master protein list diafile.IDsummary.GenerateProteinByRefIDByPepSeq(protID, true); diafile.IDsummary.ReMapProPep(); } if ("GW".equals(FilterWeight)) { diafile.IDsummary.SetFilterByGroupWeight(); } else if ("PepW".equals(FilterWeight)) { diafile.IDsummary.SetFilterByWeight(); } SummaryList.add(diafile.IDsummary); } FragmentSelection fragselection = new FragmentSelection(SummaryList); fragselection.freqPercent = Freq; fragselection.GeneratePepFragScoreMap(); fragselection.GenerateTopFragMap(TopNFrag); fragselection.GenerateProtPepScoreMap(MinWeight); fragselection.GenerateTopPepMap(TopNPep); //</editor-fold> //<editor-fold defaultstate="collapsed" desc="Writing general reports"> ExportTable export = new ExportTable(WorkFolder, SummaryList, IDSummaryFragments, protID, fragselection); export.Export(TopNPep, TopNFrag, Freq); //</editor-fold> //<editor-fold defaultstate="collapsed" desc="//<editor-fold defaultstate="collapsed" desc="Generate SAINT input files"> if (ExportSaint && protID != null) { HashMap<String, DIAPack> Filemap = new HashMap<>(); for (DIAPack DIAfile : FileList) { Filemap.put(DIAfile.GetBaseName(), DIAfile); } FileWriter baitfile = new FileWriter(WorkFolder + "SAINT_Bait_" + DateTimeTag.GetTag() + ".txt"); FileWriter preyfile = new FileWriter(WorkFolder + "SAINT_Prey_" + DateTimeTag.GetTag() + ".txt"); FileWriter interactionfileMS1 = null; FileWriter interactionfileMS2 = null; if (SAINT_MS1) { interactionfileMS1 = new FileWriter( WorkFolder + "SAINT_Interaction_MS1_" + DateTimeTag.GetTag() + ".txt"); } if (SAINT_MS2) { interactionfileMS2 = new FileWriter( WorkFolder + "SAINT_Interaction_MS2_" + DateTimeTag.GetTag() + ".txt"); } HashMap<String, String> PreyID = new HashMap<>(); for (String samplekey : ControlName.keySet()) { String name = ControlName.get(samplekey); for (String file : ControlList.get(samplekey)) { baitfile.write(FilenameUtils.getBaseName(file) + "\t" + name + "\t" + "C\n"); LCMSID IDsummary = Filemap.get(FilenameUtils.getBaseName(file)).IDsummary; if (SAINT_MS1) { SaintOutput(protID, IDsummary, fragselection, interactionfileMS1, file, name, PreyID, 1); } if (SAINT_MS2) { SaintOutput(protID, IDsummary, fragselection, interactionfileMS2, file, name, PreyID, 2); } } } for (String samplekey : BaitName.keySet()) { String name = BaitName.get(samplekey); for (String file : BaitList.get(samplekey)) { baitfile.write(FilenameUtils.getBaseName(file) + "\t" + name + "\t" + "T\n"); LCMSID IDsummary = Filemap.get(FilenameUtils.getBaseName(file)).IDsummary; if (SAINT_MS1) { SaintOutput(protID, IDsummary, fragselection, interactionfileMS1, file, name, PreyID, 1); } if (SAINT_MS2) { SaintOutput(protID, IDsummary, fragselection, interactionfileMS2, file, name, PreyID, 2); } } } baitfile.close(); if (SAINT_MS1) { interactionfileMS1.close(); } if (SAINT_MS2) { interactionfileMS2.close(); } for (String AccNo : PreyID.keySet()) { preyfile.write(AccNo + "\t" + PreyID.get(AccNo) + "\n"); } preyfile.close(); } //</editor-fold> Logger.getRootLogger().info("Job done"); Logger.getRootLogger().info( "================================================================================================="); } catch (Exception e) { Logger.getRootLogger().error(ExceptionUtils.getStackTrace(e)); throw e; } }
From source file:com.cic.datacrawl.ui.Main.java
/** * Main entry point. Creates a debugger attached to a Rhino * {@link com.cic.datacrawl.ui.shell.Shell} shell session. */// www . j a v a 2 s . co m public static void main(final String[] args) { if (System.getProperties().get("os.name").toString().toLowerCase().indexOf("linux") >= 0) System.setProperty("sun.awt.xembedserver", "true"); try { String path = null; boolean reflash = true; if (args != null && args.length > 0) { if (args[0].equalsIgnoreCase("-h")) { System.out.println("Command Format: \n" + "\t%JAVA_HOME%\\BIN\\JAVA -jar homepageCatcher.jar " + "[-d config path]\n" + "\t%JAVA_HOME%\\BIN\\JAVA -jar homepageCatcher.jar " + "[-d config path_1;path_2;....;path_n]\n"); } int pathIndex = ArrayUtils.indexOf(args, "-d"); if (pathIndex >= 0) { try { path = args[pathIndex + 1]; File f = new File(path); if (!f.exists()) { LOG.warn("Invalid path of configuration. " + "Using default configuration."); } } catch (Throwable e) { LOG.warn("Invalid path of configuration. " + "Using default configuration."); } } // int reflashIndex = ArrayUtils.indexOf(args, "-r"); // // reflash = new Boolean(args[reflashIndex + 1]).booleanValue(); } if (path == null || path.trim().length() == 0) path = Config.INSTALL_PATH + File.separator + "config" + File.separator + "beans"; LOG.debug("Config Path: \"" + path + "\""); // ?IOC // ???? // ? ApplicationContext.initialiaze(path, reflash); System.setProperty(ApplicationContext.CONFIG_PATH, path); // js??? InitializerRegister.getInstance().execute(); // ??? //VerifyCodeInputDialog.init(); // ?? //initTestData(); // ?GUI Thread thread = new Thread(new Runnable() { @Override public void run() { startupGUI(args); } }); thread.setName("UI_Thread"); thread.start(); } catch (Throwable e) { LOG.error(e.getMessage(), e); } }
From source file:edu.nyu.vida.data_polygamy.feature_identification.IndexCreation.java
/** * @param args/*from w w w .ja va 2s . com*/ */ @SuppressWarnings({ "deprecation" }) public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException { Options options = new Options(); Option forceOption = new Option("f", "force", false, "force the computation of the index and events " + "even if files already exist"); forceOption.setRequired(false); options.addOption(forceOption); Option thresholdOption = new Option("t", "use-custom-thresholds", false, "use custom thresholds for regular and rare events, defined in HDFS_HOME/" + FrameworkUtils.thresholdDir + " file"); thresholdOption.setRequired(false); options.addOption(thresholdOption); Option gOption = new Option("g", "group", true, "set group of datasets for which the indices and events" + " will be computed"); gOption.setRequired(true); gOption.setArgName("GROUP"); gOption.setArgs(Option.UNLIMITED_VALUES); options.addOption(gOption); Option machineOption = new Option("m", "machine", true, "machine identifier"); machineOption.setRequired(true); machineOption.setArgName("MACHINE"); machineOption.setArgs(1); options.addOption(machineOption); Option nodesOption = new Option("n", "nodes", true, "number of nodes"); nodesOption.setRequired(true); nodesOption.setArgName("NODES"); nodesOption.setArgs(1); options.addOption(nodesOption); Option s3Option = new Option("s3", "s3", false, "data on Amazon S3"); s3Option.setRequired(false); options.addOption(s3Option); Option awsAccessKeyIdOption = new Option("aws_id", "aws-id", true, "aws access key id; " + "this is required if the execution is on aws"); awsAccessKeyIdOption.setRequired(false); awsAccessKeyIdOption.setArgName("AWS-ACCESS-KEY-ID"); awsAccessKeyIdOption.setArgs(1); options.addOption(awsAccessKeyIdOption); Option awsSecretAccessKeyOption = new Option("aws_key", "aws-id", true, "aws secrect access key; " + "this is required if the execution is on aws"); awsSecretAccessKeyOption.setRequired(false); awsSecretAccessKeyOption.setArgName("AWS-SECRET-ACCESS-KEY"); awsSecretAccessKeyOption.setArgs(1); options.addOption(awsSecretAccessKeyOption); Option bucketOption = new Option("b", "s3-bucket", true, "bucket on s3; " + "this is required if the execution is on aws"); bucketOption.setRequired(false); bucketOption.setArgName("S3-BUCKET"); bucketOption.setArgs(1); options.addOption(bucketOption); Option helpOption = new Option("h", "help", false, "display this message"); helpOption.setRequired(false); options.addOption(helpOption); HelpFormatter formatter = new HelpFormatter(); CommandLineParser parser = new PosixParser(); CommandLine cmd = null; try { cmd = parser.parse(options, args); } catch (ParseException e) { formatter.printHelp("hadoop jar data-polygamy.jar " + "edu.nyu.vida.data_polygamy.feature_identification.IndexCreation", options, true); System.exit(0); } if (cmd.hasOption("h")) { formatter.printHelp("hadoop jar data-polygamy.jar " + "edu.nyu.vida.data_polygamy.feature_identification.IndexCreation", options, true); System.exit(0); } boolean s3 = cmd.hasOption("s3"); String s3bucket = ""; String awsAccessKeyId = ""; String awsSecretAccessKey = ""; if (s3) { if ((!cmd.hasOption("aws_id")) || (!cmd.hasOption("aws_key")) || (!cmd.hasOption("b"))) { System.out.println( "Arguments 'aws_id', 'aws_key', and 'b'" + " are mandatory if execution is on AWS."); formatter.printHelp("hadoop jar data-polygamy.jar " + "edu.nyu.vida.data_polygamy.feature_identification.IndexCreation", options, true); System.exit(0); } s3bucket = cmd.getOptionValue("b"); awsAccessKeyId = cmd.getOptionValue("aws_id"); awsSecretAccessKey = cmd.getOptionValue("aws_key"); } boolean snappyCompression = false; boolean bzip2Compression = false; String machine = cmd.getOptionValue("m"); int nbNodes = Integer.parseInt(cmd.getOptionValue("n")); Configuration s3conf = new Configuration(); if (s3) { s3conf.set("fs.s3.awsAccessKeyId", awsAccessKeyId); s3conf.set("fs.s3.awsSecretAccessKey", awsSecretAccessKey); s3conf.set("bucket", s3bucket); } String datasetNames = ""; String datasetIds = ""; ArrayList<String> shortDataset = new ArrayList<String>(); ArrayList<String> shortDatasetIndex = new ArrayList<String>(); HashMap<String, String> datasetAgg = new HashMap<String, String>(); HashMap<String, String> datasetId = new HashMap<String, String>(); HashMap<String, HashMap<Integer, Double>> datasetRegThreshold = new HashMap<String, HashMap<Integer, Double>>(); HashMap<String, HashMap<Integer, Double>> datasetRareThreshold = new HashMap<String, HashMap<Integer, Double>>(); Path path = null; FileSystem fs = FileSystem.get(new Configuration()); BufferedReader br; boolean removeExistingFiles = cmd.hasOption("f"); boolean isThresholdUserDefined = cmd.hasOption("t"); for (String dataset : cmd.getOptionValues("g")) { // getting aggregates String[] aggregate = FrameworkUtils.searchAggregates(dataset, s3conf, s3); if (aggregate.length == 0) { System.out.println("No aggregates found for " + dataset + "."); continue; } // getting aggregates header String aggregatesHeaderFileName = FrameworkUtils.searchAggregatesHeader(dataset, s3conf, s3); if (aggregatesHeaderFileName == null) { System.out.println("No aggregate header for " + dataset); continue; } String aggregatesHeader = s3bucket + FrameworkUtils.preProcessingDir + "/" + aggregatesHeaderFileName; shortDataset.add(dataset); datasetId.put(dataset, null); if (s3) { path = new Path(aggregatesHeader); fs = FileSystem.get(path.toUri(), s3conf); } else { path = new Path(fs.getHomeDirectory() + "/" + aggregatesHeader); } br = new BufferedReader(new InputStreamReader(fs.open(path))); datasetAgg.put(dataset, br.readLine().split("\t")[1]); br.close(); if (s3) fs.close(); } if (shortDataset.size() == 0) { System.out.println("No datasets to process."); System.exit(0); } // getting dataset id if (s3) { path = new Path(s3bucket + FrameworkUtils.datasetsIndexDir); fs = FileSystem.get(path.toUri(), s3conf); } else { path = new Path(fs.getHomeDirectory() + "/" + FrameworkUtils.datasetsIndexDir); } br = new BufferedReader(new InputStreamReader(fs.open(path))); String line = br.readLine(); while (line != null) { String[] dt = line.split("\t"); if (datasetId.containsKey(dt[0])) { datasetId.put(dt[0], dt[1]); datasetNames += dt[0] + ","; datasetIds += dt[1] + ","; } line = br.readLine(); } br.close(); datasetNames = datasetNames.substring(0, datasetNames.length() - 1); datasetIds = datasetIds.substring(0, datasetIds.length() - 1); Iterator<String> it = shortDataset.iterator(); while (it.hasNext()) { String dataset = it.next(); if (datasetId.get(dataset) == null) { System.out.println("No dataset id for " + dataset); System.exit(0); } } // getting user defined thresholds if (isThresholdUserDefined) { if (s3) { path = new Path(s3bucket + FrameworkUtils.thresholdDir); fs = FileSystem.get(path.toUri(), s3conf); } else { path = new Path(fs.getHomeDirectory() + "/" + FrameworkUtils.thresholdDir); } br = new BufferedReader(new InputStreamReader(fs.open(path))); line = br.readLine(); while (line != null) { // getting dataset name String dataset = line.trim(); HashMap<Integer, Double> regThresholds = new HashMap<Integer, Double>(); HashMap<Integer, Double> rareThresholds = new HashMap<Integer, Double>(); line = br.readLine(); while ((line != null) && (line.split("\t").length > 1)) { // getting attribute ids and thresholds String[] keyVals = line.trim().split("\t"); int att = Integer.parseInt(keyVals[0].trim()); regThresholds.put(att, Double.parseDouble(keyVals[1].trim())); rareThresholds.put(att, Double.parseDouble(keyVals[2].trim())); line = br.readLine(); } datasetRegThreshold.put(dataset, regThresholds); datasetRareThreshold.put(dataset, rareThresholds); } br.close(); } if (s3) fs.close(); // datasets that will use existing merge tree ArrayList<String> useMergeTree = new ArrayList<String>(); // creating index for each spatio-temporal resolution FrameworkUtils.createDir(s3bucket + FrameworkUtils.indexDir, s3conf, s3); HashSet<String> input = new HashSet<String>(); for (String dataset : shortDataset) { String indexCreationOutputFileName = s3bucket + FrameworkUtils.indexDir + "/" + dataset + "/"; String mergeTreeFileName = s3bucket + FrameworkUtils.mergeTreeDir + "/" + dataset + "/"; if (removeExistingFiles) { FrameworkUtils.removeFile(indexCreationOutputFileName, s3conf, s3); FrameworkUtils.removeFile(mergeTreeFileName, s3conf, s3); FrameworkUtils.createDir(mergeTreeFileName, s3conf, s3); } else if (datasetRegThreshold.containsKey(dataset)) { FrameworkUtils.removeFile(indexCreationOutputFileName, s3conf, s3); if (FrameworkUtils.fileExists(mergeTreeFileName, s3conf, s3)) { useMergeTree.add(dataset); } } if (!FrameworkUtils.fileExists(indexCreationOutputFileName, s3conf, s3)) { input.add(s3bucket + FrameworkUtils.aggregatesDir + "/" + dataset); shortDatasetIndex.add(dataset); } } if (input.isEmpty()) { System.out.println("All the input datasets have indices."); System.out.println("Use -f in the beginning of the command line to force the computation."); System.exit(0); } String aggregateDatasets = ""; it = input.iterator(); while (it.hasNext()) { aggregateDatasets += it.next() + ","; } Job icJob = null; Configuration icConf = new Configuration(); Machine machineConf = new Machine(machine, nbNodes); String jobName = "index"; String indexOutputDir = s3bucket + FrameworkUtils.indexDir + "/tmp/"; FrameworkUtils.removeFile(indexOutputDir, s3conf, s3); icConf.set("dataset-name", datasetNames); icConf.set("dataset-id", datasetIds); if (!useMergeTree.isEmpty()) { String useMergeTreeStr = ""; for (String dt : useMergeTree) { useMergeTreeStr += dt + ","; } icConf.set("use-merge-tree", useMergeTreeStr.substring(0, useMergeTreeStr.length() - 1)); } for (int i = 0; i < shortDataset.size(); i++) { String dataset = shortDataset.get(i); String id = datasetId.get(dataset); icConf.set("dataset-" + id + "-aggregates", datasetAgg.get(dataset)); if (datasetRegThreshold.containsKey(dataset)) { HashMap<Integer, Double> regThresholds = datasetRegThreshold.get(dataset); String thresholds = ""; for (int att : regThresholds.keySet()) { thresholds += String.valueOf(att) + "-" + String.valueOf(regThresholds.get(att)) + ","; } icConf.set("regular-" + id, thresholds.substring(0, thresholds.length() - 1)); } if (datasetRareThreshold.containsKey(dataset)) { HashMap<Integer, Double> rareThresholds = datasetRareThreshold.get(dataset); String thresholds = ""; for (int att : rareThresholds.keySet()) { thresholds += String.valueOf(att) + "-" + String.valueOf(rareThresholds.get(att)) + ","; } icConf.set("rare-" + id, thresholds.substring(0, thresholds.length() - 1)); } } icConf.set("mapreduce.tasktracker.map.tasks.maximum", String.valueOf(machineConf.getMaximumTasks())); icConf.set("mapreduce.tasktracker.reduce.tasks.maximum", String.valueOf(machineConf.getMaximumTasks())); icConf.set("mapreduce.jobtracker.maxtasks.perjob", "-1"); icConf.set("mapreduce.reduce.shuffle.parallelcopies", "20"); icConf.set("mapreduce.input.fileinputformat.split.minsize", "0"); icConf.set("mapreduce.task.io.sort.mb", "200"); icConf.set("mapreduce.task.io.sort.factor", "100"); //icConf.set("mapreduce.task.timeout", "1800000"); machineConf.setMachineConfiguration(icConf); if (s3) { machineConf.setMachineConfiguration(icConf); icConf.set("fs.s3.awsAccessKeyId", awsAccessKeyId); icConf.set("fs.s3.awsSecretAccessKey", awsSecretAccessKey); icConf.set("bucket", s3bucket); } if (snappyCompression) { icConf.set("mapreduce.map.output.compress", "true"); icConf.set("mapreduce.map.output.compress.codec", "org.apache.hadoop.io.compress.SnappyCodec"); //icConf.set("mapreduce.output.fileoutputformat.compress.codec", "org.apache.hadoop.io.compress.SnappyCodec"); } if (bzip2Compression) { icConf.set("mapreduce.map.output.compress", "true"); icConf.set("mapreduce.map.output.compress.codec", "org.apache.hadoop.io.compress.BZip2Codec"); //icConf.set("mapreduce.output.fileoutputformat.compress.codec", "org.apache.hadoop.io.compress.BZip2Codec"); } icJob = new Job(icConf); icJob.setJobName(jobName); icJob.setMapOutputKeyClass(AttributeResolutionWritable.class); icJob.setMapOutputValueClass(SpatioTemporalFloatWritable.class); icJob.setOutputKeyClass(AttributeResolutionWritable.class); icJob.setOutputValueClass(TopologyTimeSeriesWritable.class); //icJob.setOutputKeyClass(Text.class); //icJob.setOutputValueClass(Text.class); icJob.setMapperClass(IndexCreationMapper.class); icJob.setReducerClass(IndexCreationReducer.class); icJob.setNumReduceTasks(machineConf.getNumberReduces()); icJob.setInputFormatClass(SequenceFileInputFormat.class); //icJob.setOutputFormatClass(SequenceFileOutputFormat.class); LazyOutputFormat.setOutputFormatClass(icJob, SequenceFileOutputFormat.class); //LazyOutputFormat.setOutputFormatClass(icJob, TextOutputFormat.class); SequenceFileOutputFormat.setCompressOutput(icJob, true); SequenceFileOutputFormat.setOutputCompressionType(icJob, CompressionType.BLOCK); FileInputFormat.setInputDirRecursive(icJob, true); FileInputFormat.setInputPaths(icJob, aggregateDatasets.substring(0, aggregateDatasets.length() - 1)); FileOutputFormat.setOutputPath(icJob, new Path(indexOutputDir)); icJob.setJarByClass(IndexCreation.class); long start = System.currentTimeMillis(); icJob.submit(); icJob.waitForCompletion(true); System.out.println(jobName + "\t" + (System.currentTimeMillis() - start)); // moving files to right place for (String dataset : shortDatasetIndex) { String from = s3bucket + FrameworkUtils.indexDir + "/tmp/" + dataset + "/"; String to = s3bucket + FrameworkUtils.indexDir + "/" + dataset + "/"; FrameworkUtils.renameFile(from, to, s3conf, s3); } }
From source file:com.searchcode.app.App.java
public static void main(String[] args) { injector = Guice.createInjector(new InjectorConfig()); int server_port = Helpers.tryParseInt( Properties.getProperties().getProperty(Values.SERVERPORT, Values.DEFAULTSERVERPORT), Values.DEFAULTSERVERPORT);/*from w w w .ja v a 2 s .c om*/ boolean onlyLocalhost = Boolean .parseBoolean(Properties.getProperties().getProperty("only_localhost", "false")); // Database migrations happen before we start databaseMigrations(); LOGGER.info("Starting searchcode server on port " + server_port); Spark.port(server_port); JobService js = injector.getInstance(JobService.class); Repo repo = injector.getInstance(Repo.class); Data data = injector.getInstance(Data.class); Api api = injector.getInstance(Api.class); ApiService apiService = injector.getInstance(ApiService.class); StatsService statsService = new StatsService(); scl = Singleton.getSearchcodeLib(data); js.initialJobs(); Gson gson = new Gson(); Spark.staticFileLocation("/public"); before((request, response) -> { if (onlyLocalhost) { if (!request.ip().equals("127.0.0.1")) { halt(204); } } }); get("/", (req, res) -> { Map<String, Object> map = new HashMap<>(); map.put("repoCount", repo.getRepoCount()); if (req.queryParams().contains("q") && !req.queryParams("q").trim().equals("")) { String query = req.queryParams("q").trim(); int page = 0; if (req.queryParams().contains("p")) { try { page = Integer.parseInt(req.queryParams("p")); page = page > 19 ? 19 : page; } catch (NumberFormatException ex) { page = 0; } } List<String> reposList = new ArrayList<>(); List<String> langsList = new ArrayList<>(); List<String> ownsList = new ArrayList<>(); if (req.queryParams().contains("repo")) { String[] repos = new String[0]; repos = req.queryParamsValues("repo"); if (repos.length != 0) { reposList = Arrays.asList(repos); } } if (req.queryParams().contains("lan")) { String[] langs = new String[0]; langs = req.queryParamsValues("lan"); if (langs.length != 0) { langsList = Arrays.asList(langs); } } if (req.queryParams().contains("own")) { String[] owns = new String[0]; owns = req.queryParamsValues("own"); if (owns.length != 0) { ownsList = Arrays.asList(owns); } } map.put("searchValue", query); map.put("searchResultJson", gson.toJson(new CodePreload(query, page, langsList, reposList, ownsList))); map.put("logoImage", getLogo()); map.put("isCommunity", ISCOMMUNITY); return new ModelAndView(map, "search_test.ftl"); } // Totally pointless vanity but lets rotate the image every week int photoId = getWeekOfMonth(); if (photoId <= 0) { photoId = 3; } if (photoId > 4) { photoId = 2; } CodeSearcher cs = new CodeSearcher(); map.put("photoId", photoId); map.put("numDocs", cs.getTotalNumberDocumentsIndexed()); map.put("logoImage", getLogo()); map.put("isCommunity", ISCOMMUNITY); return new ModelAndView(map, "index.ftl"); }, new FreeMarkerEngine()); get("/html/", (req, res) -> { CodeSearcher cs = new CodeSearcher(); CodeMatcher cm = new CodeMatcher(data); Map<String, Object> map = new HashMap<>(); map.put("repoCount", repo.getRepoCount()); if (req.queryParams().contains("q")) { String query = req.queryParams("q").trim(); String altquery = query.replaceAll("[^A-Za-z0-9 ]", " ").trim().replaceAll(" +", " "); int page = 0; if (req.queryParams().contains("p")) { try { page = Integer.parseInt(req.queryParams("p")); page = page > 19 ? 19 : page; } catch (NumberFormatException ex) { page = 0; } } String[] repos = new String[0]; String[] langs = new String[0]; String reposFilter = ""; String langsFilter = ""; String reposQueryString = ""; String langsQueryString = ""; if (req.queryParams().contains("repo")) { repos = req.queryParamsValues("repo"); if (repos.length != 0) { List<String> reposList = Arrays.asList(repos).stream() .map((s) -> "reponame:" + QueryParser.escape(s)).collect(Collectors.toList()); reposFilter = " && (" + StringUtils.join(reposList, " || ") + ")"; List<String> reposQueryList = Arrays.asList(repos).stream() .map((s) -> "&repo=" + URLEncoder.encode(s)).collect(Collectors.toList()); reposQueryString = StringUtils.join(reposQueryList, ""); } } if (req.queryParams().contains("lan")) { langs = req.queryParamsValues("lan"); if (langs.length != 0) { List<String> langsList = Arrays.asList(langs).stream() .map((s) -> "languagename:" + QueryParser.escape(s)).collect(Collectors.toList()); langsFilter = " && (" + StringUtils.join(langsList, " || ") + ")"; List<String> langsQueryList = Arrays.asList(langs).stream() .map((s) -> "&lan=" + URLEncoder.encode(s)).collect(Collectors.toList()); langsQueryString = StringUtils.join(langsQueryList, ""); } } // split the query escape it and and it together String cleanQueryString = scl.formatQueryString(query); SearchResult searchResult = cs.search(cleanQueryString + reposFilter + langsFilter, page); searchResult.setCodeResultList(cm.formatResults(searchResult.getCodeResultList(), query, true)); for (CodeFacetRepo f : searchResult.getRepoFacetResults()) { if (Arrays.asList(repos).contains(f.getRepoName())) { f.setSelected(true); } } for (CodeFacetLanguage f : searchResult.getLanguageFacetResults()) { if (Arrays.asList(langs).contains(f.getLanguageName())) { f.setSelected(true); } } map.put("searchValue", query); map.put("searchResult", searchResult); map.put("reposQueryString", reposQueryString); map.put("langsQueryString", langsQueryString); map.put("altQuery", altquery); map.put("isHtml", true); map.put("logoImage", getLogo()); map.put("isCommunity", ISCOMMUNITY); return new ModelAndView(map, "searchresults.ftl"); } // Totally pointless vanity but lets rotate the image every week int photoId = getWeekOfMonth(); if (photoId <= 0) { photoId = 3; } if (photoId > 4) { photoId = 2; } map.put("photoId", photoId); map.put("numDocs", cs.getTotalNumberDocumentsIndexed()); map.put("logoImage", getLogo()); map.put("isCommunity", ISCOMMUNITY); return new ModelAndView(map, "index.ftl"); }, new FreeMarkerEngine()); /** * Allows one to write literal lucene search queries against the index * TODO This is still very much WIP */ get("/literal/", (req, res) -> { CodeSearcher cs = new CodeSearcher(); CodeMatcher cm = new CodeMatcher(data); Map<String, Object> map = new HashMap<>(); map.put("repoCount", repo.getRepoCount()); if (req.queryParams().contains("q")) { String query = req.queryParams("q").trim(); int page = 0; if (req.queryParams().contains("p")) { try { page = Integer.parseInt(req.queryParams("p")); page = page > 19 ? 19 : page; } catch (NumberFormatException ex) { page = 0; } } String altquery = query.replaceAll("[^A-Za-z0-9 ]", " ").trim().replaceAll(" +", " "); SearchResult searchResult = cs.search(query, page); searchResult.setCodeResultList(cm.formatResults(searchResult.getCodeResultList(), altquery, false)); map.put("searchValue", query); map.put("searchResult", searchResult); map.put("reposQueryString", ""); map.put("langsQueryString", ""); map.put("altQuery", ""); map.put("logoImage", getLogo()); map.put("isCommunity", ISCOMMUNITY); return new ModelAndView(map, "searchresults.ftl"); } map.put("numDocs", cs.getTotalNumberDocumentsIndexed()); map.put("logoImage", getLogo()); map.put("isCommunity", ISCOMMUNITY); return new ModelAndView(map, "index.ftl"); }, new FreeMarkerEngine()); /** * This is the endpoint used by the frontend. */ get("/api/codesearch/", (req, res) -> { CodeSearcher cs = new CodeSearcher(); CodeMatcher cm = new CodeMatcher(data); if (req.queryParams().contains("q") && req.queryParams("q").trim() != "") { String query = req.queryParams("q").trim(); int page = 0; if (req.queryParams().contains("p")) { try { page = Integer.parseInt(req.queryParams("p")); page = page > 19 ? 19 : page; } catch (NumberFormatException ex) { page = 0; } } String[] repos = new String[0]; String[] langs = new String[0]; String[] owners = new String[0]; String reposFilter = ""; String langsFilter = ""; String ownersFilter = ""; if (req.queryParams().contains("repo")) { repos = req.queryParamsValues("repo"); if (repos.length != 0) { List<String> reposList = Arrays.asList(repos).stream() .map((s) -> "reponame:" + QueryParser.escape(s)).collect(Collectors.toList()); reposFilter = " && (" + StringUtils.join(reposList, " || ") + ")"; } } if (req.queryParams().contains("lan")) { langs = req.queryParamsValues("lan"); if (langs.length != 0) { List<String> langsList = Arrays.asList(langs).stream() .map((s) -> "languagename:" + QueryParser.escape(s)).collect(Collectors.toList()); langsFilter = " && (" + StringUtils.join(langsList, " || ") + ")"; } } if (req.queryParams().contains("own")) { owners = req.queryParamsValues("own"); if (owners.length != 0) { List<String> ownersList = Arrays.asList(owners).stream() .map((s) -> "codeowner:" + QueryParser.escape(s)).collect(Collectors.toList()); ownersFilter = " && (" + StringUtils.join(ownersList, " || ") + ")"; } } // Need to pass in the filters into this query String cacheKey = query + page + reposFilter + langsFilter + ownersFilter; if (cache.containsKey(cacheKey)) { return cache.get(cacheKey); } // split the query escape it and and it together String cleanQueryString = scl.formatQueryString(query); SearchResult searchResult = cs.search(cleanQueryString + reposFilter + langsFilter + ownersFilter, page); searchResult.setCodeResultList(cm.formatResults(searchResult.getCodeResultList(), query, true)); searchResult.setQuery(query); for (String altQuery : scl.generateAltQueries(query)) { searchResult.addAltQuery(altQuery); } // Null out code as it isnt required and there is no point in bloating our ajax requests for (CodeResult codeSearchResult : searchResult.getCodeResultList()) { codeSearchResult.setCode(null); } cache.put(cacheKey, searchResult); return searchResult; } return null; }, new JsonTransformer()); get("/api/repo/add/", "application/json", (request, response) -> { boolean apiEnabled = Boolean .parseBoolean(Properties.getProperties().getProperty("api_enabled", "false")); boolean apiAuth = Boolean .parseBoolean(Properties.getProperties().getProperty("api_key_authentication", "true")); if (!apiEnabled) { return new ApiResponse(false, "API not enabled"); } String publicKey = request.queryParams("pub"); String signedKey = request.queryParams("sig"); String reponames = request.queryParams("reponame"); String repourls = request.queryParams("repourl"); String repotype = request.queryParams("repotype"); String repousername = request.queryParams("repousername"); String repopassword = request.queryParams("repopassword"); String reposource = request.queryParams("reposource"); String repobranch = request.queryParams("repobranch"); if (reponames == null || reponames.trim().equals(Values.EMPTYSTRING)) { return new ApiResponse(false, "reponame is a required parameter"); } if (repourls == null || repourls.trim().equals(Values.EMPTYSTRING)) { return new ApiResponse(false, "repourl is a required parameter"); } if (repotype == null) { return new ApiResponse(false, "repotype is a required parameter"); } if (repousername == null) { return new ApiResponse(false, "repousername is a required parameter"); } if (repopassword == null) { return new ApiResponse(false, "repopassword is a required parameter"); } if (reposource == null) { return new ApiResponse(false, "reposource is a required parameter"); } if (repobranch == null) { return new ApiResponse(false, "repobranch is a required parameter"); } if (apiAuth) { if (publicKey == null || publicKey.trim().equals(Values.EMPTYSTRING)) { return new ApiResponse(false, "pub is a required parameter"); } if (signedKey == null || signedKey.trim().equals(Values.EMPTYSTRING)) { return new ApiResponse(false, "sig is a required parameter"); } String toValidate = String.format( "pub=%s&reponame=%s&repourl=%s&repotype=%s&repousername=%s&repopassword=%s&reposource=%s&repobranch=%s", URLEncoder.encode(publicKey), URLEncoder.encode(reponames), URLEncoder.encode(repourls), URLEncoder.encode(repotype), URLEncoder.encode(repousername), URLEncoder.encode(repopassword), URLEncoder.encode(reposource), URLEncoder.encode(repobranch)); boolean validRequest = apiService.validateRequest(publicKey, signedKey, toValidate); if (!validRequest) { return new ApiResponse(false, "invalid signed url"); } } // Clean if (repobranch == null || repobranch.trim().equals(Values.EMPTYSTRING)) { repobranch = "master"; } repotype = repotype.trim().toLowerCase(); if (!"git".equals(repotype) && !"svn".equals(repotype)) { repotype = "git"; } RepoResult repoResult = repo.getRepoByName(reponames); if (repoResult != null) { return new ApiResponse(false, "repository name already exists"); } repo.saveRepo(new RepoResult(-1, reponames, repotype, repourls, repousername, repopassword, reposource, repobranch)); return new ApiResponse(true, "added repository successfully"); }, new JsonTransformer()); get("/api/repo/delete/", "application/json", (request, response) -> { boolean apiEnabled = Boolean .parseBoolean(Properties.getProperties().getProperty("api_enabled", "false")); boolean apiAuth = Boolean .parseBoolean(Properties.getProperties().getProperty("api_key_authentication", "true")); if (!apiEnabled) { return new ApiResponse(false, "API not enabled"); } String publicKey = request.queryParams("pub"); String signedKey = request.queryParams("sig"); String reponames = request.queryParams("reponame"); if (reponames == null || reponames.trim().equals(Values.EMPTYSTRING)) { return new ApiResponse(false, "reponame is a required parameter"); } if (apiAuth) { if (publicKey == null || publicKey.trim().equals(Values.EMPTYSTRING)) { return new ApiResponse(false, "pub is a required parameter"); } if (signedKey == null || signedKey.trim().equals(Values.EMPTYSTRING)) { return new ApiResponse(false, "sig is a required parameter"); } String toValidate = String.format("pub=%s&reponame=%s", URLEncoder.encode(publicKey), URLEncoder.encode(reponames)); boolean validRequest = apiService.validateRequest(publicKey, signedKey, toValidate); if (!validRequest) { return new ApiResponse(false, "invalid signed url"); } } RepoResult rr = repo.getRepoByName(reponames); if (rr == null) { return new ApiResponse(false, "repository already deleted"); } Singleton.getUniqueDeleteRepoQueue().add(rr); return new ApiResponse(true, "repository queued for deletion"); }, new JsonTransformer()); get("/api/repo/list/", "application/json", (request, response) -> { boolean apiEnabled = Boolean .parseBoolean(Properties.getProperties().getProperty("api_enabled", "false")); boolean apiAuth = Boolean .parseBoolean(Properties.getProperties().getProperty("api_key_authentication", "true")); if (!apiEnabled) { return new ApiResponse(false, "API not enabled"); } String publicKey = request.queryParams("pub"); String signedKey = request.queryParams("sig"); if (apiAuth) { if (publicKey == null || publicKey.trim().equals(Values.EMPTYSTRING)) { return new ApiResponse(false, "pub is a required parameter"); } if (signedKey == null || signedKey.trim().equals(Values.EMPTYSTRING)) { return new ApiResponse(false, "sig is a required parameter"); } String toValidate = String.format("pub=%s", URLEncoder.encode(publicKey)); boolean validRequest = apiService.validateRequest(publicKey, signedKey, toValidate); if (!validRequest) { return new ApiResponse(false, "invalid signed url"); } } List<RepoResult> repoResultList = repo.getAllRepo(); return new RepoResultApiResponse(true, Values.EMPTYSTRING, repoResultList); }, new JsonTransformer()); get("/admin/", (request, response) -> { if (getAuthenticatedUser(request) == null) { response.redirect("/login/"); halt(); return null; } CodeSearcher cs = new CodeSearcher(); Map<String, Object> map = new HashMap<>(); map.put("repoCount", repo.getRepoCount()); map.put("numDocs", cs.getTotalNumberDocumentsIndexed()); map.put("numSearches", statsService.getSearchCount()); map.put("uptime", statsService.getUptime()); // Put all properties here map.put(Values.SQLITEFILE, Properties.getProperties().getProperty(Values.SQLITEFILE, Values.DEFAULTSQLITEFILE)); map.put(Values.SERVERPORT, Properties.getProperties().getProperty(Values.SERVERPORT, Values.DEFAULTSERVERPORT)); map.put(Values.REPOSITORYLOCATION, Properties.getProperties().getProperty(Values.REPOSITORYLOCATION, Values.DEFAULTREPOSITORYLOCATION)); map.put(Values.INDEXLOCATION, Properties.getProperties().getProperty(Values.INDEXLOCATION, Values.DEFAULTINDEXLOCATION)); map.put(Values.FACETSLOCATION, Properties.getProperties().getProperty(Values.FACETSLOCATION, Values.DEFAULTFACETSLOCATION)); map.put(Values.CHECKREPOCHANGES, Properties.getProperties().getProperty(Values.CHECKREPOCHANGES, Values.DEFAULTCHECKREPOCHANGES)); map.put(Values.ONLYLOCALHOST, Properties.getProperties().getProperty(Values.ONLYLOCALHOST, Values.DEFAULTONLYLOCALHOST)); map.put(Values.LOWMEMORY, Properties.getProperties().getProperty(Values.LOWMEMORY, Values.DEFAULTLOWMEMORY)); map.put(Values.SPELLINGCORRECTORSIZE, Properties.getProperties() .getProperty(Values.SPELLINGCORRECTORSIZE, Values.DEFAULTSPELLINGCORRECTORSIZE)); map.put(Values.USESYSTEMGIT, Properties.getProperties().getProperty(Values.USESYSTEMGIT, Values.DEFAULTUSESYSTEMGIT)); map.put(Values.GITBINARYPATH, Properties.getProperties().getProperty(Values.GITBINARYPATH, Values.DEFAULTGITBINARYPATH)); map.put(Values.APIENABLED, Properties.getProperties().getProperty(Values.APIENABLED, Values.DEFAULTAPIENABLED)); map.put(Values.APIKEYAUTH, Properties.getProperties().getProperty(Values.APIKEYAUTH, Values.DEFAULTAPIKEYAUTH)); map.put(Values.SVNBINARYPATH, Properties.getProperties().getProperty(Values.SVNBINARYPATH, Values.DEFAULTSVNBINARYPATH)); map.put(Values.SVNENABLED, Properties.getProperties().getProperty(Values.SVNENABLED, Values.DEFAULTSVNENABLED)); map.put(Values.MAXDOCUMENTQUEUESIZE, Properties.getProperties().getProperty(Values.MAXDOCUMENTQUEUESIZE, Values.DEFAULTMAXDOCUMENTQUEUESIZE)); map.put(Values.MAXDOCUMENTQUEUELINESIZE, Properties.getProperties() .getProperty(Values.MAXDOCUMENTQUEUELINESIZE, Values.DEFAULTMAXDOCUMENTQUEUELINESIZE)); map.put(Values.MAXFILELINEDEPTH, Properties.getProperties().getProperty(Values.MAXFILELINEDEPTH, Values.DEFAULTMAXFILELINEDEPTH)); map.put("deletionQueue", Singleton.getUniqueDeleteRepoQueue().size()); map.put("version", VERSION); map.put("logoImage", getLogo()); map.put("isCommunity", ISCOMMUNITY); return new ModelAndView(map, "admin.ftl"); }, new FreeMarkerEngine()); get("/admin/repo/", (request, response) -> { if (getAuthenticatedUser(request) == null) { response.redirect("/login/"); halt(); return null; } int repoCount = repo.getRepoCount(); String offSet = request.queryParams("offset"); String searchQuery = request.queryParams("q"); int indexOffset = 0; Map<String, Object> map = new HashMap<>(); if (offSet != null) { try { indexOffset = Integer.parseInt(offSet); if (indexOffset > repoCount || indexOffset < 0) { indexOffset = 0; } } catch (NumberFormatException ex) { indexOffset = 0; } } if (searchQuery != null) { map.put("repoResults", repo.searchRepo(searchQuery)); } else { map.put("repoResults", repo.getPagedRepo(indexOffset, 100)); } map.put("searchQuery", searchQuery); map.put("hasPrevious", indexOffset > 0); map.put("hasNext", (indexOffset + 100) < repoCount); map.put("previousOffset", "" + (indexOffset - 100)); map.put("nextOffset", "" + (indexOffset + 100)); map.put("logoImage", getLogo()); map.put("isCommunity", ISCOMMUNITY); return new ModelAndView(map, "admin_repo.ftl"); }, new FreeMarkerEngine()); get("/admin/bulk/", (request, response) -> { if (getAuthenticatedUser(request) == null) { response.redirect("/login/"); halt(); return null; } Map<String, Object> map = new HashMap<>(); map.put("logoImage", getLogo()); map.put("isCommunity", ISCOMMUNITY); return new ModelAndView(map, "admin_bulk.ftl"); }, new FreeMarkerEngine()); get("/admin/api/", (request, response) -> { if (getAuthenticatedUser(request) == null) { response.redirect("/login/"); halt(); return null; } Map<String, Object> map = new HashMap<>(); map.put("apiKeys", api.getAllApi()); boolean apiEnabled = Boolean .parseBoolean(Properties.getProperties().getProperty("api_enabled", "false")); boolean apiAuth = Boolean .parseBoolean(Properties.getProperties().getProperty("api_key_authentication", "true")); map.put("apiAuthentication", apiEnabled && apiAuth); map.put("logoImage", getLogo()); map.put("isCommunity", ISCOMMUNITY); return new ModelAndView(map, "admin_api.ftl"); }, new FreeMarkerEngine()); post("/admin/api/", (request, response) -> { if (getAuthenticatedUser(request) == null) { response.redirect("/login/"); halt(); return null; } apiService.createKeys(); response.redirect("/admin/api/"); halt(); return null; }, new FreeMarkerEngine()); get("/admin/api/delete/", "application/json", (request, response) -> { if (getAuthenticatedUser(request) == null || !request.queryParams().contains("publicKey")) { response.redirect("/login/"); halt(); return false; } String publicKey = request.queryParams("publicKey"); apiService.deleteKey(publicKey); return true; }, new JsonTransformer()); get("/admin/settings/", (request, response) -> { if (getAuthenticatedUser(request) == null) { response.redirect("/login/"); halt(); return null; } String[] highlighters = "agate,androidstudio,arta,ascetic,atelier-cave.dark,atelier-cave.light,atelier-dune.dark,atelier-dune.light,atelier-estuary.dark,atelier-estuary.light,atelier-forest.dark,atelier-forest.light,atelier-heath.dark,atelier-heath.light,atelier-lakeside.dark,atelier-lakeside.light,atelier-plateau.dark,atelier-plateau.light,atelier-savanna.dark,atelier-savanna.light,atelier-seaside.dark,atelier-seaside.light,atelier-sulphurpool.dark,atelier-sulphurpool.light,brown_paper,codepen-embed,color-brewer,dark,darkula,default,docco,far,foundation,github-gist,github,googlecode,grayscale,hopscotch,hybrid,idea,ir_black,kimbie.dark,kimbie.light,magula,mono-blue,monokai,monokai_sublime,obsidian,paraiso.dark,paraiso.light,pojoaque,railscasts,rainbow,school_book,solarized_dark,solarized_light,sunburst,tomorrow-night-blue,tomorrow-night-bright,tomorrow-night-eighties,tomorrow-night,tomorrow,vs,xcode,zenburn" .split(","); Map<String, Object> map = new HashMap<>(); map.put("logoImage", getLogo()); map.put("syntaxHighlighter", getSyntaxHighlighter()); map.put("highlighters", highlighters); map.put("averageSalary", "" + (int) getAverageSalary()); map.put("matchLines", "" + (int) getMatchLines()); map.put("maxLineDepth", "" + (int) getMaxLineDepth()); map.put("minifiedLength", "" + (int) getMinifiedLength()); map.put("isCommunity", ISCOMMUNITY); return new ModelAndView(map, "admin_settings.ftl"); }, new FreeMarkerEngine()); get("/admin/reports/", (request, response) -> { if (getAuthenticatedUser(request) == null) { response.redirect("/login/"); halt(); return null; } Map<String, Object> map = new HashMap<>(); map.put("logoImage", getLogo()); map.put("isCommunity", ISCOMMUNITY); return new ModelAndView(map, "admin_reports.ftl"); }, new FreeMarkerEngine()); post("/admin/settings/", (request, response) -> { if (getAuthenticatedUser(request) == null) { response.redirect("/login/"); halt(); return null; } if (ISCOMMUNITY) { response.redirect("/admin/settings/"); halt(); } String logo = request.queryParams("logo").trim(); String syntaxHighlighter = request.queryParams("syntaxhighligher"); try { double averageSalary = Double.parseDouble(request.queryParams("averagesalary")); data.saveData(Values.AVERAGESALARY, "" + (int) averageSalary); } catch (NumberFormatException ex) { data.saveData(Values.AVERAGESALARY, Values.DEFAULTAVERAGESALARY); } try { double averageSalary = Double.parseDouble(request.queryParams("matchlines")); data.saveData(Values.MATCHLINES, "" + (int) averageSalary); } catch (NumberFormatException ex) { data.saveData(Values.MATCHLINES, Values.DEFAULTMATCHLINES); } try { double averageSalary = Double.parseDouble(request.queryParams("maxlinedepth")); data.saveData(Values.MAXLINEDEPTH, "" + (int) averageSalary); } catch (NumberFormatException ex) { data.saveData(Values.MAXLINEDEPTH, Values.DEFAULTMAXLINEDEPTH); } try { double minifiedlength = Double.parseDouble(request.queryParams("minifiedlength")); data.saveData(Values.MINIFIEDLENGTH, "" + (int) minifiedlength); } catch (NumberFormatException ex) { data.saveData(Values.MINIFIEDLENGTH, Values.DEFAULTMINIFIEDLENGTH); } data.saveData(Values.LOGO, logo); data.saveData(Values.SYNTAXHIGHLIGHTER, syntaxHighlighter); // Redo anything that requires updates at this point scl = Singleton.getSearchcodeLib(data); response.redirect("/admin/settings/"); halt(); return null; }, new FreeMarkerEngine()); post("/admin/bulk/", (request, response) -> { if (getAuthenticatedUser(request) == null) { response.redirect("/login/"); halt(); return null; } String repos = request.queryParams("repos"); String repolines[] = repos.split("\\r?\\n"); for (String line : repolines) { String[] repoparams = line.split(",", -1); if (repoparams.length == 7) { String branch = repoparams[6].trim(); if (branch.equals(Values.EMPTYSTRING)) { branch = "master"; } String scm = repoparams[1].trim().toLowerCase(); if (scm.equals(Values.EMPTYSTRING)) { scm = "git"; } RepoResult rr = repo.getRepoByName(repoparams[0]); if (rr == null) { repo.saveRepo(new RepoResult(-1, repoparams[0], scm, repoparams[2], repoparams[3], repoparams[4], repoparams[5], branch)); } } } response.redirect("/admin/bulk/"); halt(); return null; }, new FreeMarkerEngine()); post("/admin/repo/", (request, response) -> { if (getAuthenticatedUser(request) == null) { response.redirect("/login/"); halt(); return null; } String[] reponames = request.queryParamsValues("reponame"); String[] reposcms = request.queryParamsValues("reposcm"); String[] repourls = request.queryParamsValues("repourl"); String[] repousername = request.queryParamsValues("repousername"); String[] repopassword = request.queryParamsValues("repopassword"); String[] reposource = request.queryParamsValues("reposource"); String[] repobranch = request.queryParamsValues("repobranch"); for (int i = 0; i < reponames.length; i++) { if (reponames[i].trim().length() != 0) { String branch = repobranch[i].trim(); if (branch.equals(Values.EMPTYSTRING)) { branch = "master"; } repo.saveRepo(new RepoResult(-1, reponames[i], reposcms[i], repourls[i], repousername[i], repopassword[i], reposource[i], branch)); } } response.redirect("/admin/repo/"); halt(); return null; }, new FreeMarkerEngine()); get("/login/", (request, response) -> { if (getAuthenticatedUser(request) != null) { response.redirect("/admin/"); halt(); return null; } Map<String, Object> map = new HashMap<>(); map.put("logoImage", getLogo()); map.put("isCommunity", ISCOMMUNITY); return new ModelAndView(map, "login.ftl"); }, new FreeMarkerEngine()); post("/login/", (req, res) -> { if (req.queryParams().contains("password") && req.queryParams("password") .equals(com.searchcode.app.util.Properties.getProperties().getProperty("password"))) { addAuthenticatedUser(req); res.redirect("/admin/"); halt(); } Map<String, Object> map = new HashMap<>(); map.put("logoImage", getLogo()); map.put("isCommunity", ISCOMMUNITY); return new ModelAndView(map, "login.ftl"); }, new FreeMarkerEngine()); get("/logout/", (req, res) -> { removeAuthenticatedUser(req); res.redirect("/"); return null; }); get("/admin/delete/", "application/json", (request, response) -> { if (getAuthenticatedUser(request) == null || !request.queryParams().contains("repoName")) { response.redirect("/login/"); halt(); return false; } String repoName = request.queryParams("repoName"); RepoResult rr = repo.getRepoByName(repoName); if (rr != null) { Singleton.getUniqueDeleteRepoQueue().add(rr); } return true; }, new JsonTransformer()); get("/admin/checkversion/", "application/json", (request, response) -> { if (getAuthenticatedUser(request) == null) { response.redirect("/login/"); halt(); return false; } String version; try { version = IOUtils.toString(new URL("https://searchcode.com/product/version/")).replace("\"", Values.EMPTYSTRING); } catch (IOException ex) { return "Unable to determine if running the latest version. Check https://searchcode.com/product/download/ for the latest release."; } if (App.VERSION.equals(version)) { return "Your searchcode server version " + version + " is the latest."; } else { return "Your searchcode server version " + App.VERSION + " instance is out of date. The latest version is " + version + "."; } }, new JsonTransformer()); get("/file/:codeid/:reponame/*", (request, response) -> { Map<String, Object> map = new HashMap<>(); CodeSearcher cs = new CodeSearcher(); Cocomo2 coco = new Cocomo2(); StringBuilder code = new StringBuilder(); String fileName = Values.EMPTYSTRING; if (request.splat().length != 0) { fileName = request.splat()[0]; } CodeResult codeResult = cs.getByRepoFileName(request.params(":reponame"), fileName); if (codeResult == null) { int codeid = Integer.parseInt(request.params(":codeid")); codeResult = cs.getById(codeid); } if (codeResult == null) { response.redirect("/404/"); halt(); } List<String> codeLines = codeResult.code; for (int i = 0; i < codeLines.size(); i++) { code.append("<span id=\"" + (i + 1) + "\"></span>"); code.append(StringEscapeUtils.escapeHtml4(codeLines.get(i))); code.append("\n"); } boolean highlight = true; if (Integer.parseInt(codeResult.codeLines) > 1000) { highlight = false; } RepoResult repoResult = repo.getRepoByName(codeResult.repoName); if (repoResult != null) { map.put("source", repoResult.getSource()); } map.put("fileName", codeResult.fileName); map.put("codePath", codeResult.codePath); map.put("codeLength", codeResult.codeLines); map.put("languageName", codeResult.languageName); map.put("md5Hash", codeResult.md5hash); map.put("repoName", codeResult.repoName); map.put("highlight", highlight); map.put("repoLocation", codeResult.getRepoLocation()); map.put("codeValue", code.toString()); map.put("highligher", getSyntaxHighlighter()); map.put("codeOwner", codeResult.getCodeOwner()); double estimatedEffort = coco.estimateEffort(scl.countFilteredLines(codeResult.getCode())); int estimatedCost = (int) coco.estimateCost(estimatedEffort, getAverageSalary()); if (estimatedCost != 0 && !scl.languageCostIgnore(codeResult.getLanguageName())) { map.put("estimatedCost", estimatedCost); } map.put("logoImage", getLogo()); map.put("isCommunity", ISCOMMUNITY); return new ModelAndView(map, "coderesult.ftl"); }, new FreeMarkerEngine()); /** * Deprecated should not be used * TODO delete this method */ get("/codesearch/view/:codeid", (request, response) -> { Map<String, Object> map = new HashMap<>(); int codeid = Integer.parseInt(request.params(":codeid")); CodeSearcher cs = new CodeSearcher(); Cocomo2 coco = new Cocomo2(); StringBuilder code = new StringBuilder(); // escape all the lines and include deeplink for line number CodeResult codeResult = cs.getById(codeid); if (codeResult == null) { response.redirect("/404/"); halt(); } List<String> codeLines = codeResult.code; for (int i = 0; i < codeLines.size(); i++) { code.append("<span id=\"" + (i + 1) + "\"></span>"); code.append(StringEscapeUtils.escapeHtml4(codeLines.get(i))); code.append("\n"); } boolean highlight = true; if (Integer.parseInt(codeResult.codeLines) > 1000) { highlight = false; } RepoResult repoResult = repo.getRepoByName(codeResult.repoName); if (repoResult != null) { map.put("source", repoResult.getSource()); } map.put("fileName", codeResult.fileName); map.put("codePath", codeResult.codePath); map.put("codeLength", codeResult.codeLines); map.put("languageName", codeResult.languageName); map.put("md5Hash", codeResult.md5hash); map.put("repoName", codeResult.repoName); map.put("highlight", highlight); map.put("repoLocation", codeResult.getRepoLocation()); map.put("codeValue", code.toString()); map.put("highligher", getSyntaxHighlighter()); map.put("codeOwner", codeResult.getCodeOwner()); double estimatedEffort = coco.estimateEffort(scl.countFilteredLines(codeResult.getCode())); int estimatedCost = (int) coco.estimateCost(estimatedEffort, getAverageSalary()); if (estimatedCost != 0 && !scl.languageCostIgnore(codeResult.getLanguageName())) { map.put("estimatedCost", estimatedCost); } map.put("logoImage", getLogo()); map.put("isCommunity", ISCOMMUNITY); return new ModelAndView(map, "coderesult.ftl"); }, new FreeMarkerEngine()); get("/documentation/", (request, response) -> { Map<String, Object> map = new HashMap<>(); map.put("logoImage", getLogo()); map.put("isCommunity", ISCOMMUNITY); return new ModelAndView(map, "documentation.ftl"); }, new FreeMarkerEngine()); get("/search_test/", (request, response) -> { Map<String, Object> map = new HashMap<>(); map.put("logoImage", getLogo()); map.put("isCommunity", ISCOMMUNITY); return new ModelAndView(map, "search_test.ftl"); }, new FreeMarkerEngine()); get("/404/", (request, response) -> { Map<String, Object> map = new HashMap<>(); map.put("logoImage", getLogo()); map.put("isCommunity", ISCOMMUNITY); return new ModelAndView(map, "404.ftl"); }, new FreeMarkerEngine()); /** * Test that was being used to display blame information */ // get("/test/:reponame/*", (request, response) -> { // User user = injector.getInstance(User.class); // user.Blame(request.params(":reponame"), request.splat()[0]); // return ""; // }, new JsonTransformer()); }
From source file:DIA_Umpire_Quant.DIA_Umpire_Quant.java
/** * @param args the command line arguments *///w ww . j a v a2 s . c o m public static void main(String[] args) throws FileNotFoundException, IOException, Exception { System.out.println( "================================================================================================="); System.out.println("DIA-Umpire quantitation with targeted re-extraction analysis (version: " + UmpireInfo.GetInstance().Version + ")"); if (args.length != 1) { System.out.println( "command format error, it should be like: java -jar -Xmx10G DIA_Umpire_Quant.jar diaumpire_quant.params"); return; } try { ConsoleLogger.SetConsoleLogger(Level.INFO); ConsoleLogger.SetFileLogger(Level.DEBUG, FilenameUtils.getFullPath(args[0]) + "diaumpire_quant.log"); } catch (Exception e) { } try { Logger.getRootLogger().info("Version: " + UmpireInfo.GetInstance().Version); Logger.getRootLogger().info("Parameter file:" + args[0]); BufferedReader reader = new BufferedReader(new FileReader(args[0])); String line = ""; String WorkFolder = ""; int NoCPUs = 2; String UserMod = ""; String Combined_Prot = ""; String InternalLibID = ""; String ExternalLibPath = ""; String ExternalLibDecoyTag = "DECOY"; boolean DefaultProtFiltering = true; boolean DataSetLevelPepFDR = false; float ProbThreshold = 0.99f; float ExtProbThreshold = 0.99f; float Freq = 0f; int TopNPep = 6; int TopNFrag = 6; float MinFragMz = 200f; String FilterWeight = "GW"; float MinWeight = 0.9f; float RTWindow_Int = -1f; float RTWindow_Ext = -1f; TandemParam tandemPara = new TandemParam(DBSearchParam.SearchInstrumentType.TOF5600); HashMap<String, File> AssignFiles = new HashMap<>(); boolean InternalLibSearch = false; boolean ExternalLibSearch = false; boolean ExportSaint = false; boolean SAINT_MS1 = false; boolean SAINT_MS2 = true; HashMap<String, String[]> BaitList = new HashMap<>(); HashMap<String, String> BaitName = new HashMap<>(); HashMap<String, String[]> ControlList = new HashMap<>(); HashMap<String, String> ControlName = new HashMap<>(); //<editor-fold defaultstate="collapsed" desc="Reading parameter file"> while ((line = reader.readLine()) != null) { line = line.trim(); Logger.getRootLogger().info(line); if (!"".equals(line) && !line.startsWith("#")) { //System.out.println(line); if (line.equals("==File list begin")) { do { line = reader.readLine(); line = line.trim(); if (line.equals("==File list end")) { continue; } else if (!"".equals(line)) { File newfile = new File(line); if (newfile.exists()) { AssignFiles.put(newfile.getAbsolutePath(), newfile); } else { Logger.getRootLogger().info("File: " + newfile + " does not exist."); } } } while (!line.equals("==File list end")); } if (line.split("=").length < 2) { continue; } String type = line.split("=")[0].trim(); String value = line.split("=")[1].trim(); switch (type) { case "TargetedExtraction": { InternalLibSearch = Boolean.parseBoolean(value); break; } case "InternalLibSearch": { InternalLibSearch = Boolean.parseBoolean(value); break; } case "ExternalLibSearch": { ExternalLibSearch = Boolean.parseBoolean(value); break; } case "Path": { WorkFolder = value; break; } case "path": { WorkFolder = value; break; } case "Thread": { NoCPUs = Integer.parseInt(value); break; } case "Fasta": { tandemPara.FastaPath = value; break; } case "Combined_Prot": { Combined_Prot = value; break; } case "DefaultProtFiltering": { DefaultProtFiltering = Boolean.parseBoolean(value); break; } case "DecoyPrefix": { if (!"".equals(value)) { tandemPara.DecoyPrefix = value; } break; } case "UserMod": { UserMod = value; break; } case "ProteinFDR": { tandemPara.ProtFDR = Float.parseFloat(value); break; } case "PeptideFDR": { tandemPara.PepFDR = Float.parseFloat(value); break; } case "DataSetLevelPepFDR": { DataSetLevelPepFDR = Boolean.parseBoolean(value); break; } case "InternalLibID": { InternalLibID = value; break; } case "ExternalLibPath": { ExternalLibPath = value; break; } case "ExtProbThreshold": { ExtProbThreshold = Float.parseFloat(value); break; } case "RTWindow_Int": { RTWindow_Int = Float.parseFloat(value); break; } case "RTWindow_Ext": { RTWindow_Ext = Float.parseFloat(value); break; } case "ExternalLibDecoyTag": { ExternalLibDecoyTag = value; if (ExternalLibDecoyTag.endsWith("_")) { ExternalLibDecoyTag = ExternalLibDecoyTag.substring(0, ExternalLibDecoyTag.length() - 1); } break; } case "ProbThreshold": { ProbThreshold = Float.parseFloat(value); break; } case "ReSearchProb": { //ReSearchProb = Float.parseFloat(value); break; } case "FilterWeight": { FilterWeight = value; break; } case "MinWeight": { MinWeight = Float.parseFloat(value); break; } case "TopNFrag": { TopNFrag = Integer.parseInt(value); break; } case "TopNPep": { TopNPep = Integer.parseInt(value); break; } case "Freq": { Freq = Float.parseFloat(value); break; } case "MinFragMz": { MinFragMz = Float.parseFloat(value); break; } //<editor-fold defaultstate="collapsed" desc="SaintOutput"> case "ExportSaintInput": { ExportSaint = Boolean.parseBoolean(value); break; } case "QuantitationType": { switch (value) { case "MS1": { SAINT_MS1 = true; SAINT_MS2 = false; break; } case "MS2": { SAINT_MS1 = false; SAINT_MS2 = true; break; } case "BOTH": { SAINT_MS1 = true; SAINT_MS2 = true; break; } } break; } // case "BaitInputFile": { // SaintBaitFile = value; // break; // } // case "PreyInputFile": { // SaintPreyFile = value; // break; // } // case "InterationInputFile": { // SaintInteractionFile = value; // break; // } default: { if (type.startsWith("BaitName_")) { BaitName.put(type.substring(9), value); } if (type.startsWith("BaitFile_")) { BaitList.put(type.substring(9), value.split("\t")); } if (type.startsWith("ControlName_")) { ControlName.put(type.substring(12), value); } if (type.startsWith("ControlFile_")) { ControlList.put(type.substring(12), value.split("\t")); } break; } //</editor-fold> } } } //</editor-fold> //Initialize PTM manager using compomics library PTMManager.GetInstance(); if (!UserMod.equals("")) { PTMManager.GetInstance().ImportUserMod(UserMod); } //Check if the fasta file can be found if (!new File(tandemPara.FastaPath).exists()) { Logger.getRootLogger().info("Fasta file :" + tandemPara.FastaPath + " cannot be found, the process will be terminated, please check."); System.exit(1); } //Check if the prot.xml file can be found if (!new File(Combined_Prot).exists()) { Logger.getRootLogger().info("ProtXML file: " + Combined_Prot + " cannot be found, the export protein summary table will be empty."); } LCMSID protID = null; //Parse prot.xml and generate protein master list given an FDR if (Combined_Prot != null && !Combined_Prot.equals("")) { protID = LCMSID.ReadLCMSIDSerialization(Combined_Prot); if (!"".equals(Combined_Prot) && protID == null) { protID = new LCMSID(Combined_Prot, tandemPara.DecoyPrefix, tandemPara.FastaPath); ProtXMLParser protxmlparser = new ProtXMLParser(protID, Combined_Prot, 0f); //Use DIA-Umpire default protein FDR calculation if (DefaultProtFiltering) { protID.RemoveLowLocalPWProtein(0.8f); protID.RemoveLowMaxIniProbProtein(0.9f); protID.FilterByProteinDecoyFDRUsingMaxIniProb(tandemPara.DecoyPrefix, tandemPara.ProtFDR); } //Get protein FDR calculation without other filtering else { protID.FilterByProteinDecoyFDRUsingLocalPW(tandemPara.DecoyPrefix, tandemPara.ProtFDR); } protID.LoadSequence(); protID.WriteLCMSIDSerialization(Combined_Prot); } Logger.getRootLogger().info("Protein No.:" + protID.ProteinList.size()); } HashMap<String, HashMap<String, FragmentPeak>> IDSummaryFragments = new HashMap<>(); //Generate DIA file list ArrayList<DIAPack> FileList = new ArrayList<>(); File folder = new File(WorkFolder); if (!folder.exists()) { Logger.getRootLogger().info("The path : " + WorkFolder + " cannot be found."); System.exit(1); } for (final File fileEntry : folder.listFiles()) { if (fileEntry.isFile() && (fileEntry.getAbsolutePath().toLowerCase().endsWith(".mzxml") | fileEntry.getAbsolutePath().toLowerCase().endsWith(".mzml")) && !fileEntry.getAbsolutePath().toLowerCase().endsWith("q1.mzxml") && !fileEntry.getAbsolutePath().toLowerCase().endsWith("q2.mzxml") && !fileEntry.getAbsolutePath().toLowerCase().endsWith("q3.mzxml")) { AssignFiles.put(fileEntry.getAbsolutePath(), fileEntry); } if (fileEntry.isDirectory()) { for (final File fileEntry2 : fileEntry.listFiles()) { if (fileEntry2.isFile() && (fileEntry2.getAbsolutePath().toLowerCase().endsWith(".mzxml") | fileEntry2.getAbsolutePath().toLowerCase().endsWith(".mzml")) && !fileEntry2.getAbsolutePath().toLowerCase().endsWith("q1.mzxml") && !fileEntry2.getAbsolutePath().toLowerCase().endsWith("q2.mzxml") && !fileEntry2.getAbsolutePath().toLowerCase().endsWith("q3.mzxml")) { AssignFiles.put(fileEntry2.getAbsolutePath(), fileEntry2); } } } } Logger.getRootLogger().info("No. of files assigned :" + AssignFiles.size()); for (File fileEntry : AssignFiles.values()) { Logger.getRootLogger().info(fileEntry.getAbsolutePath()); String mzXMLFile = fileEntry.getAbsolutePath(); if (mzXMLFile.toLowerCase().endsWith(".mzxml") | mzXMLFile.toLowerCase().endsWith(".mzml")) { DIAPack DiaFile = new DIAPack(mzXMLFile, NoCPUs); FileList.add(DiaFile); HashMap<String, FragmentPeak> FragMap = new HashMap<>(); IDSummaryFragments.put(FilenameUtils.getBaseName(mzXMLFile), FragMap); Logger.getRootLogger().info( "================================================================================================="); Logger.getRootLogger().info("Processing " + mzXMLFile); if (!DiaFile.LoadDIASetting()) { Logger.getRootLogger().info("Loading DIA setting failed, job is incomplete"); System.exit(1); } if (!DiaFile.LoadParams()) { Logger.getRootLogger().info("Loading parameters failed, job is incomplete"); System.exit(1); } } } LCMSID combinePepID = null; if (DataSetLevelPepFDR) { combinePepID = LCMSID.ReadLCMSIDSerialization(WorkFolder + "combinePepID.SerFS"); if (combinePepID == null) { FDR_DataSetLevel fdr = new FDR_DataSetLevel(); fdr.GeneratePepIonList(FileList, tandemPara, WorkFolder + "combinePepID.SerFS"); combinePepID = fdr.combineID; combinePepID.WriteLCMSIDSerialization(WorkFolder + "combinePepID.SerFS"); } } //process each DIA file for quantification based on untargeted identifications for (DIAPack DiaFile : FileList) { long time = System.currentTimeMillis(); Logger.getRootLogger().info("Loading identification results " + DiaFile.Filename + "...."); //If the LCMSID serialization is found if (!DiaFile.ReadSerializedLCMSID()) { DiaFile.ParsePepXML(tandemPara, combinePepID); DiaFile.BuildStructure(); if (!DiaFile.MS1FeatureMap.ReadPeakCluster()) { Logger.getRootLogger().info("Loading peak and structure failed, job is incomplete"); System.exit(1); } DiaFile.MS1FeatureMap.ClearMonoisotopicPeakOfCluster(); //Generate mapping between index of precursor feature and pseudo MS/MS scan index DiaFile.GenerateClusterScanNomapping(); //Doing quantification DiaFile.AssignQuant(); DiaFile.ClearStructure(); } DiaFile.IDsummary.ReduceMemoryUsage(); time = System.currentTimeMillis() - time; Logger.getRootLogger().info(DiaFile.Filename + " processed time:" + String.format("%d hour, %d min, %d sec", TimeUnit.MILLISECONDS.toHours(time), TimeUnit.MILLISECONDS.toMinutes(time) - TimeUnit.HOURS.toMinutes(TimeUnit.MILLISECONDS.toHours(time)), TimeUnit.MILLISECONDS.toSeconds(time) - TimeUnit.MINUTES.toSeconds(TimeUnit.MILLISECONDS.toMinutes(time)))); } //<editor-fold defaultstate="collapsed" desc="Targete re-extraction using internal library"> Logger.getRootLogger().info( "================================================================================================="); if (InternalLibSearch && FileList.size() > 1) { Logger.getRootLogger().info("Module C: Targeted extraction using internal library"); FragmentLibManager libManager = FragmentLibManager.ReadFragmentLibSerialization(WorkFolder, InternalLibID); if (libManager == null) { Logger.getRootLogger().info("Building internal spectral library"); libManager = new FragmentLibManager(InternalLibID); ArrayList<LCMSID> LCMSIDList = new ArrayList<>(); for (DIAPack dia : FileList) { LCMSIDList.add(dia.IDsummary); } libManager.ImportFragLibTopFrag(LCMSIDList, Freq, TopNFrag); libManager.WriteFragmentLibSerialization(WorkFolder); } libManager.ReduceMemoryUsage(); Logger.getRootLogger() .info("Building retention time prediction model and generate candidate peptide list"); for (int i = 0; i < FileList.size(); i++) { FileList.get(i).IDsummary.ClearMappedPep(); } for (int i = 0; i < FileList.size(); i++) { for (int j = i + 1; j < FileList.size(); j++) { RTAlignedPepIonMapping alignment = new RTAlignedPepIonMapping(WorkFolder, FileList.get(i).GetParameter(), FileList.get(i).IDsummary, FileList.get(j).IDsummary); alignment.GenerateModel(); alignment.GenerateMappedPepIon(); } FileList.get(i).ExportID(); FileList.get(i).IDsummary = null; } Logger.getRootLogger().info("Targeted matching........"); for (DIAPack diafile : FileList) { if (diafile.IDsummary == null) { diafile.ReadSerializedLCMSID(); } if (!diafile.IDsummary.GetMappedPepIonList().isEmpty()) { diafile.UseMappedIon = true; diafile.FilterMappedIonByProb = false; diafile.BuildStructure(); diafile.MS1FeatureMap.ReadPeakCluster(); diafile.MS1FeatureMap.ClearMonoisotopicPeakOfCluster(); diafile.GenerateMassCalibrationRTMap(); diafile.TargetedExtractionQuant(false, libManager, 1.1f, RTWindow_Int); diafile.MS1FeatureMap.ClearAllPeaks(); diafile.IDsummary.ReduceMemoryUsage(); diafile.IDsummary.RemoveLowProbMappedIon(ProbThreshold); diafile.ExportID(); Logger.getRootLogger().info("Peptide ions: " + diafile.IDsummary.GetPepIonList().size() + " Mapped ions: " + diafile.IDsummary.GetMappedPepIonList().size()); diafile.ClearStructure(); } diafile.IDsummary = null; System.gc(); } Logger.getRootLogger().info( "================================================================================================="); } //</editor-fold> //<editor-fold defaultstate="collapsed" desc="Targeted re-extraction using external library"> //External library search if (ExternalLibSearch) { Logger.getRootLogger().info("Module C: Targeted extraction using external library"); //Read exteranl library FragmentLibManager ExlibManager = FragmentLibManager.ReadFragmentLibSerialization(WorkFolder, FilenameUtils.getBaseName(ExternalLibPath)); if (ExlibManager == null) { ExlibManager = new FragmentLibManager(FilenameUtils.getBaseName(ExternalLibPath)); //Import traML file ExlibManager.ImportFragLibByTraML(ExternalLibPath, ExternalLibDecoyTag); //Check if there are decoy spectra ExlibManager.CheckDecoys(); //ExlibManager.ImportFragLibBySPTXT(ExternalLibPath); ExlibManager.WriteFragmentLibSerialization(WorkFolder); } Logger.getRootLogger() .info("No. of peptide ions in external lib:" + ExlibManager.PeptideFragmentLib.size()); for (DIAPack diafile : FileList) { if (diafile.IDsummary == null) { diafile.ReadSerializedLCMSID(); } //Generate RT mapping RTMappingExtLib RTmap = new RTMappingExtLib(diafile.IDsummary, ExlibManager, diafile.GetParameter()); RTmap.GenerateModel(); RTmap.GenerateMappedPepIon(); diafile.BuildStructure(); diafile.MS1FeatureMap.ReadPeakCluster(); diafile.GenerateMassCalibrationRTMap(); //Perform targeted re-extraction diafile.TargetedExtractionQuant(false, ExlibManager, ProbThreshold, RTWindow_Ext); diafile.MS1FeatureMap.ClearAllPeaks(); diafile.IDsummary.ReduceMemoryUsage(); //Remove target IDs below the defined probability threshold diafile.IDsummary.RemoveLowProbMappedIon(ExtProbThreshold); diafile.ExportID(); diafile.ClearStructure(); Logger.getRootLogger().info("Peptide ions: " + diafile.IDsummary.GetPepIonList().size() + " Mapped ions: " + diafile.IDsummary.GetMappedPepIonList().size()); } } //</editor-fold> //<editor-fold defaultstate="collapsed" desc="Peptide and fragment selection"> Logger.getRootLogger().info("Peptide and fragment selection across the whole dataset"); ArrayList<LCMSID> SummaryList = new ArrayList<>(); for (DIAPack diafile : FileList) { if (diafile.IDsummary == null) { diafile.ReadSerializedLCMSID(); diafile.IDsummary.ClearAssignPeakCluster(); //diafile.IDsummary.ClearPSMs(); } if (protID != null) { //Generate protein list according to mapping of peptide ions for each DIA file to the master protein list diafile.IDsummary.GenerateProteinByRefIDByPepSeq(protID, true); diafile.IDsummary.ReMapProPep(); } if ("GW".equals(FilterWeight)) { diafile.IDsummary.SetFilterByGroupWeight(); } else if ("PepW".equals(FilterWeight)) { diafile.IDsummary.SetFilterByWeight(); } SummaryList.add(diafile.IDsummary); } FragmentSelection fragselection = new FragmentSelection(SummaryList); fragselection.freqPercent = Freq; fragselection.MinFragMZ = MinFragMz; fragselection.GeneratePepFragScoreMap(); fragselection.GenerateTopFragMap(TopNFrag); fragselection.GenerateProtPepScoreMap(MinWeight); fragselection.GenerateTopPepMap(TopNPep); //</editor-fold> //<editor-fold defaultstate="collapsed" desc="Writing general reports"> ExportTable export = new ExportTable(WorkFolder, SummaryList, IDSummaryFragments, protID, fragselection); export.Export(TopNPep, TopNFrag, Freq); //</editor-fold> //<editor-fold defaultstate="collapsed" desc="//<editor-fold defaultstate="collapsed" desc="Generate SAINT input files"> if (ExportSaint && protID != null) { HashMap<String, DIAPack> Filemap = new HashMap<>(); for (DIAPack DIAfile : FileList) { Filemap.put(DIAfile.GetBaseName(), DIAfile); } FileWriter baitfile = new FileWriter(WorkFolder + "SAINT_Bait_" + DateTimeTag.GetTag() + ".txt"); FileWriter preyfile = new FileWriter(WorkFolder + "SAINT_Prey_" + DateTimeTag.GetTag() + ".txt"); FileWriter interactionfileMS1 = null; FileWriter interactionfileMS2 = null; if (SAINT_MS1) { interactionfileMS1 = new FileWriter( WorkFolder + "SAINT_Interaction_MS1_" + DateTimeTag.GetTag() + ".txt"); } if (SAINT_MS2) { interactionfileMS2 = new FileWriter( WorkFolder + "SAINT_Interaction_MS2_" + DateTimeTag.GetTag() + ".txt"); } HashMap<String, String> PreyID = new HashMap<>(); for (String samplekey : ControlName.keySet()) { String name = ControlName.get(samplekey); for (String file : ControlList.get(samplekey)) { baitfile.write(FilenameUtils.getBaseName(file) + "\t" + name + "\t" + "C\n"); LCMSID IDsummary = Filemap.get(FilenameUtils.getBaseName(file)).IDsummary; if (SAINT_MS1) { SaintOutput(protID, IDsummary, fragselection, interactionfileMS1, file, name, PreyID, 1); } if (SAINT_MS2) { SaintOutput(protID, IDsummary, fragselection, interactionfileMS2, file, name, PreyID, 2); } } } for (String samplekey : BaitName.keySet()) { String name = BaitName.get(samplekey); for (String file : BaitList.get(samplekey)) { baitfile.write(FilenameUtils.getBaseName(file) + "\t" + name + "\t" + "T\n"); LCMSID IDsummary = Filemap.get(FilenameUtils.getBaseName(file)).IDsummary; if (SAINT_MS1) { SaintOutput(protID, IDsummary, fragselection, interactionfileMS1, file, name, PreyID, 1); } if (SAINT_MS2) { SaintOutput(protID, IDsummary, fragselection, interactionfileMS2, file, name, PreyID, 2); } } } baitfile.close(); if (SAINT_MS1) { interactionfileMS1.close(); } if (SAINT_MS2) { interactionfileMS2.close(); } for (String AccNo : PreyID.keySet()) { preyfile.write(AccNo + "\t" + PreyID.get(AccNo) + "\n"); } preyfile.close(); } //</editor-fold> Logger.getRootLogger().info("Job done"); Logger.getRootLogger().info( "================================================================================================="); } catch (Exception e) { Logger.getRootLogger().error(ExceptionUtils.getStackTrace(e)); throw e; } }
From source file:de.tudarmstadt.ukp.experiments.dip.wp1.documents.Step7CollectMTurkResults.java
public static void main(String[] args) throws Exception { // input dir - list of xml query containers // /home/user-ukp/research/data/dip/wp1-documents/step4-boiler-plate/ File inputDir = new File(args[0] + "/"); // MTurk result file // output dir File outputDir = new File(args[2]); if (!outputDir.exists()) { outputDir.mkdirs();//w w w . j a v a2 s . c om } // Folder with success files File mturkSuccessDir = new File(args[1]); Collection<File> files = FileUtils.listFiles(mturkSuccessDir, new String[] { "result" }, false); if (files.isEmpty()) { throw new IllegalArgumentException("Input folder is empty. " + mturkSuccessDir); } HashMap<String, List<MTurkAnnotation>> mturkAnnotations = new HashMap<>(); // parsing all CSV files for (File mturkCSVResultFile : files) { System.out.println("Parsing " + mturkCSVResultFile.getName()); MTurkOutputReader outputReader = new MTurkOutputReader( new HashSet<>(Arrays.asList("annotation", "workerid")), mturkCSVResultFile); // for fixing broken data input: for each hit, collect all sentence IDs Map<String, SortedSet<String>> hitSentences = new HashMap<>(); // first iteration: collect the sentences for (Map<String, String> record : outputReader) { String hitID = record.get("hitid"); if (!hitSentences.containsKey(hitID)) { hitSentences.put(hitID, new TreeSet<>()); } String relevantSentences = record.get("Answer.relevant_sentences"); String irrelevantSentences = record.get("Answer.irrelevant_sentences"); if (relevantSentences != null) { hitSentences.get(hitID).addAll(Arrays.asList(relevantSentences.split(","))); } if (irrelevantSentences != null) { hitSentences.get(hitID).addAll(Arrays.asList(irrelevantSentences.split(","))); } } // and now second iteration for (Map<String, String> record : outputReader) { String hitID = record.get("hitid"); String annotatorID = record.get("workerid"); String acceptTime = record.get("assignmentaccepttime"); String submitTime = record.get("assignmentsubmittime"); String relevantSentences = record.get("Answer.relevant_sentences"); String irrelevantSentences = record.get("Answer.irrelevant_sentences"); String reject = record.get("reject"); String filename[]; String comment; String clueWeb; String[] relevant = {}; String[] irrelevant = {}; filename = record.get("annotation").split("_"); String fileXml = filename[0]; clueWeb = filename[1].trim(); comment = record.get("Answer.comment"); if (relevantSentences != null) { relevant = relevantSentences.split(","); } if (irrelevantSentences != null) { irrelevant = irrelevantSentences.split(","); } // sanitizing data: if both relevant and irrelevant are empty, that's a bug // we're gonna look up all sentences from this HIT and treat this assignment // as if there were only irrelevant ones if (relevant.length == 0 && irrelevant.length == 0) { SortedSet<String> strings = hitSentences.get(hitID); irrelevant = new String[strings.size()]; strings.toArray(irrelevant); } if (reject != null) { System.out.println(" HIT " + hitID + " annotated by " + annotatorID + " was rejected "); } else { /* // relevant sentences is a comma-delimited string, // this regular expression is rather strange // it must contain digits, it might be that there is only one space or a comma or some other char // digits are the sentence ids. if relevant sentences do not contain digits then it is wrong if (relevantSentences.matches("^\\D*$") && irrelevantSentences.matches("^\\D*$")) { try { throw new IllegalStateException( "No annotations found for HIT " + hitID + " in " + fileXml + " for document " + clueWeb); } catch (IllegalStateException ex) { ex.printStackTrace(); } } */ MTurkAnnotation mturkAnnotation; try { mturkAnnotation = new MTurkAnnotation(hitID, annotatorID, acceptTime, submitTime, comment, clueWeb, relevant, irrelevant); } catch (IllegalArgumentException ex) { throw new IllegalArgumentException("Record: " + record, ex); } List<MTurkAnnotation> listOfAnnotations = mturkAnnotations.get(fileXml); if (listOfAnnotations == null) { listOfAnnotations = new ArrayList<>(); } listOfAnnotations.add(mturkAnnotation); mturkAnnotations.put(fileXml, listOfAnnotations); } } // parser.close(); } // Debugging: output number of HITs of a query System.out.println("Accepted HITs for a query:"); for (Map.Entry e : mturkAnnotations.entrySet()) { ArrayList<MTurkAnnotation> a = (ArrayList<MTurkAnnotation>) e.getValue(); System.out.println(e.getKey() + " " + a.size()); } for (File f : FileUtils.listFiles(inputDir, new String[] { "xml" }, false)) { QueryResultContainer queryResultContainer = QueryResultContainer .fromXML(FileUtils.readFileToString(f, "utf-8")); String fileName = f.getName(); List<MTurkAnnotation> listOfAnnotations = mturkAnnotations.get(fileName); if (listOfAnnotations == null || listOfAnnotations.isEmpty()) { throw new IllegalStateException("No annotations for " + f.getName()); } for (QueryResultContainer.SingleRankedResult rankedResults : queryResultContainer.rankedResults) { for (MTurkAnnotation mtAnnotation : listOfAnnotations) { String clueWeb = mtAnnotation.clueWeb; if (rankedResults.clueWebID.equals(clueWeb)) { List<QueryResultContainer.MTurkRelevanceVote> mTurkRelevanceVotes = rankedResults.mTurkRelevanceVotes; QueryResultContainer.MTurkRelevanceVote relevanceVote = new QueryResultContainer.MTurkRelevanceVote(); String annotatorID = mtAnnotation.annotatorID; String hitID = mtAnnotation.hitID; String acceptTime = mtAnnotation.acceptTime; String submitTime = mtAnnotation.submitTime; String comment = mtAnnotation.comment; String[] relevant = mtAnnotation.relevant; String[] irrelevant = mtAnnotation.irrelevant; relevanceVote.turkID = annotatorID.trim(); relevanceVote.hitID = hitID.trim(); relevanceVote.acceptTime = acceptTime.trim(); relevanceVote.submitTime = submitTime.trim(); relevanceVote.comment = comment != null ? comment.trim() : null; if (relevant.length == 0 && irrelevant.length == 0) { try { throw new IllegalStateException("the length of the annotations is 0" + rankedResults.clueWebID + " for HIT " + relevanceVote.hitID); } catch (IllegalStateException e) { e.printStackTrace(); } } for (String r : relevant) { String sentenceId = r.trim(); if (!sentenceId.isEmpty() && sentenceId.matches("\\d+")) { QueryResultContainer.SingleSentenceRelevanceVote singleSentenceVote = new QueryResultContainer.SingleSentenceRelevanceVote(); singleSentenceVote.sentenceID = sentenceId; singleSentenceVote.relevant = "true"; relevanceVote.singleSentenceRelevanceVotes.add(singleSentenceVote); } } for (String r : irrelevant) { String sentenceId = r.trim(); if (!sentenceId.isEmpty() && sentenceId.matches("\\d+")) { QueryResultContainer.SingleSentenceRelevanceVote singleSentenceVote = new QueryResultContainer.SingleSentenceRelevanceVote(); singleSentenceVote.sentenceID = sentenceId; singleSentenceVote.relevant = "false"; relevanceVote.singleSentenceRelevanceVotes.add(singleSentenceVote); } } mTurkRelevanceVotes.add(relevanceVote); } } } File outputFile = new File(outputDir, f.getName()); FileUtils.writeStringToFile(outputFile, queryResultContainer.toXML(), "utf-8"); System.out.println("Finished " + outputFile); } }