List of usage examples for java.util ArrayList size
int size
To view the source code for java.util ArrayList size.
Click Source Link
From source file:guardar.en.base.de.datos.MainServidor.java
public static void main(String[] args) throws ParserConfigurationException, SAXException, IOException, ClassNotFoundException { Mongo mongo = new Mongo("localhost", 27017); // nombre de la base de datos DB database = mongo.getDB("paginas"); // coleccion de la db DBCollection collection = database.getCollection("indice"); DBCollection collection_textos = database.getCollection("tabla"); ArrayList<String> lista_textos = new ArrayList(); try {/*from www. j a v a 2 s. c o m*/ ServerSocket servidor = new ServerSocket(4545); // Crear un servidor en pausa hasta que un cliente llegue. while (true) { String aux = new String(); lista_textos.clear(); Socket clienteNuevo = servidor.accept();// Si llega se acepta. // Queda en pausa otra vez hasta que un objeto llegue. ObjectInputStream entrada = new ObjectInputStream(clienteNuevo.getInputStream()); JSONObject request = (JSONObject) entrada.readObject(); String b = (String) request.get("id"); //hacer una query a la base de datos con la palabra que se quiere obtener BasicDBObject query = new BasicDBObject("palabra", b); DBCursor cursor = collection.find(query); ArrayList<DocumentosDB> lista_doc = new ArrayList<>(); // de la query tomo el campo documentos y los agrego a una lista try { while (cursor.hasNext()) { //System.out.println(cursor.next()); BasicDBList campo_documentos = (BasicDBList) cursor.next().get("documentos"); // en el for voy tomando uno por uno los elementos en el campo documentos for (Iterator<Object> it = campo_documentos.iterator(); it.hasNext();) { BasicDBObject dbo = (BasicDBObject) it.next(); //DOC tiene id y frecuencia DocumentosDB doc = new DocumentosDB(); doc.makefn2(dbo); //int id = (int)doc.getId_documento(); //int f = (int)doc.getFrecuencia(); lista_doc.add(doc); //******************************************* //******************************************** //QUERY A LA COLECCION DE TEXTOS /* BasicDBObject query_textos = new BasicDBObject("id", doc.getId_documento());//query DBCursor cursor_textos = collection_textos.find(query_textos); try { while (cursor_textos.hasNext()) { DBObject obj = cursor_textos.next(); String titulo = (String) obj.get("titulo"); titulo = titulo + "\n\n"; String texto = (String) obj.get("texto"); String texto_final = titulo + texto; aux = texto_final; lista_textos.add(texto_final); } } finally { cursor_textos.close(); }*/ //System.out.println(doc.getId_documento()); //System.out.println(doc.getFrecuencia()); } // end for } //end while query } finally { cursor.close(); } // ordeno la lista de menor a mayor Collections.sort(lista_doc, new Comparator<DocumentosDB>() { @Override public int compare(DocumentosDB o1, DocumentosDB o2) { //throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. return o1.getFrecuencia().compareTo(o2.getFrecuencia()); } }); int tam = lista_doc.size() - 1; for (int j = tam; j >= 0; j--) { BasicDBObject query_textos = new BasicDBObject("id", (int) lista_doc.get(j).getId_documento().intValue());//query DBCursor cursor_textos = collection_textos.find(query_textos);// lo busco try { while (cursor_textos.hasNext()) { DBObject obj = cursor_textos.next(); String titulo = "*******************************"; titulo += (String) obj.get("titulo"); int f = (int) lista_doc.get(j).getFrecuencia().intValue(); String strinf = Integer.toString(f); titulo += "******************************* frecuencia:" + strinf; titulo = titulo + "\n\n"; String texto = (String) obj.get("texto"); String texto_final = titulo + texto + "\n\n"; aux = aux + texto_final; //lista_textos.add(texto_final); } } finally { cursor_textos.close(); } } //actualizar el cache try { Socket cliente_cache = new Socket("localhost", 4500); // nos conectamos con el servidor ObjectOutputStream mensaje_cache = new ObjectOutputStream(cliente_cache.getOutputStream()); // get al output del servidor, que es cliente : socket del cliente q se conecto al server JSONObject actualizacion_cache = new JSONObject(); actualizacion_cache.put("actualizacion", 1); actualizacion_cache.put("busqueda", b); actualizacion_cache.put("respuesta", aux); mensaje_cache.writeObject(actualizacion_cache); // envio el msj al servidor } catch (Exception ex) { } //RESPONDER DESDE EL SERVIDORIndex al FRONT ObjectOutputStream resp = new ObjectOutputStream(clienteNuevo.getOutputStream());// obtengo el output del cliente para mandarle un msj resp.writeObject(aux); System.out.println("msj enviado desde el servidor"); } } catch (IOException ex) { Logger.getLogger(MainServidor.class.getName()).log(Level.SEVERE, null, ex); } }
From source file:DIA_Umpire_Quant.DIA_Umpire_IntLibSearch.java
/** * @param args the command line arguments *///from w w w . j a v a 2 s . co m public static void main(String[] args) throws FileNotFoundException, IOException, Exception { System.out.println( "================================================================================================="); System.out.println("DIA-Umpire targeted re-extraction analysis using internal library (version: " + UmpireInfo.GetInstance().Version + ")"); if (args.length != 1) { System.out.println( "command format error, the correct format should be : java -jar -Xmx10G DIA_Umpire_IntLibSearch.jar diaumpire_module.params"); return; } try { ConsoleLogger.SetConsoleLogger(Level.INFO); ConsoleLogger.SetFileLogger(Level.DEBUG, FilenameUtils.getFullPath(args[0]) + "diaumpire_intlibsearch.log"); } catch (Exception e) { } Logger.getRootLogger().info("Version: " + UmpireInfo.GetInstance().Version); Logger.getRootLogger().info("Parameter file:" + args[0]); BufferedReader reader = new BufferedReader(new FileReader(args[0])); String line = ""; String WorkFolder = ""; int NoCPUs = 2; String InternalLibID = ""; float ProbThreshold = 0.99f; float RTWindow_Int = -1f; float Freq = 0f; int TopNFrag = 6; TandemParam tandemPara = new TandemParam(DBSearchParam.SearchInstrumentType.TOF5600); HashMap<String, File> AssignFiles = new HashMap<>(); //<editor-fold defaultstate="collapsed" desc="Reading parameter file"> while ((line = reader.readLine()) != null) { line = line.trim(); Logger.getRootLogger().info(line); if (!"".equals(line) && !line.startsWith("#")) { //System.out.println(line); if (line.equals("==File list begin")) { do { line = reader.readLine(); line = line.trim(); if (line.equals("==File list end")) { continue; } else if (!"".equals(line)) { File newfile = new File(line); if (newfile.exists()) { AssignFiles.put(newfile.getAbsolutePath(), newfile); } else { Logger.getRootLogger().info("File: " + newfile + " does not exist."); } } } while (!line.equals("==File list end")); } if (line.split("=").length < 2) { continue; } String type = line.split("=")[0].trim(); String value = line.split("=")[1].trim(); switch (type) { case "Path": { WorkFolder = value; break; } case "path": { WorkFolder = value; break; } case "Thread": { NoCPUs = Integer.parseInt(value); break; } case "InternalLibID": { InternalLibID = value; break; } case "RTWindow_Int": { RTWindow_Int = Float.parseFloat(value); break; } case "ProbThreshold": { ProbThreshold = Float.parseFloat(value); break; } case "TopNFrag": { TopNFrag = Integer.parseInt(value); break; } case "Freq": { Freq = Float.parseFloat(value); break; } case "Fasta": { tandemPara.FastaPath = value; break; } } } } //</editor-fold> //Initialize PTM manager using compomics library PTMManager.GetInstance(); //Check if the fasta file can be found if (!new File(tandemPara.FastaPath).exists()) { Logger.getRootLogger().info("Fasta file :" + tandemPara.FastaPath + " cannot be found, the process will be terminated, please check."); System.exit(1); } //Generate DIA file list ArrayList<DIAPack> FileList = new ArrayList<>(); try { File folder = new File(WorkFolder); if (!folder.exists()) { Logger.getRootLogger().info("The path : " + WorkFolder + " cannot be found."); System.exit(1); } for (final File fileEntry : folder.listFiles()) { if (fileEntry.isFile() && (fileEntry.getAbsolutePath().toLowerCase().endsWith(".mzxml") | fileEntry.getAbsolutePath().toLowerCase().endsWith(".mzml")) && !fileEntry.getAbsolutePath().toLowerCase().endsWith("q1.mzxml") && !fileEntry.getAbsolutePath().toLowerCase().endsWith("q2.mzxml") && !fileEntry.getAbsolutePath().toLowerCase().endsWith("q3.mzxml")) { AssignFiles.put(fileEntry.getAbsolutePath(), fileEntry); } if (fileEntry.isDirectory()) { for (final File fileEntry2 : fileEntry.listFiles()) { if (fileEntry2.isFile() && (fileEntry2.getAbsolutePath().toLowerCase().endsWith(".mzxml") | fileEntry2.getAbsolutePath().toLowerCase().endsWith(".mzml")) && !fileEntry2.getAbsolutePath().toLowerCase().endsWith("q1.mzxml") && !fileEntry2.getAbsolutePath().toLowerCase().endsWith("q2.mzxml") && !fileEntry2.getAbsolutePath().toLowerCase().endsWith("q3.mzxml")) { AssignFiles.put(fileEntry2.getAbsolutePath(), fileEntry2); } } } } Logger.getRootLogger().info("No. of files assigned :" + AssignFiles.size()); for (File fileEntry : AssignFiles.values()) { Logger.getRootLogger().info(fileEntry.getAbsolutePath()); } for (File fileEntry : AssignFiles.values()) { String mzXMLFile = fileEntry.getAbsolutePath(); if (mzXMLFile.toLowerCase().endsWith(".mzxml") | mzXMLFile.toLowerCase().endsWith(".mzml")) { DIAPack DiaFile = new DIAPack(mzXMLFile, NoCPUs); Logger.getRootLogger().info( "================================================================================================="); Logger.getRootLogger().info("Processing " + mzXMLFile); if (!DiaFile.LoadDIASetting()) { Logger.getRootLogger().info("Loading DIA setting failed, job is incomplete"); System.exit(1); } if (!DiaFile.LoadParams()) { Logger.getRootLogger().info("Loading parameters failed, job is incomplete"); System.exit(1); } Logger.getRootLogger().info("Loading identification results " + mzXMLFile + "...."); //If the serialization file for ID file existed if (DiaFile.ReadSerializedLCMSID()) { DiaFile.IDsummary.ReduceMemoryUsage(); DiaFile.IDsummary.FastaPath = tandemPara.FastaPath; FileList.add(DiaFile); } } } //<editor-fold defaultstate="collapsed" desc="Targete re-extraction using internal library"> Logger.getRootLogger().info( "================================================================================================="); if (FileList.size() > 1) { Logger.getRootLogger().info("Targeted re-extraction using internal library"); FragmentLibManager libManager = FragmentLibManager.ReadFragmentLibSerialization(WorkFolder, InternalLibID); if (libManager == null) { Logger.getRootLogger().info("Building internal spectral library"); libManager = new FragmentLibManager(InternalLibID); ArrayList<LCMSID> LCMSIDList = new ArrayList<>(); for (DIAPack dia : FileList) { LCMSIDList.add(dia.IDsummary); } libManager.ImportFragLibTopFrag(LCMSIDList, Freq, TopNFrag); libManager.WriteFragmentLibSerialization(WorkFolder); } libManager.ReduceMemoryUsage(); Logger.getRootLogger() .info("Building retention time prediction model and generate candidate peptide list"); for (int i = 0; i < FileList.size(); i++) { FileList.get(i).IDsummary.ClearMappedPep(); } for (int i = 0; i < FileList.size(); i++) { for (int j = i + 1; j < FileList.size(); j++) { RTAlignedPepIonMapping alignment = new RTAlignedPepIonMapping(WorkFolder, FileList.get(i).GetParameter(), FileList.get(i).IDsummary, FileList.get(j).IDsummary); alignment.GenerateModel(); alignment.GenerateMappedPepIon(); } FileList.get(i).ExportID(); FileList.get(i).IDsummary = null; } Logger.getRootLogger().info("Targeted matching........"); for (DIAPack diafile : FileList) { if (diafile.IDsummary == null) { diafile.ReadSerializedLCMSID(); } if (!diafile.IDsummary.GetMappedPepIonList().isEmpty()) { diafile.UseMappedIon = true; diafile.FilterMappedIonByProb = false; diafile.BuildStructure(); diafile.MS1FeatureMap.ReadPeakCluster(); diafile.MS1FeatureMap.ClearMonoisotopicPeakOfCluster(); diafile.GenerateMassCalibrationRTMap(); diafile.TargetedExtractionQuant(false, libManager, ProbThreshold, RTWindow_Int); diafile.MS1FeatureMap.ClearAllPeaks(); diafile.IDsummary.ReduceMemoryUsage(); diafile.IDsummary.RemoveLowProbMappedIon(ProbThreshold); diafile.ExportID(); Logger.getRootLogger().info("Peptide ions: " + diafile.IDsummary.GetPepIonList().size() + " Mapped ions: " + diafile.IDsummary.GetMappedPepIonList().size()); diafile.ClearStructure(); } diafile.IDsummary = null; System.gc(); } Logger.getRootLogger().info( "================================================================================================="); } //</editor-fold> Logger.getRootLogger().info("Job done"); Logger.getRootLogger().info( "================================================================================================="); } catch (Exception e) { Logger.getRootLogger().error(ExceptionUtils.getStackTrace(e)); throw e; } }
From source file:net.java.sen.tools.MkSenDic.java
/** * Build sen dictionary./*www . j a v a 2 s .c o m*/ * * @param args * custom dictionary files. see dic/build.xml. */ public static void main(String args[]) { ResourceBundle rb = ResourceBundle.getBundle("dictionary"); DictionaryMaker dm1 = new DictionaryMaker(); DictionaryMaker dm2 = new DictionaryMaker(); DictionaryMaker dm3 = new DictionaryMaker(); // 1st field information of connect file. Vector rule1 = new Vector(); // 2nd field information of connect file. Vector rule2 = new Vector(); // 3rd field information of connect file. Vector rule3 = new Vector(); // 4th field information of connect file. // this field shows cost of morpheme connection // [size3*(x3*size2+x2)+x1] // [size3*(Attr1*size2+Attr2)+Attl] short score[] = new short[20131]; long start = System.currentTimeMillis(); // ///////////////////////////////////////// // // Step1. Loading connetion file. // log.info("(1/7): reading connection matrix ... "); try { log.info("connection file = " + rb.getString("text_connection_file")); log.info("charset = " + rb.getString("dic.charset")); CSVParser csvparser = new CSVParser(new FileInputStream(rb.getString("text_connection_file")), rb.getString("dic.charset")); String t[]; int line = 0; while ((t = csvparser.nextTokens()) != null) { if (t.length < 4) { log.warn("invalid line in " + rb.getString("text_connection_file") + ":" + line); log.warn(rb.getString("text_connection_file") + "may be broken."); break; } dm1.add(t[0]); rule1.add(t[0]); dm2.add(t[1]); rule2.add(t[1]); dm3.add(t[2]); rule3.add(t[2]); if (line == score.length) { score = resize(score); } score[line++] = (short) Integer.parseInt(t[3]); } // ///////////////////////////////////////// // // Step2. Building internal dictionary // log.info("(2/7): building type dictionary ... "); dm1.build(); dm2.build(); dm3.build(); // if you want check specified morpheme, you uncomment and modify // following line: /* * System.out.print("22="); dm3.getById(22); * System.out.print("368="); dm3.getById(368); * * System.out.println(dm3.getDicId("?????*,*,*,*,?")); * DictionaryMaker.debug = true; * System.out.println(dm3.getDicId("?????*,*,*,*,?")); * System.out.println(dm3.getDicIdNoCache("?????*,*,*,*,?")); */ } catch (IOException e) { e.printStackTrace(); System.exit(0); } // ------------------------------------------------- int size1 = dm1.size(); int size2 = dm2.size(); int size3 = dm3.size(); int ruleSize = rule1.size(); short matrix[] = new short[size1 * size2 * size3]; short default_cost = (short) Integer.parseInt(rb.getString("default_connection_cost")); // ///////////////////////////////////////// // // Step3. Writing Connection Matrix // log.info("(3/7): writing conection matrix (" + size1 + " x " + size2 + " x " + size3 + " = " + size1 * size2 * size3 + ") ..."); for (int i = 0; i < (int) (size1 * size2 * size3); i++) matrix[i] = default_cost; for (int i = 0; i < ruleSize; i++) { Vector r1 = dm1.getRuleIdList((String) rule1.get(i)); Vector r2 = dm2.getRuleIdList((String) rule2.get(i)); Vector r3 = dm3.getRuleIdList((String) rule3.get(i)); for (Iterator i1 = r1.iterator(); i1.hasNext();) { int ii1 = ((Integer) i1.next()).intValue(); for (Iterator i2 = r2.iterator(); i2.hasNext();) { int ii2 = ((Integer) i2.next()).intValue(); for (Iterator i3 = r3.iterator(); i3.hasNext();) { int ii3 = ((Integer) i3.next()).intValue(); int pos = size3 * (size2 * ii1 + ii2) + ii3; matrix[pos] = score[i]; } } } } try { DataOutputStream out = new DataOutputStream( new BufferedOutputStream(new FileOutputStream(rb.getString("matrix_file")))); out.writeShort(size1); out.writeShort(size2); out.writeShort(size3); for (int i1 = 0; i1 < size1; i1++) for (int i2 = 0; i2 < size2; i2++) for (int i3 = 0; i3 < size3; i3++) { out.writeShort(matrix[size3 * (size2 * i1 + i2) + i3]); // if (matrix[size3 * (size2 * i1 + i2) + i3] != // default_cost) { // } } out.close(); } catch (IOException e) { e.printStackTrace(); System.exit(0); } matrix = null; score = null; // ------------------------------------------------- int pos_start = Integer.parseInt(rb.getString("pos_start")); int pos_size = Integer.parseInt(rb.getString("pos_size")); int di = 0; int offset = 0; ArrayList dicList = new ArrayList(); // ///////////////////////////////////////// // // Step4. Reading Morpheme Information // log.info("(4/7): reading morpheme information ... "); String t = null; String[] csv = null; try { // writer for feature file. BufferedWriter bw = new BufferedWriter(new OutputStreamWriter( new FileOutputStream(rb.getString("pos_file")), rb.getString("sen.charset"))); log.info("load dic: " + rb.getString("text_dic_file")); BufferedReader dicStream = null; int custom_dic = -1; if (args.length == 0) { dicStream = new BufferedReader(new InputStreamReader( new FileInputStream(rb.getString("text_dic_file")), rb.getString("dic.charset"))); } else { custom_dic = 0; dicStream = new BufferedReader( new InputStreamReader(new FileInputStream(args[custom_dic]), rb.getString("dic.charset"))); } int line = 0; CSVData key_b = new CSVData(); CSVData pos_b = new CSVData(); while (true) { t = dicStream.readLine(); if (t == null) { dicStream.close(); custom_dic++; if (args.length == custom_dic) { break; } else { // read custum dictionary log.info("load dic: " + "args[custum_dic]"); dicStream = new BufferedReader(new InputStreamReader(new FileInputStream(args[custom_dic]), rb.getString("dic.charset"))); } continue; } CSVParser parser = new CSVParser(t); csv = parser.nextTokens(); if (csv.length < (pos_size + pos_start)) { throw new RuntimeException("format error:" + t); } key_b.clear(); pos_b.clear(); for (int i = pos_start; i < (pos_start + pos_size - 1); i++) { key_b.append(csv[i]); pos_b.append(csv[i]); } key_b.append(csv[pos_start + pos_size - 1]); pos_b.append(csv[pos_start + pos_size - 1]); for (int i = pos_start + pos_size; i < (csv.length - 1); i++) { pos_b.append(csv[i]); } pos_b.append(csv[csv.length - 1]); CToken token = new CToken(); token.rcAttr2 = (short) dm1.getDicId(key_b.toString()); token.rcAttr1 = (short) dm2.getDicId(key_b.toString()); token.lcAttr = (short) dm3.getDicId(key_b.toString()); token.posid = 0; token.posID = offset; token.length = (short) csv[0].length(); token.cost = (short) Integer.parseInt(csv[1]); dicList.add(new PairObject(csv[0], token)); byte b[] = pos_b.toString().getBytes(rb.getString("sen.charset")); offset += (b.length + 1); String pos_b_str = pos_b.toString(); bw.write(pos_b_str, 0, pos_b_str.length()); // bw.write(b, 0, b.length); bw.write(0); if (++di % 50000 == 0) log.info("" + di + "... "); } bw.close(); // ----end of writing feature.cha ---- } catch (Exception e) { log.error("Error: " + t); e.printStackTrace(); System.exit(1); } rule1 = null; rule2 = null; rule3 = null; // ///////////////////////////////////////// // // Step5. Sort lexs and write to file // log.info("(5/7): sorting lex... "); int value[] = new int[dicList.size()]; char key[][] = new char[dicList.size()][]; int spos = 0; int dsize = 0; int bsize = 0; String prev = ""; Collections.sort(dicList); // ///////////////////////////////////////// // // Step6. Writing Token Information // log.info("(6/7): writing token... "); try { // writer for token file. DataOutputStream out = new DataOutputStream( new BufferedOutputStream(new FileOutputStream(rb.getString("token_file")))); // writing 'bos' and 'eos' and 'unknown' token. CToken token = new CToken(); token.rcAttr2 = (short) dm1.getDicId(rb.getString("bos_pos")); token.rcAttr1 = (short) dm2.getDicId(rb.getString("bos_pos")); token.lcAttr = (short) dm3.getDicId(rb.getString("bos_pos")); token.write(out); token.rcAttr2 = (short) dm1.getDicId(rb.getString("eos_pos")); token.rcAttr1 = (short) dm2.getDicId(rb.getString("eos_pos")); token.lcAttr = (short) dm3.getDicId(rb.getString("eos_pos")); token.write(out); token.rcAttr2 = (short) dm1.getDicId(rb.getString("unknown_pos")); token.rcAttr1 = (short) dm2.getDicId(rb.getString("unknown_pos")); token.lcAttr = (short) dm3.getDicId(rb.getString("unknown_pos")); token.posID = -1; token.write(out); log.info("key size = " + key.length); for (int i = 0; i < key.length; i++) { String k = (String) ((PairObject) dicList.get(i)).key; if (!prev.equals(k) && i != 0) { key[dsize] = ((String) ((PairObject) dicList.get(spos)).key).toCharArray(); value[dsize] = bsize + (spos << 8); dsize++; bsize = 1; spos = i; } else { bsize++; } prev = (String) ((PairObject) dicList.get(i)).key; ((CToken) (((PairObject) dicList.get(i)).value)).write(out); } out.flush(); out.close(); } catch (Exception e) { e.printStackTrace(); System.exit(1); } key[dsize] = ((String) ((PairObject) dicList.get(spos)).key).toCharArray(); value[dsize] = bsize + (spos << 8); dsize++; dm1 = null; dm2 = null; dm3 = null; dicList = null; // ///////////////////////////////////////// // // Step7. Build Double Array // log.info("(7/7): building Double-Array (size = " + dsize + ") ..."); DoubleArrayTrie da = new DoubleArrayTrie(); da.build(key, null, value, dsize); try { da.save(rb.getString("double_array_file")); } catch (Exception e) { e.printStackTrace(); } log.info("total time = " + (System.currentTimeMillis() - start) / 1000 + "[ms]"); }
From source file:gov.noaa.pfel.coastwatch.pointdata.MakeErdJavaZip.java
/** * This runs MakeErdJava.zip in <contextdirectory> * (usually <tomcat>/webapps/cwexperimental). * * @param args is ignored//from w w w .j a v a 2 s .co m */ public static void main(String args[]) throws Exception { String2.log("\n*** MakeErdJavaZip"); String errorInMethod = String2.ERROR + " while generating ErdJava.zip:\n"; //define directories String baseDir = SSR.getContextDirectory() + //with / separator and / at the end "WEB-INF/"; String classPath = baseDir + "classes/"; String coastWatchDir = classPath + "gov/noaa/pfel/coastwatch/"; //make the JavaDocs String coastWatchClass = "gov.noaa.pfel.coastwatch."; String commandLine0 = "C:\\Progra~1\\Java\\jdk1.8.0_144\\bin\\javadoc" + //" -source 1.4" + //use 1.4 for the DODS classes that use "enum" //2011-02-22 Bob Simons changed enum to en. " -sourcepath " + classPath + //root directory of the classes " -d "; //+ baseDir + "ConvertTableDoc" + //dir to hold results String commandLine2 = //space separated .java files " " + coastWatchDir + "pointdata/ConvertTable.java " + coastWatchDir + "pointdata/DigirHelper.java " + coastWatchDir + "pointdata/DigirIobisTDSP.java " + coastWatchDir + "pointdata/DigirObisTDSP.java " + coastWatchDir + "pointdata/Table.java " + coastWatchDir + "TimePeriods.java " + coastWatchDir + "ValidateDataSetProperties.java " + //recursively searched packages "-classpath " + //';' separated; //external packages are important here //baseDir + "lib/activation.jar;" + baseDir + "lib/commons-compress.jar;" + //baseDir + "lib/commons-discovery.jar;" + //baseDir + "lib/commons-codec-1.3.jar;" + //these 3 now in netcdfAll-latest //baseDir + "lib/commons-httpclient-3.0.1.jar;" + //baseDir + "lib/commons-logging-1.1.jar;" + baseDir + "lib/mail.jar;" + baseDir + "lib/slf4j.jar;" + baseDir + "lib/netcdfAll-latest.jar " + //space after last one "-subpackages " + //the packages to be doc'd // ':' separated //adding a package? add it to dirName below, too //"com.sshtools:org.apache.commons.logging:" + //this external package not very relevant "dods:" + //this external package is relevant "com.cohort.array:" + "com.cohort.util:" + coastWatchClass + "griddata:" + coastWatchClass + "netcheck:" + coastWatchClass + "util"; //generate javadocs once so it will be in zip file String tDir = baseDir + "docs/ErdJavaDoc"; //dir to hold results SSR.dosShell("del /s /q " + //delete (/s=recursive /q=quiet) previous results String2.replaceAll(tDir, "/", "\\"), 60); String checkNames[] = { "/index.html", "/com/cohort/array/DoubleArray.html", "/gov/noaa/pfel/coastwatch/pointdata/ConvertTable.html", "/gov/noaa/pfel/coastwatch/netcheck/NetCheck.html", "/gov/noaa/pfel/coastwatch/griddata/GenerateThreddsXml.html", "/gov/noaa/pfel/coastwatch/griddata/GridSaveAs.html" }; for (int i = 0; i < checkNames.length; i++) Test.ensureTrue(!File2.isFile(tDir + checkNames[i]), errorInMethod + tDir + checkNames[i] + " not deleted."); try { String2.log(String2.toNewlineString(SSR.dosShell(commandLine0 + tDir + commandLine2, 120).toArray())); } catch (Exception e) { String2.log(MustBe.throwable(errorInMethod + "(expected) [for zip]:", e)); } for (int i = 0; i < checkNames.length; i++) Test.ensureTrue(File2.isFile(tDir + checkNames[i]), errorInMethod + tDir + checkNames[i] + " not found."); //generate javadocs again for online use tDir = SSR.getContextDirectory() + //with / separator and / at the end "ErdJavaDoc"; //dir to hold results SSR.dosShell("del /s /q " + //delete (/s=recursive /q=quiet) previous results String2.replaceAll(tDir, "/", "\\"), 60); for (int i = 0; i < checkNames.length; i++) Test.ensureTrue(!File2.isFile(tDir + checkNames[i]), errorInMethod + tDir + checkNames[i] + " not deleted."); try { String2.log(String2.toNewlineString(SSR.dosShell(commandLine0 + tDir + commandLine2, 120).toArray())); } catch (Exception e) { String2.log(MustBe.throwable(errorInMethod + " (expected) [for online]:", e)); } for (int i = 0; i < checkNames.length; i++) Test.ensureTrue(File2.isFile(tDir + checkNames[i]), errorInMethod + tDir + checkNames[i] + " not found."); //make sure relevant files are compiled ConvertTable convertTable = new ConvertTable(); GenerateThreddsXml gtdsh = new GenerateThreddsXml(); GridSaveAs gridSaveAs = new GridSaveAs(); ValidateDataSetProperties validateDataSetProperties = new ValidateDataSetProperties(); DoubleCenterGrids doubleCenterGrids = new DoubleCenterGrids(); try { NetCheck netCheck = new NetCheck(baseDir + "DoesntExist.xml", true); } catch (Exception e) { //don't care if error. deployment may be on another computer //String2.log(MustBe.throwable( // "MakeNetCheckZip.main test constructors. Ignore this exception:\n", e)); } //I think that is what generates the .xml.log file: delete it File2.delete(baseDir + "DoesntExit.xml.log"); //delete the log created by DoubleCenterGrids.test File2.delete("c:/programs/_tomcat/webapps/cwexperimental/WEB-INF/DoubleCenterGrids.log"); //delete the test file from ConvertTable File2.delete("c:/programs/_tomcat/webapps/cwexperimental/WEB-INF/result.nc"); //accumulate the file names to be zipped ArrayList<String> dirNames = new ArrayList(); dirNames.add(baseDir + "ConvertTable.sh"); dirNames.add(baseDir + "ConvertTable.bat"); dirNames.add(baseDir + "DoubleCenterGrids.sh"); dirNames.add(baseDir + "GridSaveAs.sh"); dirNames.add(baseDir + "GridSaveAs.bat"); dirNames.add(baseDir + "GenerateOceanwatchThreddsXml.sh"); dirNames.add(baseDir + "GenerateOtterThreddsXml.sh"); dirNames.add(baseDir + "GenerateThredds1ThreddsXml.sh"); dirNames.add(baseDir + "GenerateThreddsXml.sh"); dirNames.add(baseDir + "incompleteMainCatalog.xml"); dirNames.add(baseDir + "iobis.m"); dirNames.add(baseDir + "NetCheck.sh"); dirNames.add(baseDir + "NetCheck.bat"); dirNames.add(baseDir + "NetCheck.xml"); dirNames.add(baseDir + "obis.m"); dirNames.add(baseDir + "QN2005193_2005193_ux10_westus.grd"); dirNames.add(baseDir + "ValidateDataSetProperties.sh"); dirNames.add(baseDir + "ValidateDataSetProperties.bat"); //dirNames.add(baseDir + "lib/activation.jar"); dirNames.add(baseDir + "lib/commons-compress.jar"); //dirNames.add(baseDir + "lib/commons-discovery.jar"); //dirNames.add(baseDir + "lib/commons-codec-1.3.jar"); //these 3 are now in netcdfAll-latest //dirNames.add(baseDir + "lib/commons-httpclient-3.0.1.jar"); //dirNames.add(baseDir + "lib/commons-logging-1.1.jar"); dirNames.add(baseDir + "lib/mail.jar"); dirNames.add(baseDir + "lib/netcdfAll-latest.jar"); dirNames.add(baseDir + "lib/slf4j.jar"); String2.add(dirNames, RegexFilenameFilter.recursiveFullNameList(baseDir + "docs/ErdJavaDoc/", ".+", false)); //javadocs String2.add(dirNames, RegexFilenameFilter.recursiveFullNameList(baseDir + "classes/dods/", ".+", false)); String2.add(dirNames, RegexFilenameFilter.recursiveFullNameList(baseDir + "classes/com/sshtools/", ".+", false)); String2.add(dirNames, RegexFilenameFilter.recursiveFullNameList(baseDir + "classes/org/jdom/", ".+", false)); String2.add(dirNames, RegexFilenameFilter.fullNameList(baseDir + "classes/com/cohort/array/", ".+")); String2.add(dirNames, RegexFilenameFilter.fullNameList(baseDir + "classes/com/cohort/util/", ".+")); String2.add(dirNames, RegexFilenameFilter.fullNameList(coastWatchDir, "CWBrowser.properties")); String2.add(dirNames, RegexFilenameFilter.fullNameList(coastWatchDir, "BrowserDefault.properties")); String2.add(dirNames, RegexFilenameFilter.fullNameList(coastWatchDir, "DataSet.properties")); String2.add(dirNames, RegexFilenameFilter.fullNameList(coastWatchDir, "LICENSE\\.txt")); String2.add(dirNames, RegexFilenameFilter.fullNameList(coastWatchDir, "OneOf.+")); String2.add(dirNames, RegexFilenameFilter.fullNameList(coastWatchDir, "TimePeriods.+")); String2.add(dirNames, RegexFilenameFilter.fullNameList(coastWatchDir, "ValidateDataSetProperties.+")); String2.add(dirNames, RegexFilenameFilter.fullNameList(coastWatchDir + "griddata/", ".+")); String2.add(dirNames, RegexFilenameFilter.fullNameList(coastWatchDir + "hdf/", ".+")); String2.add(dirNames, RegexFilenameFilter.fullNameList(coastWatchDir + "netcheck/", ".+")); String2.add(dirNames, RegexFilenameFilter.fullNameList(coastWatchDir + "pointdata/", "ConvertTable.+")); String2.add(dirNames, RegexFilenameFilter.fullNameList(coastWatchDir + "pointdata/", "Digir.+")); String2.add(dirNames, RegexFilenameFilter.fullNameList(coastWatchDir + "pointdata/", "Table.+")); String2.add(dirNames, RegexFilenameFilter.fullNameList(coastWatchDir + "pointdata/", "LICENSE\\.txt")); String2.add(dirNames, RegexFilenameFilter.fullNameList(coastWatchDir + "util/", ".+")); //convert to sorted String array String dirNameArray[] = dirNames.toArray(new String[0]); Arrays.sort(dirNameArray); //String2.log(String2.toNewlineString(dirNameArray)); //make the zip file String zipName = SSR.getContextDirectory() + //with / separator and / at the end "ErdJava.zip"; String2.log("MakeErdJavaZip is making " + zipName + "."); File2.delete(zipName); SSR.zip(zipName, dirNameArray, 60, baseDir); String2.log("\nMakeErdJavaZip successfully finished making " + zipName + ".\nnFiles=" + dirNames.size()); }
From source file:ServerStatus.java
License:asdf
/** * @param args the command line arguments *///from w ww .ja v a 2 s . c o m public static void main(String[] args) throws InterruptedException, FileNotFoundException, IOException, ParseException { FileReader reader = null; ArrayList<BankInfo2> BankArray = new ArrayList<BankInfo2>(); reader = new FileReader(args[0]); JSONParser jp = new JSONParser(); JSONObject doc = (JSONObject) jp.parse(reader); JSONObject banks = (JSONObject) doc.get("banks"); //Set bankKeys = banks.keySet(); //Object [] bankNames = bankKeys.toArray(); Object[] bankNames = banks.keySet().toArray(); for (int i = 0; i < bankNames.length; i++) { //System.out.println(bankNames[i]); String bname = (String) bankNames[i]; BankInfo2 binfo = new BankInfo2(bname); JSONObject banki = (JSONObject) banks.get(bname); JSONArray chain = (JSONArray) banki.get("chain"); int chainLength = chain.size(); //System.out.println(chainLength); for (Object chain1 : chain) { JSONObject serv = (JSONObject) chain1; ServerInfo sinfo = new ServerInfo((String) serv.get("ip"), serv.get("port").toString(), serv.get("start_delay").toString(), serv.get("lifetime").toString(), serv.get("receive").toString(), serv.get("send").toString()); binfo.servers.add(sinfo); //System.out.println(serv.get("ip") + ":" + serv.get("port")); } BankArray.add(binfo); } //System.out.println("Done Processing Servers"); JSONArray clients = (JSONArray) doc.get("clients"); ArrayList<ClientInfo> clientsList = new ArrayList<ClientInfo>(); for (int i = 0; i < clients.size(); i++) { JSONObject client_i = (JSONObject) clients.get(i); //This is for hard coded requests in the json file //System.out.println(client_i); //System.out.println(client_i.getClass()); String typeOfClient = client_i.get("requests").getClass().toString(); //This is for a client that has hardCoded requests if (typeOfClient.equals("class org.json.simple.JSONArray")) { //System.out.println("JSONArray"); JSONArray requests = (JSONArray) client_i.get("requests"); ClientInfo c = new ClientInfo(client_i.get("reply_timeout").toString(), client_i.get("request_retries").toString(), client_i.get("resend_head").toString()); c.prob_failure = client_i.get("prob_failure").toString(); c.msg_send_delay = client_i.get("msg_delay").toString(); System.out.println( "Successfully added prob failure and msg_send " + c.prob_failure + "," + c.msg_send_delay); ArrayList<RequestInfo> req_list = new ArrayList<RequestInfo>(); for (int j = 0; j < requests.size(); j++) { JSONObject request_j = (JSONObject) requests.get(j); String req = request_j.get("request").toString(); String bank = request_j.get("" + "bank").toString(); String acc = request_j.get("account").toString(); String seq = request_j.get("seq_num").toString(); String amt = null; try { amt = request_j.get("amount").toString(); } catch (NullPointerException e) { //System.out.println("Amount not specified."); } RequestInfo r; if (amt == null) { r = new RequestInfo(req, bank, acc, seq); } else { r = new RequestInfo(req, bank, acc, amt, seq); } //RequestInfo r = new RequestInfo(request_j.get("request").toString(), request_j.get("bank").toString(), request_j.get("account").toString(), request_j.get("amount").toString()); req_list.add(r); } c.requests = req_list; c.PortNumber = 60000 + i; clientsList.add(c); //System.out.println(client_i); } //This is for Random client requests else if (typeOfClient.equals("class org.json.simple.JSONObject")) { JSONObject randomReq = (JSONObject) client_i.get("requests"); String seed = randomReq.get("seed").toString(); String num_requests = randomReq.get("num_requests").toString(); String prob_balance = randomReq.get("prob_balance").toString(); String prob_deposit = randomReq.get("prob_deposit").toString(); String prob_withdraw = randomReq.get("prob_withdrawal").toString(); String prob_transfer = randomReq.get("prob_transfer").toString(); //ClientInfo c = new ClientInfo(true, seed, num_requests, prob_balance, prob_deposit, prob_withdraw, prob_transfer); ClientInfo c = new ClientInfo(client_i.get("reply_timeout").toString(), client_i.get("request_retries").toString(), client_i.get("resend_head").toString(), seed, num_requests, prob_balance, prob_deposit, prob_withdraw, prob_transfer); c.PortNumber = 60000 + i; clientsList.add(c); } } //System.out.println(clients.size()); double lowerPercent = 0.0; double upperPercent = 1.0; double result; String bankChainInfoMaster = ""; for (int x = 0; x < BankArray.size(); x++) { BankInfo2 analyze = BankArray.get(x); String chain = analyze.bank_name + "#"; //analyze.servers for (int j = 0; j < analyze.servers.size(); j++) { if (analyze.servers.get(j).Start_delay.equals("0")) { if (j == 0) { chain += analyze.servers.get(j).Port; } else { chain += "#" + analyze.servers.get(j).Port; } } } if (x == 0) { bankChainInfoMaster += chain; } else { bankChainInfoMaster += "@" + chain; } } //System.out.println("CHAIN: "+ bankChainInfoMaster); String clientInfoMaster = ""; for (int x = 0; x < clientsList.size(); x++) { ClientInfo analyze = clientsList.get(x); if (x == 0) { clientInfoMaster += analyze.PortNumber; } else { clientInfoMaster += "#" + analyze.PortNumber; } } //System.out.println("Clients: "+ clientInfoMaster); //RUN MASTER HERE String MasterPort = "49999"; String masterExec = "java Master " + MasterPort + " " + clientInfoMaster + " " + bankChainInfoMaster; Process masterProcess = Runtime.getRuntime().exec(masterExec); System.out.println(masterExec); ArrayList<ServerInfoForClient> servInfoCli = new ArrayList<ServerInfoForClient>(); // List of all servers is saved so that we can wait for them to exit. ArrayList<Process> serverPros = new ArrayList<Process>(); //ArrayList<String> execServs = new ArrayList<String>(); for (int i = 0; i < BankArray.size(); i++) { BankInfo2 analyze = BankArray.get(i); //System.out.println(analyze.bank_name); //One server in the chain String execCmd = "java Server "; String hIP = "", hPort = "", tIP = "", tPort = "", bn = ""; bn = analyze.bank_name; boolean joinFlag = false; if (analyze.servers.size() == 2 && analyze.servers.get(1).Start_delay.equals("0")) { joinFlag = false; } else { joinFlag = true; } if (analyze.servers.size() == 1 && joinFlag == false) { //if(analyze.servers.size() == 1){ ServerInfo si = analyze.servers.get(0); execCmd += "HEAD_TAIL " + si.IP + ":" + si.Port; execCmd += " localhost:0 localhost:0 localhost:" + MasterPort + " " + si.Start_delay + " " + si.Lifetime + " " + si.Receive + " " + si.Send + " " + analyze.bank_name; ; hIP = si.IP; hPort = si.Port; tIP = si.IP; tPort = si.Port; System.out.println(execCmd); Thread.sleep(500); Process pro = Runtime.getRuntime().exec(execCmd); serverPros.add(pro); //} } else if (analyze.servers.size() == 2 && joinFlag == true) { ServerInfo si = analyze.servers.get(0); execCmd += "HEAD_TAIL " + si.IP + ":" + si.Port; execCmd += " localhost:0 localhost:0 localhost:" + MasterPort + " " + si.Start_delay + " " + si.Lifetime + " " + si.Receive + " " + si.Send + " " + analyze.bank_name; ; hIP = si.IP; hPort = si.Port; tIP = si.IP; tPort = si.Port; System.out.println(execCmd); Thread.sleep(500); Process pro = Runtime.getRuntime().exec(execCmd); serverPros.add(pro); execCmd = "java Server "; ServerInfo si2 = analyze.servers.get(1); execCmd += "TAIL " + si2.IP + ":" + si2.Port; execCmd += " localhost:0 localhost:0 localhost:" + MasterPort + " " + si2.Start_delay + " " + si2.Lifetime + " " + si2.Receive + " " + si2.Send + " " + analyze.bank_name; ; hIP = si.IP; hPort = si.Port; tIP = si.IP; tPort = si.Port; System.out.println(execCmd); Thread.sleep(500); Process pro2 = Runtime.getRuntime().exec(execCmd); serverPros.add(pro2); } else { int icount = 0; for (int x = 0; x < analyze.servers.size(); x++) { ServerInfo si = analyze.servers.get(x); if (si.Start_delay.equals("0")) { icount++; } } System.out.println("icount:" + icount); for (int j = 0; j < icount; j++) { //for(int j = 0; j < analyze.servers.size(); j++){ execCmd = "java Server "; ServerInfo si = analyze.servers.get(j); //Head server if (j == 0) { ServerInfo siSucc = analyze.servers.get(j + 1); execCmd += "HEAD " + si.IP + ":" + si.Port + " "; execCmd += "localhost:0 " + siSucc.IP + ":" + siSucc.Port + " localhost:" + MasterPort; execCmd += " " + si.Start_delay + " " + si.Lifetime + " " + si.Receive + " " + si.Send + " " + analyze.bank_name; System.out.println(execCmd); hIP = si.IP; hPort = si.Port; } //Tail Server else if (j == (icount - 1)) {//analyze.servers.size() - 1) ){ ServerInfo siPred = analyze.servers.get(j - 1); execCmd += "TAIL " + si.IP + ":" + si.Port + " "; execCmd += siPred.IP + ":" + siPred.Port + " localhost:0 localhost:" + MasterPort; execCmd += " " + si.Start_delay + " " + si.Lifetime + " " + si.Receive + " " + si.Send + " " + analyze.bank_name; tIP = si.IP; tPort = si.Port; System.out.println(execCmd); } //Middle Server else { ServerInfo siSucc = analyze.servers.get(j + 1); ServerInfo siPred = analyze.servers.get(j - 1); execCmd += "MIDDLE " + si.IP + ":" + si.Port + " "; execCmd += siPred.IP + ":" + siPred.Port + " " + siSucc.IP + ":" + siSucc.Port + " localhost:" + MasterPort; execCmd += " " + si.Start_delay + " " + si.Lifetime + " " + si.Receive + " " + si.Send + " " + analyze.bank_name; System.out.println(execCmd); } Thread.sleep(500); Process pro = Runtime.getRuntime().exec(execCmd); serverPros.add(pro); } for (int j = icount; j < analyze.servers.size(); j++) { execCmd = "java Server "; ServerInfo si = analyze.servers.get(j); ServerInfo siPred = analyze.servers.get(j - 1); execCmd += "TAIL " + si.IP + ":" + si.Port + " "; execCmd += siPred.IP + ":" + siPred.Port + " localhost:0 localhost:" + MasterPort; execCmd += " " + si.Start_delay + " " + si.Lifetime + " " + si.Receive + " " + si.Send + " " + analyze.bank_name; tIP = si.IP; tPort = si.Port; System.out.println(execCmd); Thread.sleep(500); Process pro = Runtime.getRuntime().exec(execCmd); serverPros.add(pro); } } ServerInfoForClient newServInfoForCli = new ServerInfoForClient(hPort, hIP, tPort, tIP, bn); servInfoCli.add(newServInfoForCli); } String banksCliParam = ""; for (int i = 0; i < servInfoCli.size(); i++) { ServerInfoForClient temp = servInfoCli.get(i); String add = "@" + temp.bank_name + "#" + temp.HeadIP + ":" + temp.HeadPort + "#" + temp.TailIP + ":" + temp.TailPort; banksCliParam += add; } banksCliParam = banksCliParam.replaceFirst("@", ""); //System.out.println(banksCliParam); // List of clients is saved so that we can wait for them to exit. ArrayList<Process> clientPros = new ArrayList<Process>(); for (int i = 0; i < clientsList.size(); i++) { ClientInfo analyze = clientsList.get(i); String requestsString = ""; if (analyze.isRandom) { double balance = Double.parseDouble(analyze.prob_balance); //System.out.println(analyze.prob_balance); double deposit = Double.parseDouble(analyze.prob_deposit); double withdraw = Double.parseDouble(analyze.prob_withdraw); int numRequests = Integer.parseInt(analyze.num_requests); for (int j = 0; j < numRequests; j++) { result = Math.random() * (1.0 - 0.0) + 0.0; int randAccount = (int) (Math.random() * (10001 - 0) + 0); double randAmount = Math.random() * (10001.00 - 0.0) + 0; int adjustMoney = (int) randAmount * 100; randAmount = (double) adjustMoney / 100.00; int randBank = (int) (Math.random() * (bankNames.length - 0) + 0); if (result < balance) { //withdrawal#clientIPPORT%bank_name%accountnum%seq#amount requestsString += "@balance#localhost:" + analyze.PortNumber + "%" + bankNames[randBank] + "%" + randAccount + "%" + j; } else if (result < (deposit + balance)) { requestsString += "@deposit#localhost:" + analyze.PortNumber + "%" + bankNames[randBank] + "%" + randAccount + "%" + j + "#" + randAmount; } else { requestsString += "@withdrawal#localhost:" + analyze.PortNumber + "%" + bankNames[randBank] + "%" + randAccount + "%" + j + "#" + randAmount; } } } else { for (int j = 0; j < analyze.requests.size(); j++) { RequestInfo req = analyze.requests.get(j); //System.out.println("Sequence ###" + req.sequenceNum); if (req.request.equals("balance")) { requestsString += "@" + req.request + "#localhost:" + analyze.PortNumber + "%" + req.bankName + "%" + req.accountNum + "%" + req.sequenceNum; } else { requestsString += "@" + req.request + "#localhost:" + analyze.PortNumber + "%" + req.bankName + "%" + req.accountNum + "%" + req.sequenceNum + "#" + req.amount; } } } requestsString = requestsString.replaceFirst("@", ""); String execCommand; int p = 60000 + i; if (analyze.isRandom) { execCommand = "java Client localhost:" + p + " " + banksCliParam + " " + requestsString + " " + analyze.reply_timeout + " " + analyze.request_retries + " " + analyze.resend_head + " " + analyze.prob_failure + " " + analyze.msg_send_delay + " " + analyze.prob_balance + "," + analyze.prob_deposit + "," + analyze.prob_withdraw + "," + analyze.prob_transfer; } else { execCommand = "java Client localhost:" + p + " " + banksCliParam + " " + requestsString + " " + analyze.reply_timeout + " " + analyze.request_retries + " " + analyze.resend_head + " " + analyze.prob_failure + " " + analyze.msg_send_delay; } Thread.sleep(500); System.out.println(execCommand); System.out.println("Client " + (i + 1) + " started"); Process cliPro = Runtime.getRuntime().exec(execCommand); clientPros.add(cliPro); //System.out.println(requestsString); } // Wait for all the clients to terminate for (Process clientPro : clientPros) { try { clientPro.waitFor(); System.out.println("Client process finished."); } catch (InterruptedException e) { System.out.println("Interrupted while waiting for client."); } } // Sleep for two seconds Thread.sleep(2000); // Force termination of the servers for (Process serverPro : serverPros) { serverPro.destroy(); System.out.println("Killed server."); } masterProcess.destroy(); System.out.println("Killed Master"); //System.out.println("asdf"); }
From source file:DIA_Umpire_Quant.DIA_Umpire_Quant.java
/** * @param args the command line arguments *///www. j a v a 2 s.c o m public static void main(String[] args) throws FileNotFoundException, IOException, Exception { System.out.println( "================================================================================================="); System.out.println("DIA-Umpire quantitation with targeted re-extraction analysis (version: " + UmpireInfo.GetInstance().Version + ")"); if (args.length != 1) { System.out.println( "command format error, it should be like: java -jar -Xmx10G DIA_Umpire_Quant.jar diaumpire_quant.params"); return; } try { ConsoleLogger.SetConsoleLogger(Level.INFO); ConsoleLogger.SetFileLogger(Level.DEBUG, FilenameUtils.getFullPath(args[0]) + "diaumpire_quant.log"); } catch (Exception e) { } try { Logger.getRootLogger().info("Version: " + UmpireInfo.GetInstance().Version); Logger.getRootLogger().info("Parameter file:" + args[0]); BufferedReader reader = new BufferedReader(new FileReader(args[0])); String line = ""; String WorkFolder = ""; int NoCPUs = 2; String UserMod = ""; String Combined_Prot = ""; String InternalLibID = ""; String ExternalLibPath = ""; String ExternalLibDecoyTag = "DECOY"; boolean DefaultProtFiltering = true; boolean DataSetLevelPepFDR = false; float ProbThreshold = 0.99f; float ExtProbThreshold = 0.99f; float Freq = 0f; int TopNPep = 6; int TopNFrag = 6; float MinFragMz = 200f; String FilterWeight = "GW"; float MinWeight = 0.9f; float RTWindow_Int = -1f; float RTWindow_Ext = -1f; TandemParam tandemPara = new TandemParam(DBSearchParam.SearchInstrumentType.TOF5600); HashMap<String, File> AssignFiles = new HashMap<>(); boolean InternalLibSearch = false; boolean ExternalLibSearch = false; boolean ExportSaint = false; boolean SAINT_MS1 = false; boolean SAINT_MS2 = true; HashMap<String, String[]> BaitList = new HashMap<>(); HashMap<String, String> BaitName = new HashMap<>(); HashMap<String, String[]> ControlList = new HashMap<>(); HashMap<String, String> ControlName = new HashMap<>(); //<editor-fold defaultstate="collapsed" desc="Reading parameter file"> while ((line = reader.readLine()) != null) { line = line.trim(); Logger.getRootLogger().info(line); if (!"".equals(line) && !line.startsWith("#")) { //System.out.println(line); if (line.equals("==File list begin")) { do { line = reader.readLine(); line = line.trim(); if (line.equals("==File list end")) { continue; } else if (!"".equals(line)) { File newfile = new File(line); if (newfile.exists()) { AssignFiles.put(newfile.getAbsolutePath(), newfile); } else { Logger.getRootLogger().info("File: " + newfile + " does not exist."); } } } while (!line.equals("==File list end")); } if (line.split("=").length < 2) { continue; } String type = line.split("=")[0].trim(); String value = line.split("=")[1].trim(); switch (type) { case "TargetedExtraction": { InternalLibSearch = Boolean.parseBoolean(value); break; } case "InternalLibSearch": { InternalLibSearch = Boolean.parseBoolean(value); break; } case "ExternalLibSearch": { ExternalLibSearch = Boolean.parseBoolean(value); break; } case "Path": { WorkFolder = value; break; } case "path": { WorkFolder = value; break; } case "Thread": { NoCPUs = Integer.parseInt(value); break; } case "Fasta": { tandemPara.FastaPath = value; break; } case "Combined_Prot": { Combined_Prot = value; break; } case "DefaultProtFiltering": { DefaultProtFiltering = Boolean.parseBoolean(value); break; } case "DecoyPrefix": { if (!"".equals(value)) { tandemPara.DecoyPrefix = value; } break; } case "UserMod": { UserMod = value; break; } case "ProteinFDR": { tandemPara.ProtFDR = Float.parseFloat(value); break; } case "PeptideFDR": { tandemPara.PepFDR = Float.parseFloat(value); break; } case "DataSetLevelPepFDR": { DataSetLevelPepFDR = Boolean.parseBoolean(value); break; } case "InternalLibID": { InternalLibID = value; break; } case "ExternalLibPath": { ExternalLibPath = value; break; } case "ExtProbThreshold": { ExtProbThreshold = Float.parseFloat(value); break; } case "RTWindow_Int": { RTWindow_Int = Float.parseFloat(value); break; } case "RTWindow_Ext": { RTWindow_Ext = Float.parseFloat(value); break; } case "ExternalLibDecoyTag": { ExternalLibDecoyTag = value; if (ExternalLibDecoyTag.endsWith("_")) { ExternalLibDecoyTag = ExternalLibDecoyTag.substring(0, ExternalLibDecoyTag.length() - 1); } break; } case "ProbThreshold": { ProbThreshold = Float.parseFloat(value); break; } case "ReSearchProb": { //ReSearchProb = Float.parseFloat(value); break; } case "FilterWeight": { FilterWeight = value; break; } case "MinWeight": { MinWeight = Float.parseFloat(value); break; } case "TopNFrag": { TopNFrag = Integer.parseInt(value); break; } case "TopNPep": { TopNPep = Integer.parseInt(value); break; } case "Freq": { Freq = Float.parseFloat(value); break; } case "MinFragMz": { MinFragMz = Float.parseFloat(value); break; } //<editor-fold defaultstate="collapsed" desc="SaintOutput"> case "ExportSaintInput": { ExportSaint = Boolean.parseBoolean(value); break; } case "QuantitationType": { switch (value) { case "MS1": { SAINT_MS1 = true; SAINT_MS2 = false; break; } case "MS2": { SAINT_MS1 = false; SAINT_MS2 = true; break; } case "BOTH": { SAINT_MS1 = true; SAINT_MS2 = true; break; } } break; } // case "BaitInputFile": { // SaintBaitFile = value; // break; // } // case "PreyInputFile": { // SaintPreyFile = value; // break; // } // case "InterationInputFile": { // SaintInteractionFile = value; // break; // } default: { if (type.startsWith("BaitName_")) { BaitName.put(type.substring(9), value); } if (type.startsWith("BaitFile_")) { BaitList.put(type.substring(9), value.split("\t")); } if (type.startsWith("ControlName_")) { ControlName.put(type.substring(12), value); } if (type.startsWith("ControlFile_")) { ControlList.put(type.substring(12), value.split("\t")); } break; } //</editor-fold> } } } //</editor-fold> //Initialize PTM manager using compomics library PTMManager.GetInstance(); if (!UserMod.equals("")) { PTMManager.GetInstance().ImportUserMod(UserMod); } //Check if the fasta file can be found if (!new File(tandemPara.FastaPath).exists()) { Logger.getRootLogger().info("Fasta file :" + tandemPara.FastaPath + " cannot be found, the process will be terminated, please check."); System.exit(1); } //Check if the prot.xml file can be found if (!new File(Combined_Prot).exists()) { Logger.getRootLogger().info("ProtXML file: " + Combined_Prot + " cannot be found, the export protein summary table will be empty."); } LCMSID protID = null; //Parse prot.xml and generate protein master list given an FDR if (Combined_Prot != null && !Combined_Prot.equals("")) { protID = LCMSID.ReadLCMSIDSerialization(Combined_Prot); if (!"".equals(Combined_Prot) && protID == null) { protID = new LCMSID(Combined_Prot, tandemPara.DecoyPrefix, tandemPara.FastaPath); ProtXMLParser protxmlparser = new ProtXMLParser(protID, Combined_Prot, 0f); //Use DIA-Umpire default protein FDR calculation if (DefaultProtFiltering) { protID.RemoveLowLocalPWProtein(0.8f); protID.RemoveLowMaxIniProbProtein(0.9f); protID.FilterByProteinDecoyFDRUsingMaxIniProb(tandemPara.DecoyPrefix, tandemPara.ProtFDR); } //Get protein FDR calculation without other filtering else { protID.FilterByProteinDecoyFDRUsingLocalPW(tandemPara.DecoyPrefix, tandemPara.ProtFDR); } protID.LoadSequence(); protID.WriteLCMSIDSerialization(Combined_Prot); } Logger.getRootLogger().info("Protein No.:" + protID.ProteinList.size()); } HashMap<String, HashMap<String, FragmentPeak>> IDSummaryFragments = new HashMap<>(); //Generate DIA file list ArrayList<DIAPack> FileList = new ArrayList<>(); File folder = new File(WorkFolder); if (!folder.exists()) { Logger.getRootLogger().info("The path : " + WorkFolder + " cannot be found."); System.exit(1); } for (final File fileEntry : folder.listFiles()) { if (fileEntry.isFile() && (fileEntry.getAbsolutePath().toLowerCase().endsWith(".mzxml") | fileEntry.getAbsolutePath().toLowerCase().endsWith(".mzml")) && !fileEntry.getAbsolutePath().toLowerCase().endsWith("q1.mzxml") && !fileEntry.getAbsolutePath().toLowerCase().endsWith("q2.mzxml") && !fileEntry.getAbsolutePath().toLowerCase().endsWith("q3.mzxml")) { AssignFiles.put(fileEntry.getAbsolutePath(), fileEntry); } if (fileEntry.isDirectory()) { for (final File fileEntry2 : fileEntry.listFiles()) { if (fileEntry2.isFile() && (fileEntry2.getAbsolutePath().toLowerCase().endsWith(".mzxml") | fileEntry2.getAbsolutePath().toLowerCase().endsWith(".mzml")) && !fileEntry2.getAbsolutePath().toLowerCase().endsWith("q1.mzxml") && !fileEntry2.getAbsolutePath().toLowerCase().endsWith("q2.mzxml") && !fileEntry2.getAbsolutePath().toLowerCase().endsWith("q3.mzxml")) { AssignFiles.put(fileEntry2.getAbsolutePath(), fileEntry2); } } } } Logger.getRootLogger().info("No. of files assigned :" + AssignFiles.size()); for (File fileEntry : AssignFiles.values()) { Logger.getRootLogger().info(fileEntry.getAbsolutePath()); String mzXMLFile = fileEntry.getAbsolutePath(); if (mzXMLFile.toLowerCase().endsWith(".mzxml") | mzXMLFile.toLowerCase().endsWith(".mzml")) { DIAPack DiaFile = new DIAPack(mzXMLFile, NoCPUs); FileList.add(DiaFile); HashMap<String, FragmentPeak> FragMap = new HashMap<>(); IDSummaryFragments.put(FilenameUtils.getBaseName(mzXMLFile), FragMap); Logger.getRootLogger().info( "================================================================================================="); Logger.getRootLogger().info("Processing " + mzXMLFile); if (!DiaFile.LoadDIASetting()) { Logger.getRootLogger().info("Loading DIA setting failed, job is incomplete"); System.exit(1); } if (!DiaFile.LoadParams()) { Logger.getRootLogger().info("Loading parameters failed, job is incomplete"); System.exit(1); } } } LCMSID combinePepID = null; if (DataSetLevelPepFDR) { combinePepID = LCMSID.ReadLCMSIDSerialization(WorkFolder + "combinePepID.SerFS"); if (combinePepID == null) { FDR_DataSetLevel fdr = new FDR_DataSetLevel(); fdr.GeneratePepIonList(FileList, tandemPara, WorkFolder + "combinePepID.SerFS"); combinePepID = fdr.combineID; combinePepID.WriteLCMSIDSerialization(WorkFolder + "combinePepID.SerFS"); } } //process each DIA file for quantification based on untargeted identifications for (DIAPack DiaFile : FileList) { long time = System.currentTimeMillis(); Logger.getRootLogger().info("Loading identification results " + DiaFile.Filename + "...."); //If the LCMSID serialization is found if (!DiaFile.ReadSerializedLCMSID()) { DiaFile.ParsePepXML(tandemPara, combinePepID); DiaFile.BuildStructure(); if (!DiaFile.MS1FeatureMap.ReadPeakCluster()) { Logger.getRootLogger().info("Loading peak and structure failed, job is incomplete"); System.exit(1); } DiaFile.MS1FeatureMap.ClearMonoisotopicPeakOfCluster(); //Generate mapping between index of precursor feature and pseudo MS/MS scan index DiaFile.GenerateClusterScanNomapping(); //Doing quantification DiaFile.AssignQuant(); DiaFile.ClearStructure(); } DiaFile.IDsummary.ReduceMemoryUsage(); time = System.currentTimeMillis() - time; Logger.getRootLogger().info(DiaFile.Filename + " processed time:" + String.format("%d hour, %d min, %d sec", TimeUnit.MILLISECONDS.toHours(time), TimeUnit.MILLISECONDS.toMinutes(time) - TimeUnit.HOURS.toMinutes(TimeUnit.MILLISECONDS.toHours(time)), TimeUnit.MILLISECONDS.toSeconds(time) - TimeUnit.MINUTES.toSeconds(TimeUnit.MILLISECONDS.toMinutes(time)))); } //<editor-fold defaultstate="collapsed" desc="Targete re-extraction using internal library"> Logger.getRootLogger().info( "================================================================================================="); if (InternalLibSearch && FileList.size() > 1) { Logger.getRootLogger().info("Module C: Targeted extraction using internal library"); FragmentLibManager libManager = FragmentLibManager.ReadFragmentLibSerialization(WorkFolder, InternalLibID); if (libManager == null) { Logger.getRootLogger().info("Building internal spectral library"); libManager = new FragmentLibManager(InternalLibID); ArrayList<LCMSID> LCMSIDList = new ArrayList<>(); for (DIAPack dia : FileList) { LCMSIDList.add(dia.IDsummary); } libManager.ImportFragLibTopFrag(LCMSIDList, Freq, TopNFrag); libManager.WriteFragmentLibSerialization(WorkFolder); } libManager.ReduceMemoryUsage(); Logger.getRootLogger() .info("Building retention time prediction model and generate candidate peptide list"); for (int i = 0; i < FileList.size(); i++) { FileList.get(i).IDsummary.ClearMappedPep(); } for (int i = 0; i < FileList.size(); i++) { for (int j = i + 1; j < FileList.size(); j++) { RTAlignedPepIonMapping alignment = new RTAlignedPepIonMapping(WorkFolder, FileList.get(i).GetParameter(), FileList.get(i).IDsummary, FileList.get(j).IDsummary); alignment.GenerateModel(); alignment.GenerateMappedPepIon(); } FileList.get(i).ExportID(); FileList.get(i).IDsummary = null; } Logger.getRootLogger().info("Targeted matching........"); for (DIAPack diafile : FileList) { if (diafile.IDsummary == null) { diafile.ReadSerializedLCMSID(); } if (!diafile.IDsummary.GetMappedPepIonList().isEmpty()) { diafile.UseMappedIon = true; diafile.FilterMappedIonByProb = false; diafile.BuildStructure(); diafile.MS1FeatureMap.ReadPeakCluster(); diafile.MS1FeatureMap.ClearMonoisotopicPeakOfCluster(); diafile.GenerateMassCalibrationRTMap(); diafile.TargetedExtractionQuant(false, libManager, 1.1f, RTWindow_Int); diafile.MS1FeatureMap.ClearAllPeaks(); diafile.IDsummary.ReduceMemoryUsage(); diafile.IDsummary.RemoveLowProbMappedIon(ProbThreshold); diafile.ExportID(); Logger.getRootLogger().info("Peptide ions: " + diafile.IDsummary.GetPepIonList().size() + " Mapped ions: " + diafile.IDsummary.GetMappedPepIonList().size()); diafile.ClearStructure(); } diafile.IDsummary = null; System.gc(); } Logger.getRootLogger().info( "================================================================================================="); } //</editor-fold> //<editor-fold defaultstate="collapsed" desc="Targeted re-extraction using external library"> //External library search if (ExternalLibSearch) { Logger.getRootLogger().info("Module C: Targeted extraction using external library"); //Read exteranl library FragmentLibManager ExlibManager = FragmentLibManager.ReadFragmentLibSerialization(WorkFolder, FilenameUtils.getBaseName(ExternalLibPath)); if (ExlibManager == null) { ExlibManager = new FragmentLibManager(FilenameUtils.getBaseName(ExternalLibPath)); //Import traML file ExlibManager.ImportFragLibByTraML(ExternalLibPath, ExternalLibDecoyTag); //Check if there are decoy spectra ExlibManager.CheckDecoys(); //ExlibManager.ImportFragLibBySPTXT(ExternalLibPath); ExlibManager.WriteFragmentLibSerialization(WorkFolder); } Logger.getRootLogger() .info("No. of peptide ions in external lib:" + ExlibManager.PeptideFragmentLib.size()); for (DIAPack diafile : FileList) { if (diafile.IDsummary == null) { diafile.ReadSerializedLCMSID(); } //Generate RT mapping RTMappingExtLib RTmap = new RTMappingExtLib(diafile.IDsummary, ExlibManager, diafile.GetParameter()); RTmap.GenerateModel(); RTmap.GenerateMappedPepIon(); diafile.BuildStructure(); diafile.MS1FeatureMap.ReadPeakCluster(); diafile.GenerateMassCalibrationRTMap(); //Perform targeted re-extraction diafile.TargetedExtractionQuant(false, ExlibManager, ProbThreshold, RTWindow_Ext); diafile.MS1FeatureMap.ClearAllPeaks(); diafile.IDsummary.ReduceMemoryUsage(); //Remove target IDs below the defined probability threshold diafile.IDsummary.RemoveLowProbMappedIon(ExtProbThreshold); diafile.ExportID(); diafile.ClearStructure(); Logger.getRootLogger().info("Peptide ions: " + diafile.IDsummary.GetPepIonList().size() + " Mapped ions: " + diafile.IDsummary.GetMappedPepIonList().size()); } } //</editor-fold> //<editor-fold defaultstate="collapsed" desc="Peptide and fragment selection"> Logger.getRootLogger().info("Peptide and fragment selection across the whole dataset"); ArrayList<LCMSID> SummaryList = new ArrayList<>(); for (DIAPack diafile : FileList) { if (diafile.IDsummary == null) { diafile.ReadSerializedLCMSID(); diafile.IDsummary.ClearAssignPeakCluster(); //diafile.IDsummary.ClearPSMs(); } if (protID != null) { //Generate protein list according to mapping of peptide ions for each DIA file to the master protein list diafile.IDsummary.GenerateProteinByRefIDByPepSeq(protID, true); diafile.IDsummary.ReMapProPep(); } if ("GW".equals(FilterWeight)) { diafile.IDsummary.SetFilterByGroupWeight(); } else if ("PepW".equals(FilterWeight)) { diafile.IDsummary.SetFilterByWeight(); } SummaryList.add(diafile.IDsummary); } FragmentSelection fragselection = new FragmentSelection(SummaryList); fragselection.freqPercent = Freq; fragselection.MinFragMZ = MinFragMz; fragselection.GeneratePepFragScoreMap(); fragselection.GenerateTopFragMap(TopNFrag); fragselection.GenerateProtPepScoreMap(MinWeight); fragselection.GenerateTopPepMap(TopNPep); //</editor-fold> //<editor-fold defaultstate="collapsed" desc="Writing general reports"> ExportTable export = new ExportTable(WorkFolder, SummaryList, IDSummaryFragments, protID, fragselection); export.Export(TopNPep, TopNFrag, Freq); //</editor-fold> //<editor-fold defaultstate="collapsed" desc="//<editor-fold defaultstate="collapsed" desc="Generate SAINT input files"> if (ExportSaint && protID != null) { HashMap<String, DIAPack> Filemap = new HashMap<>(); for (DIAPack DIAfile : FileList) { Filemap.put(DIAfile.GetBaseName(), DIAfile); } FileWriter baitfile = new FileWriter(WorkFolder + "SAINT_Bait_" + DateTimeTag.GetTag() + ".txt"); FileWriter preyfile = new FileWriter(WorkFolder + "SAINT_Prey_" + DateTimeTag.GetTag() + ".txt"); FileWriter interactionfileMS1 = null; FileWriter interactionfileMS2 = null; if (SAINT_MS1) { interactionfileMS1 = new FileWriter( WorkFolder + "SAINT_Interaction_MS1_" + DateTimeTag.GetTag() + ".txt"); } if (SAINT_MS2) { interactionfileMS2 = new FileWriter( WorkFolder + "SAINT_Interaction_MS2_" + DateTimeTag.GetTag() + ".txt"); } HashMap<String, String> PreyID = new HashMap<>(); for (String samplekey : ControlName.keySet()) { String name = ControlName.get(samplekey); for (String file : ControlList.get(samplekey)) { baitfile.write(FilenameUtils.getBaseName(file) + "\t" + name + "\t" + "C\n"); LCMSID IDsummary = Filemap.get(FilenameUtils.getBaseName(file)).IDsummary; if (SAINT_MS1) { SaintOutput(protID, IDsummary, fragselection, interactionfileMS1, file, name, PreyID, 1); } if (SAINT_MS2) { SaintOutput(protID, IDsummary, fragselection, interactionfileMS2, file, name, PreyID, 2); } } } for (String samplekey : BaitName.keySet()) { String name = BaitName.get(samplekey); for (String file : BaitList.get(samplekey)) { baitfile.write(FilenameUtils.getBaseName(file) + "\t" + name + "\t" + "T\n"); LCMSID IDsummary = Filemap.get(FilenameUtils.getBaseName(file)).IDsummary; if (SAINT_MS1) { SaintOutput(protID, IDsummary, fragselection, interactionfileMS1, file, name, PreyID, 1); } if (SAINT_MS2) { SaintOutput(protID, IDsummary, fragselection, interactionfileMS2, file, name, PreyID, 2); } } } baitfile.close(); if (SAINT_MS1) { interactionfileMS1.close(); } if (SAINT_MS2) { interactionfileMS2.close(); } for (String AccNo : PreyID.keySet()) { preyfile.write(AccNo + "\t" + PreyID.get(AccNo) + "\n"); } preyfile.close(); } //</editor-fold> Logger.getRootLogger().info("Job done"); Logger.getRootLogger().info( "================================================================================================="); } catch (Exception e) { Logger.getRootLogger().error(ExceptionUtils.getStackTrace(e)); throw e; } }
From source file:Main.java
public static byte[] asByteList(ArrayList<Byte> list) { int size = list.size(); byte[] byteList = new byte[size]; for (int i = 0; i < size; i++) { byteList[i] = list.get(i);/*from w w w . jav a 2 s . c om*/ } return byteList; }
From source file:Main.java
public static boolean isIn(Object obj, ArrayList list) { for (int i = 0; i < list.size(); i++) { if (obj.equals(list.get(i))) return true; }/* ww w.j a v a 2 s . c o m*/ return false; }
From source file:Main.java
public static void permute(Random random, ArrayList array) { for (int i = 0; i < array.size(); i++) { int j = i + random.nextInt(array.size() - i); Object tmp = array.get(i); array.set(i, array.get(j));/*from w w w.ja v a 2 s . co m*/ array.set(j, tmp); } }
From source file:Main.java
public static boolean isNotBlank(ArrayList<Object> list) { return null != list && list.size() > 0; }