List of usage examples for java.util.concurrent ExecutorService shutdown
void shutdown();
From source file:eu.itesla_project.online.mpi.Master.java
public static void main(String[] args) throws Exception { try {/*from ww w . j a v a2 s. c o m*/ CommandLineParser parser = new GnuParser(); CommandLine line = parser.parse(OPTIONS, args); String mode = line.getOptionValue("m"); Path tmpDir = Paths.get(line.getOptionValue("t")); Class<?> statisticsFactoryClass = Class.forName(line.getOptionValue("f")); Path statisticsDbDir = Paths.get(line.getOptionValue("s")); String statisticsDbName = line.getOptionValue("d"); int coresPerRank = Integer.parseInt(line.getOptionValue("n")); Path stdOutArchive = line.hasOption("o") ? Paths.get(line.getOptionValue("o")) : null; MpiExecutorContext mpiExecutorContext = new MultiStateNetworkAwareMpiExecutorContext(); ScheduledExecutorService scheduledExecutorService = Executors.newScheduledThreadPool(1); ExecutorService executorService = MultiStateNetworkAwareExecutors.newCachedThreadPool(); try { MpiStatisticsFactory statisticsFactory = statisticsFactoryClass .asSubclass(MpiStatisticsFactory.class).newInstance(); MpiStatistics statistics = statisticsFactory.create(statisticsDbDir, statisticsDbName); try (ComputationManager computationManager = new MpiComputationManager(tmpDir, statistics, mpiExecutorContext, coresPerRank, false, stdOutArchive)) { OnlineConfig config = OnlineConfig.load(); try (LocalOnlineApplication application = new LocalOnlineApplication(config, computationManager, scheduledExecutorService, executorService, true)) { switch (mode) { case "ui": System.out.println("LocalOnlineApplication created"); System.out.println("Waiting till shutdown"); // indefinitely wait for JMX commands //TimeUnit.DAYS.sleep(Integer.MAX_VALUE); synchronized (application) { try { application.wait(); } catch (InterruptedException ex) { } } break; default: throw new IllegalArgumentException("Invalid mode " + mode); } } } } finally { mpiExecutorContext.shutdown(); executorService.shutdown(); scheduledExecutorService.shutdown(); executorService.awaitTermination(15, TimeUnit.MINUTES); scheduledExecutorService.awaitTermination(15, TimeUnit.MINUTES); } } catch (ParseException e) { System.err.println(e.getMessage()); HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("master", OPTIONS, true); System.exit(-1); } catch (Throwable t) { LOGGER.error(t.toString(), t); System.exit(-1); } }
From source file:edu.msu.cme.rdp.kmer.KmerFilter.java
public static void main(String[] args) throws Exception { final KmerTrie kmerTrie; final SeqReader queryReader; final SequenceType querySeqType; final File queryFile; final KmerStartsWriter out; final boolean translQuery; final int wordSize; final int translTable; final boolean alignedSeqs; final List<String> refLabels = new ArrayList(); final int maxThreads; try {//from w w w.j a v a2s .c om CommandLine cmdLine = new PosixParser().parse(options, args); args = cmdLine.getArgs(); if (args.length < 3) { throw new Exception("Unexpected number of arguments"); } if (cmdLine.hasOption("out")) { out = new KmerStartsWriter(cmdLine.getOptionValue("out")); } else { out = new KmerStartsWriter(System.out); } if (cmdLine.hasOption("aligned")) { alignedSeqs = true; } else { alignedSeqs = false; } if (cmdLine.hasOption("transl-table")) { translTable = Integer.valueOf(cmdLine.getOptionValue("transl-table")); } else { translTable = 11; } if (cmdLine.hasOption("threads")) { maxThreads = Integer.valueOf(cmdLine.getOptionValue("threads")); } else { maxThreads = Runtime.getRuntime().availableProcessors(); } queryFile = new File(args[1]); wordSize = Integer.valueOf(args[0]); SequenceType refSeqType = null; querySeqType = SeqUtils.guessSequenceType(queryFile); queryReader = new SequenceReader(queryFile); if (querySeqType == SequenceType.Protein) { throw new Exception("Expected nucl query sequences"); } refSeqType = SeqUtils .guessSequenceType(new File(args[2].contains("=") ? args[2].split("=")[1] : args[2])); translQuery = refSeqType == SequenceType.Protein; if (translQuery && wordSize % 3 != 0) { throw new Exception("Word size must be a multiple of 3 for nucl ref seqs"); } int trieWordSize; if (translQuery) { trieWordSize = wordSize / 3; } else { trieWordSize = wordSize; } kmerTrie = new KmerTrie(trieWordSize, translQuery); for (int index = 2; index < args.length; index++) { String refName; String refFileName = args[index]; if (refFileName.contains("=")) { String[] lexemes = refFileName.split("="); refName = lexemes[0]; refFileName = lexemes[1]; } else { String tmpName = new File(refFileName).getName(); if (tmpName.contains(".")) { refName = tmpName.substring(0, tmpName.lastIndexOf(".")); } else { refName = tmpName; } } File refFile = new File(refFileName); if (refSeqType != SeqUtils.guessSequenceType(refFile)) { throw new Exception( "Reference file " + refFile + " contains " + SeqUtils.guessFileFormat(refFile) + " sequences but expected " + refSeqType + " sequences"); } SequenceReader seqReader = new SequenceReader(refFile); Sequence seq; while ((seq = seqReader.readNextSequence()) != null) { if (seq.getSeqName().startsWith("#")) { continue; } if (alignedSeqs) { kmerTrie.addModelSequence(seq, refLabels.size()); } else { kmerTrie.addSequence(seq, refLabels.size()); } } seqReader.close(); refLabels.add(refName); } } catch (Exception e) { new HelpFormatter().printHelp("KmerSearch <word_size> <query_file> [name=]<ref_file> ...", options); System.err.println(e.getMessage()); e.printStackTrace(); System.exit(1); throw new RuntimeException("Stupid jvm"); //While this will never get thrown it is required to make sure javac doesn't get confused about uninitialized variables } long startTime = System.currentTimeMillis(); long seqCount = 0; final int maxTasks = 25000; /* * if (args.length == 4) { maxThreads = Integer.valueOf(args[3]); } else { */ //} System.err.println("Starting kmer mapping at " + new Date()); System.err.println("* Number of threads: " + maxThreads); System.err.println("* References: " + refLabels); System.err.println("* Reads file: " + queryFile); System.err.println("* Kmer length: " + kmerTrie.getWordSize()); final AtomicInteger processed = new AtomicInteger(); final AtomicInteger outstandingTasks = new AtomicInteger(); ExecutorService service = Executors.newFixedThreadPool(maxThreads); Sequence querySeq; while ((querySeq = queryReader.readNextSequence()) != null) { seqCount++; String seqString = querySeq.getSeqString(); if (seqString.length() < 3) { System.err.println("Sequence " + querySeq.getSeqName() + "'s length is less than 3"); continue; } final Sequence threadSeq = querySeq; Runnable r = new Runnable() { public void run() { try { processSeq(threadSeq, refLabels, kmerTrie, out, wordSize, translQuery, translTable, false); processSeq(threadSeq, refLabels, kmerTrie, out, wordSize, translQuery, translTable, true); } catch (IOException e) { throw new RuntimeException(e); } processed.incrementAndGet(); outstandingTasks.decrementAndGet(); } }; outstandingTasks.incrementAndGet(); service.submit(r); while (outstandingTasks.get() >= maxTasks) ; if ((processed.get() + 1) % 1000000 == 0) { System.err.println("Processed " + processed + " sequences in " + (System.currentTimeMillis() - startTime) + " ms"); } } service.shutdown(); service.awaitTermination(1, TimeUnit.DAYS); System.err.println("Finished Processed " + processed + " sequences in " + (System.currentTimeMillis() - startTime) + " ms"); out.close(); }
From source file:accumulo.AccumuloStuff.java
public static void main(String[] args) throws Exception { File tmp = new File(System.getProperty("user.dir") + "/target/mac-test"); if (tmp.exists()) { FileUtils.deleteDirectory(tmp);/* w ww. ja v a 2 s . c om*/ } tmp.mkdirs(); String passwd = "password"; MiniAccumuloConfigImpl cfg = new MiniAccumuloConfigImpl(tmp, passwd); cfg.setNumTservers(1); // cfg.useMiniDFS(true); final MiniAccumuloClusterImpl cluster = cfg.build(); setCoreSite(cluster); cluster.start(); ExecutorService svc = Executors.newFixedThreadPool(2); try { Connector conn = cluster.getConnector("root", passwd); String table = "table"; conn.tableOperations().create(table); final BatchWriter bw = conn.createBatchWriter(table, new BatchWriterConfig()); final AtomicBoolean flushed = new AtomicBoolean(false); Runnable writer = new Runnable() { @Override public void run() { try { Mutation m = new Mutation("row"); m.put("colf", "colq", "value"); bw.addMutation(m); bw.flush(); flushed.set(true); } catch (Exception e) { log.error("Got exception trying to flush mutation", e); } log.info("Exiting batchwriter thread"); } }; Runnable restarter = new Runnable() { @Override public void run() { try { for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) { cluster.killProcess(ServerType.TABLET_SERVER, proc); } cluster.exec(TabletServer.class); } catch (Exception e) { log.error("Caught exception restarting tabletserver", e); } log.info("Exiting restart thread"); } }; svc.execute(writer); svc.execute(restarter); log.info("Waiting for shutdown"); svc.shutdown(); if (!svc.awaitTermination(120, TimeUnit.SECONDS)) { log.info("Timeout on shutdown exceeded"); svc.shutdownNow(); } else { log.info("Cleanly shutdown"); log.info("Threadpool is terminated? " + svc.isTerminated()); } if (flushed.get()) { log.info("****** BatchWriter was flushed *********"); } else { log.info("****** BatchWriter was NOT flushed *********"); } bw.close(); log.info("Got record {}", Iterables.getOnlyElement(conn.createScanner(table, Authorizations.EMPTY))); } finally { cluster.stop(); } }
From source file:ms1quant.MS1Quant.java
/** * @param args the command line arguments MS1Quant parameterfile *///from w w w. j a v a 2 s .co m public static void main(String[] args) throws Exception { BufferedReader reader = null; try { System.out.println( "================================================================================================="); System.out.println("Umpire MS1 quantification and feature detection analysis (version: " + UmpireInfo.GetInstance().Version + ")"); if (args.length < 3 || !args[1].startsWith("-mode")) { System.out .println("command : java -jar -Xmx10G MS1Quant.jar ms1quant.params -mode[1 or 2] [Option]"); System.out.println("\n-mode"); System.out.println("\t1:Single file mode--> mzXML_file PepXML_file"); System.out.println("\t\tEx: -mode1 file1.mzXML file1.pep.xml"); System.out.println( "\t2:Folder mode--> mzXML_Folder PepXML_Folder, all generated csv tables will be merged into a single csv file"); System.out.println("\t\tEx: -mode2 /data/mzxml/ /data/pepxml/"); System.out.println("\nOptions"); System.out.println( "\t-C\tNo of concurrent files to be processed (only for folder mode), Ex. -C5, default:1"); System.out.println("\t-p\tMinimum probability, Ex. -p0.9, default:0.9"); System.out.println("\t-ID\tDetect identified feature only"); System.out.println("\t-O\toutput folder, Ex. -O/data/"); return; } ConsoleLogger consoleLogger = new ConsoleLogger(); consoleLogger.SetConsoleLogger(Level.DEBUG); consoleLogger.SetFileLogger(Level.DEBUG, FilenameUtils.getFullPath(args[0]) + "ms1quant_debug.log"); Logger logger = Logger.getRootLogger(); logger.debug("Command: " + Arrays.toString(args)); logger.info("MS1Quant version: " + UmpireInfo.GetInstance().Version); String parameterfile = args[0]; logger.info("Parameter file: " + parameterfile); File paramfile = new File(parameterfile); if (!paramfile.exists()) { logger.error("Parameter file " + paramfile.getAbsolutePath() + " cannot be found. The program will exit."); } reader = new BufferedReader(new FileReader(paramfile.getAbsolutePath())); String line = ""; InstrumentParameter param = new InstrumentParameter(InstrumentParameter.InstrumentType.TOF5600); int NoCPUs = 2; int NoFile = 1; param.DetermineBGByID = false; param.EstimateBG = true; //<editor-fold defaultstate="collapsed" desc="Read parameter file"> while ((line = reader.readLine()) != null) { if (!"".equals(line) && !line.startsWith("#")) { logger.info(line); //System.out.println(line); if (line.split("=").length < 2) { continue; } if (line.split("=").length < 2) { continue; } String type = line.split("=")[0].trim(); if (type.startsWith("para.")) { type = type.replace("para.", "SE."); } String value = line.split("=")[1].trim(); switch (type) { case "Thread": { NoCPUs = Integer.parseInt(value); break; } //<editor-fold defaultstate="collapsed" desc="instrument parameters"> case "SE.MS1PPM": { param.MS1PPM = Float.parseFloat(value); break; } case "SE.MS2PPM": { param.MS2PPM = Float.parseFloat(value); break; } case "SE.SN": { param.SNThreshold = Float.parseFloat(value); break; } case "SE.MS2SN": { param.MS2SNThreshold = Float.parseFloat(value); break; } case "SE.MinMSIntensity": { param.MinMSIntensity = Float.parseFloat(value); break; } case "SE.MinMSMSIntensity": { param.MinMSMSIntensity = Float.parseFloat(value); break; } case "SE.MinRTRange": { param.MinRTRange = Float.parseFloat(value); break; } case "SE.MaxNoPeakCluster": { param.MaxNoPeakCluster = Integer.parseInt(value); param.MaxMS2NoPeakCluster = Integer.parseInt(value); break; } case "SE.MinNoPeakCluster": { param.MinNoPeakCluster = Integer.parseInt(value); param.MinMS2NoPeakCluster = Integer.parseInt(value); break; } case "SE.MinMS2NoPeakCluster": { param.MinMS2NoPeakCluster = Integer.parseInt(value); break; } case "SE.MaxCurveRTRange": { param.MaxCurveRTRange = Float.parseFloat(value); break; } case "SE.Resolution": { param.Resolution = Integer.parseInt(value); break; } case "SE.RTtol": { param.RTtol = Float.parseFloat(value); break; } case "SE.NoPeakPerMin": { param.NoPeakPerMin = Integer.parseInt(value); break; } case "SE.StartCharge": { param.StartCharge = Integer.parseInt(value); break; } case "SE.EndCharge": { param.EndCharge = Integer.parseInt(value); break; } case "SE.MS2StartCharge": { param.MS2StartCharge = Integer.parseInt(value); break; } case "SE.MS2EndCharge": { param.MS2EndCharge = Integer.parseInt(value); break; } case "SE.NoMissedScan": { param.NoMissedScan = Integer.parseInt(value); break; } case "SE.Denoise": { param.Denoise = Boolean.valueOf(value); break; } case "SE.EstimateBG": { param.EstimateBG = Boolean.valueOf(value); break; } case "SE.RemoveGroupedPeaks": { param.RemoveGroupedPeaks = Boolean.valueOf(value); break; } case "SE.MinFrag": { param.MinFrag = Integer.parseInt(value); break; } case "SE.IsoPattern": { param.IsoPattern = Float.valueOf(value); break; } case "SE.StartRT": { param.startRT = Float.valueOf(value); } case "SE.EndRT": { param.endRT = Float.valueOf(value); } //</editor-fold> } } } //</editor-fold> int mode = 1; if (args[1].equals("-mode2")) { mode = 2; } else if (args[1].equals("-mode1")) { mode = 1; } else { logger.error("-mode number not recongized. The program will exit."); } String mzXML = ""; String pepXML = ""; String mzXMLPath = ""; String pepXMLPath = ""; File mzXMLfile = null; File pepXMLfile = null; File mzXMLfolder = null; File pepXMLfolder = null; int idx = 0; if (mode == 1) { mzXML = args[2]; logger.info("Mode1 mzXML file: " + mzXML); mzXMLfile = new File(mzXML); if (!mzXMLfile.exists()) { logger.error("Mode1 mzXML file " + mzXMLfile.getAbsolutePath() + " cannot be found. The program will exit."); return; } pepXML = args[3]; logger.info("Mode1 pepXML file: " + pepXML); pepXMLfile = new File(pepXML); if (!pepXMLfile.exists()) { logger.error("Mode1 pepXML file " + pepXMLfile.getAbsolutePath() + " cannot be found. The program will exit."); return; } idx = 4; } else if (mode == 2) { mzXMLPath = args[2]; logger.info("Mode2 mzXML folder: " + mzXMLPath); mzXMLfolder = new File(mzXMLPath); if (!mzXMLfolder.exists()) { logger.error("Mode2 mzXML folder " + mzXMLfolder.getAbsolutePath() + " does not exist. The program will exit."); return; } pepXMLPath = args[3]; logger.info("Mode2 pepXML folder: " + pepXMLPath); pepXMLfolder = new File(pepXMLPath); if (!pepXMLfolder.exists()) { logger.error("Mode2 pepXML folder " + pepXMLfolder.getAbsolutePath() + " does not exist. The program will exit."); return; } idx = 4; } String outputfolder = ""; float MinProb = 0f; for (int i = idx; i < args.length; i++) { if (args[i].startsWith("-")) { if (args[i].equals("-ID")) { param.TargetIDOnly = true; logger.info("Detect ID feature only: true"); } if (args[i].startsWith("-O")) { outputfolder = args[i].substring(2); logger.info("Output folder: " + outputfolder); File outputfile = new File(outputfolder); if (!outputfolder.endsWith("\\") | outputfolder.endsWith("/")) { outputfolder += "/"; } if (!outputfile.exists()) { outputfile.mkdir(); } } if (args[i].startsWith("-C")) { try { NoFile = Integer.parseInt(args[i].substring(2)); logger.info("No of concurrent files: " + NoFile); } catch (Exception ex) { logger.error(args[i] + " is not a correct integer format, will process only one file at a time."); } } if (args[i].startsWith("-p")) { try { MinProb = Float.parseFloat(args[i].substring(2)); logger.info("probability threshold: " + MinProb); } catch (Exception ex) { logger.error(args[i] + " is not a correct format, will use 0 as threshold instead."); } } } } reader.close(); TandemParam tandemparam = new TandemParam(DBSearchParam.SearchInstrumentType.TOF5600); PTMManager.GetInstance(); if (param.TargetIDOnly) { param.EstimateBG = false; param.ApexDelta = 1.5f; param.NoMissedScan = 10; param.MiniOverlapP = 0.2f; param.RemoveGroupedPeaks = false; param.CheckMonoIsotopicApex = false; param.DetectByCWT = false; param.FillGapByBK = false; param.IsoCorrThreshold = -1f; param.SmoothFactor = 3; } if (mode == 1) { logger.info("Processing " + mzXMLfile.getAbsolutePath() + "...."); long time = System.currentTimeMillis(); LCMSPeakMS1 LCMS1 = new LCMSPeakMS1(mzXMLfile.getAbsolutePath(), NoCPUs); LCMS1.SetParameter(param); LCMS1.Resume = false; if (!param.TargetIDOnly) { LCMS1.CreatePeakFolder(); } LCMS1.ExportPeakClusterTable = true; if (pepXMLfile.exists()) { tandemparam.InteractPepXMLPath = pepXMLfile.getAbsolutePath(); LCMS1.ParsePepXML(tandemparam, MinProb); logger.info("No. of PSMs included: " + LCMS1.IDsummary.PSMList.size()); logger.info("No. of Peptide ions included: " + LCMS1.IDsummary.GetPepIonList().size()); } if (param.TargetIDOnly) { LCMS1.SaveSerializationFile = false; } if (param.TargetIDOnly || !LCMS1.ReadPeakCluster()) { LCMS1.PeakClusterDetection(); } if (pepXMLfile.exists()) { LCMS1.AssignQuant(false); LCMS1.IDsummary.ExportPepID(outputfolder); } time = System.currentTimeMillis() - time; logger.info(LCMS1.ParentmzXMLName + " processed time:" + String.format("%d hour, %d min, %d sec", TimeUnit.MILLISECONDS.toHours(time), TimeUnit.MILLISECONDS.toMinutes(time) - TimeUnit.HOURS.toMinutes(TimeUnit.MILLISECONDS.toHours(time)), TimeUnit.MILLISECONDS.toSeconds(time) - TimeUnit.MINUTES.toSeconds(TimeUnit.MILLISECONDS.toMinutes(time)))); LCMS1.BaseClearAllPeaks(); LCMS1.SetSpectrumParser(null); LCMS1.IDsummary = null; LCMS1 = null; System.gc(); } else if (mode == 2) { LCMSID IDsummary = new LCMSID("", "", ""); logger.info("Parsing all pepXML files in " + pepXMLPath + "...."); for (File file : pepXMLfolder.listFiles()) { if (file.getName().toLowerCase().endsWith("pep.xml") || file.getName().toLowerCase().endsWith("pepxml")) { PepXMLParser pepXMLParser = new PepXMLParser(IDsummary, file.getAbsolutePath(), MinProb); } } HashMap<String, LCMSID> LCMSIDMap = IDsummary.GetLCMSIDFileMap(); ExecutorService executorPool = null; executorPool = Executors.newFixedThreadPool(NoFile); logger.info("Processing all mzXML files in " + mzXMLPath + "...."); for (File file : mzXMLfolder.listFiles()) { if (file.getName().toLowerCase().endsWith("mzxml")) { LCMSID id = LCMSIDMap.get(FilenameUtils.getBaseName(file.getName())); if (id == null || id.PSMList == null) { logger.warn("No IDs found in :" + FilenameUtils.getBaseName(file.getName()) + ". Quantification for this file is skipped"); continue; } if (!id.PSMList.isEmpty()) { MS1TargetQuantThread thread = new MS1TargetQuantThread(file, id, NoCPUs, outputfolder, param); executorPool.execute(thread); } } } LCMSIDMap.clear(); LCMSIDMap = null; IDsummary = null; executorPool.shutdown(); try { executorPool.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); } catch (InterruptedException e) { logger.info("interrupted.."); } if (outputfolder == null | outputfolder.equals("")) { outputfolder = mzXMLPath; } logger.info("Merging PSM files.."); File output = new File(outputfolder); FileWriter writer = new FileWriter(output.getAbsolutePath() + "/PSM_merge.csv"); boolean header = false; for (File csvfile : output.listFiles()) { if (csvfile.getName().toLowerCase().endsWith("_psms.csv")) { BufferedReader outreader = new BufferedReader(new FileReader(csvfile)); String outline = outreader.readLine(); if (!header) { writer.write(outline + "\n"); header = true; } while ((outline = outreader.readLine()) != null) { writer.write(outline + "\n"); } outreader.close(); csvfile.delete(); } } writer.close(); } logger.info("MS1 quant module is complete."); } catch (Exception e) { Logger.getRootLogger().error(ExceptionUtils.getStackTrace(e)); throw e; } }
From source file:com.siva.javamultithreading.MultiThreadExecutor.java
public static void main(String[] args) throws ExecutionException, IOException { //Populate the data List<DomainObject> list = new ArrayList<>(); DomainObject object = null;/*from ww w .ja v a2s.c o m*/ for (int i = 0; i < 230000; i++) { object = new DomainObject(); object.setId("ID" + i); object.setName("NAME" + i); object.setComment("COMMENT" + i); list.add(object); } int maxNoOfRows = 40000; int noOfThreads = 1; int remaining = 0; if (list.size() > 40000) { noOfThreads = list.size() / maxNoOfRows; remaining = list.size() % maxNoOfRows; if (remaining > 0) { noOfThreads++; } } List<List<DomainObject>> dos = ListUtils.partition(list, maxNoOfRows); ExecutorService threadPool = Executors.newFixedThreadPool(noOfThreads); CompletionService<HSSFWorkbook> pool = new ExecutorCompletionService<>(threadPool); // Excel creation through multiple threads long startTime = System.currentTimeMillis(); for (List<DomainObject> listObj : dos) { pool.submit(new ExcelChunkSheetWriter(listObj)); } HSSFWorkbook hSSFWorkbook = null; HSSFWorkbook book = new HSSFWorkbook(); HSSFSheet sheet = book.createSheet("Report"); try { for (int i = 0; i < 5; i++) { hSSFWorkbook = pool.take().get(); System.out.println( "sheet row count : sheet.PhysicalNumberOfRows() = " + sheet.getPhysicalNumberOfRows()); int currentCount = sheet.getPhysicalNumberOfRows(); int incomingCount = hSSFWorkbook.getSheetAt(0).getPhysicalNumberOfRows(); if ((currentCount + incomingCount) > 60000) { sheet = book.createSheet("Report" + i); } ExcelUtil.copySheets(book, sheet, hSSFWorkbook.getSheetAt(0)); } } catch (InterruptedException ex) { Logger.getLogger(MultiThreadExecutor.class.getName()).log(Level.SEVERE, null, ex); } catch (ExecutionException ex) { Logger.getLogger(MultiThreadExecutor.class.getName()).log(Level.SEVERE, null, ex); } try { writeFile(book, new FileOutputStream("Report.xls")); } catch (Exception e) { e.printStackTrace(); } //System.out.println("No of Threads : " + noOfThreads + " Size : " + list.size() + " remaining : " + remaining); long endTime = System.currentTimeMillis(); System.out.println("Time taken: " + (endTime - startTime) + " ms"); threadPool.shutdown(); //startProcess(); }
From source file:edu.msu.cme.rdp.kmer.cli.FastKmerFilter.java
public static void main(String[] args) throws Exception { final KmerSet<Set<RefKmer>> kmerSet; final SeqReader queryReader; final SequenceType querySeqType; final File queryFile; final KmerStartsWriter out; final boolean translQuery; final int wordSize; final int translTable; final boolean alignedSeqs; final List<String> refLabels = new ArrayList(); final int maxThreads; final int trieWordSize; try {//from ww w.j av a 2 s .com CommandLine cmdLine = new PosixParser().parse(options, args); args = cmdLine.getArgs(); if (args.length < 3) { throw new Exception("Unexpected number of arguments"); } if (cmdLine.hasOption("out")) { out = new KmerStartsWriter(cmdLine.getOptionValue("out")); } else { out = new KmerStartsWriter(System.out); } if (cmdLine.hasOption("aligned")) { alignedSeqs = true; } else { alignedSeqs = false; } if (cmdLine.hasOption("transl-table")) { translTable = Integer.valueOf(cmdLine.getOptionValue("transl-table")); } else { translTable = 11; } if (cmdLine.hasOption("threads")) { maxThreads = Integer.valueOf(cmdLine.getOptionValue("threads")); } else { maxThreads = Runtime.getRuntime().availableProcessors(); } queryFile = new File(args[1]); wordSize = Integer.valueOf(args[0]); SequenceType refSeqType = null; querySeqType = SeqUtils.guessSequenceType(queryFile); queryReader = new SequenceReader(queryFile); if (querySeqType == SequenceType.Protein) { throw new Exception("Expected nucl query sequences"); } refSeqType = SeqUtils .guessSequenceType(new File(args[2].contains("=") ? args[2].split("=")[1] : args[2])); translQuery = refSeqType == SequenceType.Protein; if (translQuery && wordSize % 3 != 0) { throw new Exception("Word size must be a multiple of 3 for nucl ref seqs"); } if (translQuery) { trieWordSize = wordSize / 3; } else { trieWordSize = wordSize; } kmerSet = new KmerSet<Set<RefKmer>>();//new KmerTrie(trieWordSize, translQuery); for (int index = 2; index < args.length; index++) { String refName; String refFileName = args[index]; if (refFileName.contains("=")) { String[] lexemes = refFileName.split("="); refName = lexemes[0]; refFileName = lexemes[1]; } else { String tmpName = new File(refFileName).getName(); if (tmpName.contains(".")) { refName = tmpName.substring(0, tmpName.lastIndexOf(".")); } else { refName = tmpName; } } File refFile = new File(refFileName); if (refSeqType != SeqUtils.guessSequenceType(refFile)) { throw new Exception( "Reference file " + refFile + " contains " + SeqUtils.guessFileFormat(refFile) + " sequences but expected " + refSeqType + " sequences"); } SequenceReader seqReader = new SequenceReader(refFile); Sequence seq; while ((seq = seqReader.readNextSequence()) != null) { if (seq.getSeqName().startsWith("#")) { continue; } KmerGenerator kmers; try { if (translQuery) { //protein ref kmers = new ProtKmerGenerator(seq.getSeqString(), trieWordSize, alignedSeqs); } else { kmers = new NuclKmerGenerator(seq.getSeqString(), trieWordSize, alignedSeqs); } while (kmers.hasNext()) { Kmer temp = kmers.next(); long[] next = temp.getLongKmers(); Set<RefKmer> refKmers = kmerSet.get(next); if (refKmers == null) { refKmers = new HashSet(); kmerSet.add(next, refKmers); } RefKmer kmerRef = new RefKmer(); kmerRef.modelPos = kmers.getPosition(); kmerRef.refFileIndex = refLabels.size(); kmerRef.refSeqid = seq.getSeqName(); refKmers.add(kmerRef); } } catch (IllegalArgumentException ex) { //System.err.println(seq.getSeqName()+ " " + ex.getMessage()); } } seqReader.close(); refLabels.add(refName); } } catch (Exception e) { new HelpFormatter().printHelp( "KmerSearch <kmerSize> <query_file> [name=]<ref_file> ...\nkmerSize should be multiple of 3, (recommend 45, minimum 30, maximum 63) ", options); e.printStackTrace(); System.exit(1); throw new RuntimeException("Stupid jvm"); //While this will never get thrown it is required to make sure javac doesn't get confused about uninitialized variables } long startTime = System.currentTimeMillis(); long seqCount = 0; final int maxTasks = 25000; System.err.println("Starting kmer mapping at " + new Date()); System.err.println("* Number of threads: " + maxThreads); System.err.println("* References: " + refLabels); System.err.println("* Reads file: " + queryFile); System.err.println("* Kmer length: " + trieWordSize); System.err.println("* Kmer Refset Size: " + kmerSet.size()); final AtomicInteger processed = new AtomicInteger(); final AtomicInteger outstandingTasks = new AtomicInteger(); ExecutorService service = Executors.newFixedThreadPool(maxThreads); Sequence querySeq; while ((querySeq = queryReader.readNextSequence()) != null) { seqCount++; String seqString = querySeq.getSeqString(); if ((!translQuery && seqString.length() < wordSize) || (translQuery && seqString.length() < wordSize + 2)) { //System.err.println(querySeq.getSeqName() + "\t" + seqString.length()); continue; } final Sequence threadSeq = querySeq; Runnable r = new Runnable() { public void run() { try { processSeq(threadSeq, refLabels, kmerSet, out, wordSize, translQuery, translTable, false); processSeq(threadSeq, refLabels, kmerSet, out, wordSize, translQuery, translTable, true); processed.incrementAndGet(); outstandingTasks.decrementAndGet(); } catch (Exception e) { e.printStackTrace(); } } }; outstandingTasks.incrementAndGet(); service.submit(r); while (outstandingTasks.get() >= maxTasks) ; if ((processed.get() + 1) % 1000000 == 0) { System.err.println("Processed " + processed + " sequences in " + (System.currentTimeMillis() - startTime) + " ms"); } } service.shutdown(); service.awaitTermination(1, TimeUnit.DAYS); System.err.println("Finished Processed " + processed + " sequences in " + (System.currentTimeMillis() - startTime) + " ms"); out.close(); }
From source file:acmi.l2.clientmod.l2_version_switcher.Main.java
public static void main(String[] args) { if (args.length != 3 && args.length != 4) { System.out.println("USAGE: l2_version_switcher.jar host game version <--splash> <filter>"); System.out.println("EXAMPLE: l2_version_switcher.jar " + L2.NCWEST_HOST + " " + L2.NCWEST_GAME + " 1 \"system\\*\""); System.out.println(// w ww .ja v a 2 s . co m " l2_version_switcher.jar " + L2.PLAYNC_TEST_HOST + " " + L2.PLAYNC_TEST_GAME + " 48"); System.exit(0); } List<String> argsList = new ArrayList<>(Arrays.asList(args)); String host = argsList.get(0); String game = argsList.get(1); int version = Integer.parseInt(argsList.get(2)); Helper helper = new Helper(host, game, version); boolean available = false; try { available = helper.isAvailable(); } catch (IOException e) { System.err.print(e.getClass().getSimpleName()); if (e.getMessage() != null) { System.err.print(": " + e.getMessage()); } System.err.println(); } System.out.println(String.format("Version %d available: %b", version, available)); if (!available) { System.exit(0); } List<FileInfo> fileInfoList = null; try { fileInfoList = helper.getFileInfoList(); } catch (IOException e) { System.err.println("Couldn\'t get file info map"); System.exit(1); } boolean splash = argsList.remove("--splash"); if (splash) { Optional<FileInfo> splashObj = fileInfoList.stream() .filter(fi -> fi.getPath().contains("sp_32b_01.bmp")).findAny(); if (splashObj.isPresent()) { try (InputStream is = new FilterInputStream( Util.getUnzipStream(helper.getDownloadStream(splashObj.get().getPath()))) { @Override public int read() throws IOException { int b = super.read(); if (b >= 0) b ^= 0x36; return b; } @Override public int read(byte[] b, int off, int len) throws IOException { int r = super.read(b, off, len); if (r >= 0) { for (int i = 0; i < r; i++) b[off + i] ^= 0x36; } return r; } }) { new DataInputStream(is).readFully(new byte[28]); BufferedImage bi = ImageIO.read(is); JFrame frame = new JFrame("Lineage 2 [" + version + "] " + splashObj.get().getPath()); frame.setContentPane(new JComponent() { { setPreferredSize(new Dimension(bi.getWidth(), bi.getHeight())); } @Override protected void paintComponent(Graphics g) { g.drawImage(bi, 0, 0, null); } }); frame.setDefaultCloseOperation(WindowConstants.EXIT_ON_CLOSE); frame.pack(); frame.setLocationRelativeTo(null); frame.setVisible(true); } catch (IOException e) { e.printStackTrace(); } } else { System.out.println("Splash not found"); } return; } String filter = argsList.size() > 3 ? separatorsToSystem(argsList.get(3)) : null; File l2Folder = new File(System.getProperty("user.dir")); List<FileInfo> toUpdate = fileInfoList.parallelStream().filter(fi -> { String filePath = separatorsToSystem(fi.getPath()); if (filter != null && !wildcardMatch(filePath, filter, IOCase.INSENSITIVE)) return false; File file = new File(l2Folder, filePath); try { if (file.exists() && file.length() == fi.getSize() && Util.hashEquals(file, fi.getHash())) { System.out.println(filePath + ": OK"); return false; } } catch (IOException e) { System.out.println(filePath + ": couldn't check hash: " + e); return true; } System.out.println(filePath + ": need update"); return true; }).collect(Collectors.toList()); List<String> errors = Collections.synchronizedList(new ArrayList<>()); ExecutorService executor = Executors.newFixedThreadPool(16); CompletableFuture[] tasks = toUpdate.stream().map(fi -> CompletableFuture.runAsync(() -> { String filePath = separatorsToSystem(fi.getPath()); File file = new File(l2Folder, filePath); File folder = file.getParentFile(); if (!folder.exists()) { if (!folder.mkdirs()) { errors.add(filePath + ": couldn't create parent dir"); return; } } try (InputStream input = Util .getUnzipStream(new BufferedInputStream(helper.getDownloadStream(fi.getPath()))); OutputStream output = new BufferedOutputStream(new FileOutputStream(file))) { byte[] buffer = new byte[Math.min(fi.getSize(), 1 << 24)]; int pos = 0; int r; while ((r = input.read(buffer, pos, buffer.length - pos)) >= 0) { pos += r; if (pos == buffer.length) { output.write(buffer, 0, pos); pos = 0; } } if (pos != 0) { output.write(buffer, 0, pos); } System.out.println(filePath + ": OK"); } catch (IOException e) { String msg = filePath + ": FAIL: " + e.getClass().getSimpleName(); if (e.getMessage() != null) { msg += ": " + e.getMessage(); } errors.add(msg); } }, executor)).toArray(CompletableFuture[]::new); CompletableFuture.allOf(tasks).thenRun(() -> { for (String err : errors) System.err.println(err); executor.shutdown(); }); }
From source file:friendsandfollowers.FilesThreaderFriendsIDs.java
public static void main(String[] args) throws InterruptedException { // Check how many arguments were passed in if ((args == null) || (args.length < 4)) { System.err.println("4 Parameters are required plus one optional " + "parameter to launch a Job."); System.err.println("First: String 'INPUT: /path/to/files/'"); System.err.println("Second: String 'OUTPUT: /output/path/'"); System.err.println("Third: (int) Total Number Of Jobs"); System.err.println("Fourth: (int) Number of seconds to pause"); System.err/* w ww .j a v a 2 s .c om*/ .println("Fifth: (int) Number of ids to fetch. " + "Provide number which increment by 5000 eg " + "(5000, 10000, 15000 etc) " + "or -1 to fetch all ids."); System.err.println("Example: fileToRun /input/path/ " + "/output/path/ 10 2 75000"); System.exit(-1); } // to write output in a file ThreadPrintStream.replaceSystemOut(); try { INPUT = StringEscapeUtils.escapeJava(args[0]); } catch (Exception e) { System.err.println("Argument" + args[0] + " must be an String."); System.exit(-1); } try { OUTPUT = StringEscapeUtils.escapeJava(args[1]); } catch (Exception e) { System.err.println("Argument" + args[1] + " must be an String."); System.exit(-1); } try { TOTAL_JOBS_STR = StringEscapeUtils.escapeJava(args[2]); } catch (Exception e) { System.err.println("Argument" + args[2] + " must be an integer."); System.exit(-1); } try { PAUSE_STR = StringEscapeUtils.escapeJava(args[3]); } catch (Exception e) { System.err.println("Argument" + args[3] + " must be an integer."); System.exit(-1); } if (args.length == 5) { try { IDS_TO_FETCH = StringEscapeUtils.escapeJava(args[4]); } catch (Exception e) { System.err.println("Argument" + args[4] + " must be an integer."); System.exit(-1); } } try { PAUSE = Integer.parseInt(args[3]); } catch (NumberFormatException e) { System.err.println("Argument" + args[3] + " must be an integer."); System.exit(-1); } try { TOTAL_JOBS = Integer.parseInt(TOTAL_JOBS_STR); } catch (NumberFormatException e) { System.err.println("Argument" + TOTAL_JOBS_STR + " must be an integer."); System.exit(-1); } System.out.println("Going to launch jobs. " + "Please see logs files to track jobs."); ExecutorService threadPool = Executors.newFixedThreadPool(TOTAL_JOBS); for (int i = 0; i < TOTAL_JOBS; i++) { final String JOB_NO = Integer.toString(i); threadPool.submit(new Runnable() { public void run() { try { // Creating a text file where System.out.println() // will send its output for this thread. String name = Thread.currentThread().getName(); FileOutputStream fos = null; try { fos = new FileOutputStream(name + "-logs.txt"); } catch (Exception e) { System.err.println(e.getMessage()); System.exit(0); } // Create a PrintStream that will write to the new file. PrintStream stream = new PrintStream(new BufferedOutputStream(fos)); // Install the PrintStream to be used as // System.out for this thread. ((ThreadPrintStream) System.out).setThreadOut(stream); // Output three messages to System.out. System.out.println(name); System.out.println(); System.out.println(); FilesThreaderFriendsIDs.execTask(INPUT, OUTPUT, TOTAL_JOBS_STR, JOB_NO, PAUSE_STR, IDS_TO_FETCH); } catch (IOException | ClassNotFoundException | SQLException | JSONException e) { // e.printStackTrace(); System.out.println(e.getMessage()); } } }); helpers.pause(PAUSE); } threadPool.shutdown(); }
From source file:friendsandfollowers.FilesThreaderFollowersIDs.java
public static void main(String[] args) throws InterruptedException { // Check how many arguments were passed in if ((args == null) || (args.length < 4)) { System.err.println("4 Parameters are required plus one optional " + "parameter to launch a Job."); System.err.println("First: String 'INPUT: /path/to/files/'"); System.err.println("Second: String 'OUTPUT: /output/path/'"); System.err.println("Third: (int) Total Number Of Jobs"); System.err.println("Fourth: (int) Number of seconds to pause"); System.err/*from w ww .ja va 2s.c o m*/ .println("Fifth: (int) Number of ids to fetch. " + "Provide number which increment by 5000 eg " + "(5000, 10000, 15000 etc) " + "or -1 to fetch all ids."); System.err.println("Example: fileToRun /input/path/ " + "/output/path/ 10 2 75000"); System.exit(-1); } // to write output in a file ThreadPrintStream.replaceSystemOut(); try { INPUT = StringEscapeUtils.escapeJava(args[0]); } catch (Exception e) { System.err.println("Argument" + args[0] + " must be an String."); System.exit(-1); } try { OUTPUT = StringEscapeUtils.escapeJava(args[1]); } catch (Exception e) { System.err.println("Argument" + args[1] + " must be an String."); System.exit(-1); } try { TOTAL_JOBS_STR = StringEscapeUtils.escapeJava(args[2]); } catch (Exception e) { System.err.println("Argument" + args[2] + " must be an integer."); System.exit(-1); } try { PAUSE_STR = StringEscapeUtils.escapeJava(args[3]); } catch (Exception e) { System.err.println("Argument" + args[3] + " must be an integer."); System.exit(-1); } if (args.length == 5) { try { IDS_TO_FETCH = StringEscapeUtils.escapeJava(args[4]); } catch (Exception e) { System.err.println("Argument" + args[4] + " must be an integer."); System.exit(-1); } } try { PAUSE = Integer.parseInt(args[3]); } catch (NumberFormatException e) { System.err.println("Argument" + args[3] + " must be an integer."); System.exit(-1); } try { TOTAL_JOBS = Integer.parseInt(TOTAL_JOBS_STR); } catch (NumberFormatException e) { System.err.println("Argument" + TOTAL_JOBS_STR + " must be an integer."); System.exit(-1); } System.out.println("Going to launch jobs. " + "Please see logs files to track jobs."); ExecutorService threadPool = Executors.newFixedThreadPool(TOTAL_JOBS); for (int i = 0; i < TOTAL_JOBS; i++) { final String JOB_NO = Integer.toString(i); threadPool.submit(new Runnable() { public void run() { try { // Creating a text file where System.out.println() // will send its output for this thread. String name = Thread.currentThread().getName(); FileOutputStream fos = null; try { fos = new FileOutputStream(name + "-logs.txt"); } catch (Exception e) { System.err.println(e.getMessage()); System.exit(0); } // Create a PrintStream that will write to the new file. PrintStream stream = new PrintStream(new BufferedOutputStream(fos)); // Install the PrintStream to be used as // System.out for this thread. ((ThreadPrintStream) System.out).setThreadOut(stream); // Output three messages to System.out. System.out.println(name); System.out.println(); System.out.println(); FilesThreaderFollowersIDs.execTask(INPUT, OUTPUT, TOTAL_JOBS_STR, JOB_NO, PAUSE_STR, IDS_TO_FETCH); } catch (IOException | ClassNotFoundException | SQLException | JSONException e) { // e.printStackTrace(); System.out.println(e.getMessage()); } } }); helpers.pause(PAUSE); } threadPool.shutdown(); }
From source file:com.btoddb.fastpersitentqueue.speedtest.SpeedTest.java
public static void main(String[] args) throws Exception { if (0 == args.length) { System.out.println();/*w ww . j a v a 2 s . co m*/ System.out.println("ERROR: must specify the config file path/name"); System.out.println(); System.exit(1); } SpeedTestConfig config = SpeedTestConfig.create(args[0]); System.out.println(config.toString()); File theDir = new File(config.getDirectory(), "speed-" + UUID.randomUUID().toString()); FileUtils.forceMkdir(theDir); Fpq queue = config.getFpq(); queue.setJournalDirectory(new File(theDir, "journals")); queue.setPagingDirectory(new File(theDir, "pages")); try { queue.init(); // // start workers // AtomicLong counter = new AtomicLong(); AtomicLong pushSum = new AtomicLong(); AtomicLong popSum = new AtomicLong(); long startTime = System.currentTimeMillis(); Set<SpeedPushWorker> pushWorkers = new HashSet<SpeedPushWorker>(); for (int i = 0; i < config.getNumberOfPushers(); i++) { pushWorkers.add(new SpeedPushWorker(queue, config, counter, pushSum)); } Set<SpeedPopWorker> popWorkers = new HashSet<SpeedPopWorker>(); for (int i = 0; i < config.getNumberOfPoppers(); i++) { popWorkers.add(new SpeedPopWorker(queue, config, popSum)); } ExecutorService pusherExecSrvc = Executors.newFixedThreadPool( config.getNumberOfPushers() + config.getNumberOfPoppers(), new ThreadFactory() { @Override public Thread newThread(Runnable runnable) { Thread t = new Thread(runnable); t.setName("SpeedTest-Pusher"); return t; } }); ExecutorService popperExecSrvc = Executors.newFixedThreadPool( config.getNumberOfPushers() + config.getNumberOfPoppers(), new ThreadFactory() { @Override public Thread newThread(Runnable runnable) { Thread t = new Thread(runnable); t.setName("SpeedTest-Popper"); return t; } }); long startPushing = System.currentTimeMillis(); for (SpeedPushWorker sw : pushWorkers) { pusherExecSrvc.submit(sw); } long startPopping = System.currentTimeMillis(); for (SpeedPopWorker sw : popWorkers) { popperExecSrvc.submit(sw); } // // wait to finish // long endTime = startTime + config.getDurationOfTest() * 1000; long endPushing = 0; long displayTimer = 0; while (0 == endPushing || !queue.isEmpty()) { // display status every second if (1000 < (System.currentTimeMillis() - displayTimer)) { System.out.println(String.format("status (%ds) : journals = %d : memory segments = %d", (endTime - System.currentTimeMillis()) / 1000, queue.getJournalMgr().getJournalIdMap().size(), queue.getMemoryMgr().getSegments().size())); displayTimer = System.currentTimeMillis(); } pusherExecSrvc.shutdown(); if (pusherExecSrvc.awaitTermination(100, TimeUnit.MILLISECONDS)) { endPushing = System.currentTimeMillis(); // tell poppers, all pushers are finished for (SpeedPopWorker sw : popWorkers) { sw.stopWhenQueueEmpty(); } } } long endPopping = System.currentTimeMillis(); popperExecSrvc.shutdown(); popperExecSrvc.awaitTermination(10, TimeUnit.SECONDS); long numberOfPushes = 0; for (SpeedPushWorker sw : pushWorkers) { numberOfPushes += sw.getNumberOfEntries(); } long numberOfPops = 0; for (SpeedPopWorker sw : popWorkers) { numberOfPops += sw.getNumberOfEntries(); } long pushDuration = endPushing - startPushing; long popDuration = endPopping - startPopping; System.out.println("push - pop checksum = " + pushSum.get() + " - " + popSum.get() + " = " + (pushSum.get() - popSum.get())); System.out.println("push duration = " + pushDuration); System.out.println("pop duration = " + popDuration); System.out.println(); System.out.println("pushed = " + numberOfPushes); System.out.println("popped = " + numberOfPops); System.out.println(); System.out.println("push entries/sec = " + numberOfPushes / (pushDuration / 1000f)); System.out.println("pop entries/sec = " + numberOfPops / (popDuration / 1000f)); System.out.println(); System.out.println("journals created = " + queue.getJournalsCreated()); System.out.println("journals removed = " + queue.getJournalsRemoved()); } finally { if (null != queue) { queue.shutdown(); } // FileUtils.deleteDirectory(theDir); } }