List of usage examples for java.util Iterator hasNext
boolean hasNext();
From source file:ClassFileUtilities.java
/** * Program that computes the dependencies between the Batik jars. * <p>//from w w w.j av a 2 s .co m * Run this from the main Batik distribution directory, after building * the jars. For every jar file in the batik-xxx/ build directory, * it will determine which other jar files it directly depends on. * The output is lines of the form: * </p> * <pre> <i>number</i>,<i>from</i>,<i>to</i></pre> * <p> * where mean that the <i>from</i> jar has <i>number</i> class files * that depend on class files in the <i>to</i> jar. * </p> */ public static void main(String[] args) { boolean showFiles = false; if (args.length == 1 && args[0].equals("-f")) { showFiles = true; } else if (args.length != 0) { System.err.println("usage: ClassFileUtilities [-f]"); System.err.println(); System.err.println(" -f list files that cause each jar file dependency"); System.exit(1); } File cwd = new File("."); File buildDir = null; String[] cwdFiles = cwd.list(); for (int i = 0; i < cwdFiles.length; i++) { if (cwdFiles[i].startsWith("batik-")) { buildDir = new File(cwdFiles[i]); if (!buildDir.isDirectory()) { buildDir = null; } else { break; } } } if (buildDir == null || !buildDir.isDirectory()) { System.out.println("Directory 'batik-xxx' not found in current directory!"); return; } try { Map cs = new HashMap(); Map js = new HashMap(); collectJars(buildDir, js, cs); Set classpath = new HashSet(); Iterator i = js.values().iterator(); while (i.hasNext()) { classpath.add(((Jar) i.next()).jarFile); } i = cs.values().iterator(); while (i.hasNext()) { ClassFile fromFile = (ClassFile) i.next(); // System.out.println(fromFile.name); Set result = getClassDependencies(fromFile.getInputStream(), classpath, false); Iterator j = result.iterator(); while (j.hasNext()) { ClassFile toFile = (ClassFile) cs.get(j.next()); if (fromFile != toFile && toFile != null) { fromFile.deps.add(toFile); } } } i = cs.values().iterator(); while (i.hasNext()) { ClassFile fromFile = (ClassFile) i.next(); Iterator j = fromFile.deps.iterator(); while (j.hasNext()) { ClassFile toFile = (ClassFile) j.next(); Jar fromJar = fromFile.jar; Jar toJar = toFile.jar; if (fromFile.name.equals(toFile.name) || toJar == fromJar || fromJar.files.contains(toFile.name)) { continue; } Integer n = (Integer) fromJar.deps.get(toJar); if (n == null) { fromJar.deps.put(toJar, new Integer(1)); } else { fromJar.deps.put(toJar, new Integer(n.intValue() + 1)); } } } List triples = new ArrayList(10); i = js.values().iterator(); while (i.hasNext()) { Jar fromJar = (Jar) i.next(); Iterator j = fromJar.deps.keySet().iterator(); while (j.hasNext()) { Jar toJar = (Jar) j.next(); Triple t = new Triple(); t.from = fromJar; t.to = toJar; t.count = ((Integer) fromJar.deps.get(toJar)).intValue(); triples.add(t); } } Collections.sort(triples); i = triples.iterator(); while (i.hasNext()) { Triple t = (Triple) i.next(); System.out.println(t.count + "," + t.from.name + "," + t.to.name); if (showFiles) { Iterator j = t.from.files.iterator(); while (j.hasNext()) { ClassFile fromFile = (ClassFile) j.next(); Iterator k = fromFile.deps.iterator(); while (k.hasNext()) { ClassFile toFile = (ClassFile) k.next(); if (toFile.jar == t.to && !t.from.files.contains(toFile.name)) { System.out.println("\t" + fromFile.name + " --> " + toFile.name); } } } } } } catch (IOException e) { e.printStackTrace(); } }
From source file:com.github.xbn.examples.regexutil.non_xbn.MatchEachWordInEveryLine.java
public static final void main(String[] as_1RqdTxtFilePath) { Iterator<String> lineItr = null; try {// w w w . ja va 2 s .c o m lineItr = FileUtils.lineIterator(new File(as_1RqdTxtFilePath[0])); //Throws npx if null } catch (IOException iox) { throw new RuntimeException("Attempting to open \"" + as_1RqdTxtFilePath[0] + "\"", iox); } catch (RuntimeException rx) { throw new RuntimeException("One required parameter: The path to the text file.", rx); } //Dummy search string (""), so it can be reused (reset) Matcher mWord = Pattern.compile("\\b\\w+\\b").matcher(""); while (lineItr.hasNext()) { String sLine = lineItr.next(); mWord.reset(sLine); while (mWord.find()) { System.out.println(mWord.group()); } } }
From source file:com.moss.veracity.core.Veracity.java
public static void main(String[] args) throws Exception { File log4jConfigFile = new File("log4j.xml"); if (log4jConfigFile.exists()) { DOMConfigurator.configureAndWatch(log4jConfigFile.getAbsolutePath(), 1000); }/* w w w . j a v a 2s . co m*/ final Log log = LogFactory.getLog(Veracity.class); File homeDir = new File(System.getProperty("user.home")); File currentDir = new File(System.getProperty("user.dir")); List<File> configLocations = new LinkedList<File>(); configLocations.addAll(Arrays.asList(new File[] { new File("/etc/veracity.config"), new File(homeDir, ".veracity.config"), new File(currentDir, "config.xml") })); String customConfigFileProperty = System.getProperty("veracity.configFile"); if (customConfigFileProperty != null) { configLocations.clear(); configLocations.add(new File(customConfigFileProperty)); } File configFile = null; Iterator<File> i = configLocations.iterator(); while ((configFile == null || !configFile.exists()) && i.hasNext()) { configFile = i.next(); } LaunchParameters parameters; if (!configFile.exists()) { if (log.isDebugEnabled()) { log.debug("Creating default config file at " + configFile.getAbsolutePath()); } parameters = new LaunchParameters(); parameters.save(configFile); } else { if (log.isDebugEnabled()) { log.debug("Loading parameters from config file at " + configFile.getAbsolutePath()); } parameters = LaunchParameters.load(configFile); } parameters.readSystemProperties(); new Veracity(parameters); }
From source file:fr.iphc.grid.jobmanager.JobManager.java
/** * @param args/* ww w .j a va 2s .c o m*/ */ public static void main(String[] args) throws Exception { JobManager command = new JobManager(); CommandLine line = command.parse(args); ArrayList<File> JdlList = new ArrayList<File>(); Global.getOutputexecutor = Executors.newFixedThreadPool(10); Initialize init = new Initialize(); String SetupFile = "setup_vigrid.xml"; if (line.hasOption(OPT_SETUP)) { SetupFile = line.getOptionValue(OPT_SETUP); } if ((new File(SetupFile).isFile())) { init.GlobalSetup(SetupFile); } // Init Job if (line.hasOption(OPT_JOB)) { File file = new File(line.getOptionValue(OPT_JOB)); if ((file.isFile())) { JdlList.add(file); } else { System.err.println("The file " + file + " doesn't exist"); System.exit(-1); } } else { File file = new File(line.getOptionValue(OPT_FILEJOB)); if ((file.isFile())) { JdlList = init.InitJdl(file); } else { System.err.println("The file " + file + " doesn't exist"); System.exit(-1); } } if (line.hasOption(OPT_WAIT)) { Global.TIMEOUTWAIT = Integer.parseInt(line.getOptionValue(OPT_WAIT)); } if (line.hasOption(OPT_RUN)) { Global.TIMEOUTRUN = Integer.parseInt(line.getOptionValue(OPT_RUN)); } if (line.hasOption(OPT_END)) { Global.TIMEOUTEND = Integer.parseInt(line.getOptionValue(OPT_END)); } if (line.hasOption(OPT_LOGDISPLAY)) { Global.SEUILDISPLAYLOG = Float.parseFloat(line.getOptionValue(OPT_LOGDISPLAY)); } init.InitJob(JdlList); // Init Url Ce if (line.hasOption(OPT_QUEUE)) { Global.file = new File(line.getOptionValue(OPT_QUEUE)); } if (line.hasOption(OPT_BAD)) { Global.BadCe = new File(line.getOptionValue(OPT_BAD)); } if (line.hasOption(OPT_OPTIMIZETIMEOUTRUN)) { Global.OPTTIMEOUTRUN = false; } if (line.hasOption(OPT_CWD)) { File theDir = new File(line.getOptionValue(OPT_CWD)); if (!theDir.exists()) { if (!theDir.mkdirs()) { System.err.println("Working directory create failed: " + line.getOptionValue(OPT_CWD)); System.exit(-1); } } Global.Cwd = line.getOptionValue(OPT_CWD); } else { Global.Cwd = System.getProperty("user.dir"); } if (!(new File(Global.Cwd)).canWrite()) { System.err.println(" Write permission denied : " + Global.Cwd); System.exit(-1); } System.out.println("Current working directory : " + Global.Cwd); Date start = new Date(); init.PrintGlobalSetup(); init.InitUrl(Global.file); init.InitSosCe(); init.rmLoadFailed(Global.Cwd + "/loadFailed.txt"); System.out.println("CE: " + Global.ListUrl.size() + " Nb JOB: " + Global.ListJob.size() + " " + new Date()); if (Global.ListJob.size() < 6) { // pour obtenir rapport de 0.8 Global.OPTTIMEOUTRUN = false; } // check if we can connect to the grid try { SessionFactory.createSession(true); } catch (NoSuccessException e) { System.err.println("Could not connect to the grid at all (" + e.getMessage() + ")"); System.err.println("Aborting"); System.exit(0); } // Launch Tread Job JobThread st = new JobThread(Global.ListJob, Global.ListUrl); st.start(); LoggingThread logst = new LoggingThread(Global.ListJob, Global.ListUrl, Global.SEUILDISPLAYLOG); logst.start(); // create Thread Hook intercept kill +CNTL+C Thread hook = new Thread() { public void run() { try { for (Jdl job : Global.ListJob) { if (job.getJobId() != null) { JobThread.jobCancel(job.getJobId()); } } } catch (Exception e) { System.err.println("Thread Hook:\n" + e.getMessage()); } // give it a change to display final job state try { sleep(5000); } catch (InterruptedException e) { e.printStackTrace(); } } }; Runtime.getRuntime().addShutdownHook(hook); // Integer timer = 180 * 60 * 1000; Date now = new Date(); // Boolean Fin = false; while ((!Global.END) && ((now.getTime() - start.getTime()) < Global.TIMEOUTEND * 60 * 1000)) { // TOEND // en // minutes now = new Date(); // int mb = 1024*1024; // Getting the runtime reference from system // Runtime runtime = Runtime.getRuntime(); // System.out.println("##### Heap utilization statistics [MB] // #####"); // Print used memory // System.out.println("Used Memory:" // + (runtime.totalMemory() - runtime.freeMemory()) / mb); // Print free memory // System.out.println("Free Memory:" // + runtime.freeMemory() / mb); // Print total available memory // System.out.println("Total Memory:" + runtime.totalMemory() / mb); // Print Maximum available memory // System.out.println("Max Memory:" + runtime.maxMemory() / mb); // // System.out.println("NB: "+nb_end); // if ((float)(runtime.totalMemory() - // runtime.freeMemory())/(float)runtime.maxMemory() > (float)0.3){ // System.out.println ("GC: "+(float)(runtime.totalMemory() - // runtime.freeMemory())/runtime.maxMemory()); // System.gc(); // }; sleep(15 * 1000); // in ms // System.gc(); // Fin=true; // for (Jdl job : Global.ListJob) { // if (job.getJob() != null) { // System.out.println("JOB: "+job.getId()+"\t"+job.getStatus()); // if (job.getStatus().compareTo("END")==0){ // ((JobImpl) job.getJob()).postStagingAndCleanup(); // System.out.println("END JOB: "+job.getId()); // job.setStatus("END"); // } // if (job.getStatus().compareTo("END")!=0){ // Fin=false; // } // System.out.println("JOB: "+job.getId()+"\t"+job.getStatus() + // "\t"+job.getFail()+"\t"+job.getNodeCe()); // } // } // while ((Global.END==0) && ((new // Date().getTime()-start.getTime())<timer)){ } // Boolean end_load=false; // while (!end_load){ // end_load=true; // for(Jdl job:Global.ListJob){ // if (job.getStatus().equals("LOAD")){ // end_load=false; // } // } // } System.out.println("END JOB: " + now); st.halt(); logst.halt(); Iterator<Url> k = Global.ListUrl.iterator(); while (k.hasNext()) { Url url = k.next(); System.out.println("URL: " + url.getUrl()); } Iterator<Jdl> m = Global.ListJob.iterator(); while (m.hasNext()) { Jdl job = m.next(); System.out.println( "JOB: " + job.getId() + "\t" + job.getFail() + "\t" + job.getStatus() + "\t" + job.getNodeCe()); } System.out.println(start + " " + new Date()); System.exit(0); }
From source file:hepple.postag.POSTagger.java
/** * Main method. Runs the tagger using the arguments to find the resources * to be used for initialisation and the input file. *///from w ww.jav a 2 s .c om public static void main(String[] args) { if (args.length == 0) help(); try { LongOpt[] options = new LongOpt[] { new LongOpt("help", LongOpt.NO_ARGUMENT, null, 'h'), new LongOpt("lexicon", LongOpt.NO_ARGUMENT, null, 'l'), new LongOpt("rules", LongOpt.NO_ARGUMENT, null, 'r') }; Getopt getopt = new Getopt("HepTag", args, "hl:r:", options); String lexiconUrlString = null; String rulesUrlString = null; int opt; while ((opt = getopt.getopt()) != -1) { switch (opt) { // -h case 'h': { help(); System.exit(0); break; } // -l new lexicon case 'l': { lexiconUrlString = getopt.getOptarg(); break; } // -l new lexicon case 'r': { rulesUrlString = getopt.getOptarg(); break; } default: { System.err.println("Invalid option " + args[getopt.getOptind() - 1] + "!"); System.exit(1); } }//switch(opt) } //while( (opt = g.getopt()) != -1 ) String[] fileNames = new String[args.length - getopt.getOptind()]; for (int i = getopt.getOptind(); i < args.length; i++) { fileNames[i - getopt.getOptind()] = args[i]; } URL lexiconURL = (lexiconUrlString == null) ? POSTagger.class.getResource("/hepple/resources/sample_lexicon") : new File(lexiconUrlString).toURI().toURL(); URL rulesURL = (rulesUrlString == null) ? POSTagger.class.getResource("/hepple/resources/sample_ruleset.big") : new File(rulesUrlString).toURI().toURL(); POSTagger tagger = new POSTagger(lexiconURL, rulesURL); for (int i = 0; i < fileNames.length; i++) { String file = fileNames[i]; BufferedReader reader = null; try { reader = new BufferedReader(new FileReader(file)); String line = reader.readLine(); while (line != null) { StringTokenizer tokens = new StringTokenizer(line); List<String> sentence = new ArrayList<String>(); while (tokens.hasMoreTokens()) sentence.add(tokens.nextToken()); List<List<String>> sentences = new ArrayList<List<String>>(); sentences.add(sentence); List<List<String[]>> result = tagger.runTagger(sentences); Iterator<List<String[]>> iter = result.iterator(); while (iter.hasNext()) { List<String[]> sentenceFromTagger = iter.next(); Iterator<String[]> sentIter = sentenceFromTagger.iterator(); while (sentIter.hasNext()) { String[] tag = sentIter.next(); System.out.print(tag[0] + "/" + tag[1]); if (sentIter.hasNext()) System.out.print(" "); else System.out.println(); } //while(sentIter.hasNext()) } //while(iter.hasNext()) line = reader.readLine(); } //while(line != null) } finally { IOUtils.closeQuietly(reader); } // // // // List result = tagger.runTagger(readInput(file)); // Iterator iter = result.iterator(); // while(iter.hasNext()){ // List sentence = (List)iter.next(); // Iterator sentIter = sentence.iterator(); // while(sentIter.hasNext()){ // String[] tag = (String[])sentIter.next(); // System.out.print(tag[0] + "/" + tag[1]); // if(sentIter.hasNext()) System.out.print(" "); // else System.out.println(); // }//while(sentIter.hasNext()) // }//while(iter.hasNext()) } //for(int i = 0; i < fileNames.length; i++) } catch (Exception e) { e.printStackTrace(); } }
From source file:com.zimbra.cs.lmtpserver.utils.LmtpInject.java
public static void main(String[] args) { CliUtil.toolSetup();/*from w w w .ja v a2 s . c om*/ CommandLine cl = parseArgs(args); if (cl.hasOption("h")) { usage(null); } boolean quietMode = cl.hasOption("q"); int threads = 1; if (cl.hasOption("t")) { threads = Integer.valueOf(cl.getOptionValue("t")).intValue(); } String host = null; if (cl.hasOption("a")) { host = cl.getOptionValue("a"); } else { host = "localhost"; } int port; Protocol proto = null; if (cl.hasOption("smtp")) { proto = Protocol.SMTP; port = 25; } else port = 7025; if (cl.hasOption("p")) port = Integer.valueOf(cl.getOptionValue("p")).intValue(); String[] recipients = cl.getOptionValues("r"); String sender = cl.getOptionValue("s"); boolean tracingEnabled = cl.hasOption("T"); int everyN; if (cl.hasOption("N")) { everyN = Integer.valueOf(cl.getOptionValue("N")).intValue(); } else { everyN = 100; } // Process files from the -d option. List<File> files = new ArrayList<File>(); if (cl.hasOption("d")) { File dir = new File(cl.getOptionValue("d")); if (!dir.isDirectory()) { System.err.format("%s is not a directory.\n", dir.getPath()); System.exit(1); } File[] fileArray = dir.listFiles(); if (fileArray == null || fileArray.length == 0) { System.err.format("No files found in directory %s.\n", dir.getPath()); } Collections.addAll(files, fileArray); } // Process files specified as arguments. for (String arg : cl.getArgs()) { files.add(new File(arg)); } // Validate file content. if (!cl.hasOption("noValidation")) { Iterator<File> i = files.iterator(); while (i.hasNext()) { InputStream in = null; File file = i.next(); boolean valid = false; try { in = new FileInputStream(file); if (FileUtil.isGzipped(file)) { in = new GZIPInputStream(in); } in = new BufferedInputStream(in); // Required for RFC 822 check if (!EmailUtil.isRfc822Message(in)) { System.err.format("%s does not contain a valid RFC 822 message.\n", file.getPath()); } else { valid = true; } } catch (IOException e) { System.err.format("Unable to validate %s: %s.\n", file.getPath(), e.toString()); } finally { ByteUtil.closeStream(in); } if (!valid) { i.remove(); } } } if (files.size() == 0) { System.err.println("No files to inject."); System.exit(1); } if (!quietMode) { System.out.format( "Injecting %d message(s) to %d recipient(s). Server %s, port %d, using %d thread(s).\n", files.size(), recipients.length, host, port, threads); } int totalFailed = 0; int totalSucceeded = 0; long startTime = System.currentTimeMillis(); boolean verbose = cl.hasOption("v"); boolean skipTLSCertValidation = cl.hasOption("skipTLSCertValidation"); LmtpInject injector = null; try { injector = new LmtpInject(threads, sender, recipients, files, host, port, proto, quietMode, tracingEnabled, verbose, skipTLSCertValidation); } catch (Exception e) { mLog.error("Unable to initialize LmtpInject", e); System.exit(1); } injector.setReportEvery(everyN); injector.markStartTime(); try { injector.run(); } catch (IOException e) { e.printStackTrace(); System.exit(1); } int succeeded = injector.getSuccessCount(); int failedThisTime = injector.getFailureCount(); long elapsedMS = System.currentTimeMillis() - startTime; double elapsed = elapsedMS / 1000.0; double msPerMsg = 0.0; double msgSizeKB = 0.0; if (succeeded > 0) { msPerMsg = elapsedMS; msPerMsg /= succeeded; msgSizeKB = injector.mFileSizeTotal / 1024.0; msgSizeKB /= succeeded; } double msgPerSec = ((double) succeeded / (double) elapsedMS) * 1000; if (!quietMode) { System.out.println(); System.out.printf( "LmtpInject Finished\n" + "submitted=%d failed=%d\n" + "%.2fs, %.2fms/msg, %.2fmsg/s\n" + "average message size = %.2fKB\n", succeeded, failedThisTime, elapsed, msPerMsg, msgPerSec, msgSizeKB); } totalFailed += failedThisTime; totalSucceeded += succeeded; if (totalFailed != 0) System.exit(1); }
From source file:at.gv.egiz.pdfas.cli.Main.java
public static void main(String[] args) { // create the command line parser CommandLineParser parser = new GnuParser(); ModeOfOperation mode = ModeOfOperation.INVALID; try {// w w w . j a va2s.c o m CommandLine cli = parser.parse(createOptions(), args); if (cli.hasOption(CLI_ARG_DEPLOY_SHORT)) { PdfAsFactory.deployDefaultConfiguration(new File(STANDARD_CONFIG_LOCATION)); System.out.println("Configuration was deployed to: " + STANDARD_CONFIG_LOCATION); } if (cli.hasOption(CLI_ARG_MODE_SHORT)) { String modevalue = cli.getOptionValue(CLI_ARG_MODE_SHORT); if (modevalue.toLowerCase().trim().equals("sign")) { mode = ModeOfOperation.SIGN; } else if (modevalue.toLowerCase().trim().equals("verify")) { mode = ModeOfOperation.VERIFY; } else { throw new ParseException("Invalid value for option " + CLI_ARG_MODE_SHORT + ": " + modevalue); } } if (cli.hasOption(CLI_ARG_HELP_SHORT)) { usage(); System.exit(0); } if (mode == ModeOfOperation.INVALID && !cli.hasOption(CLI_ARG_DEPLOY_SHORT)) { usage(); } else if (mode == ModeOfOperation.SIGN) { perform_sign(cli); } else if (mode == ModeOfOperation.VERIFY) { perform_verify(cli); } } catch (ParseException e) { System.err.println("Invalid arguments: " + e.getMessage()); usage(); System.exit(-1); } catch (PDFASError e) { System.err.println("PDF-AS Error: [" + e.getCode() + "]" + e.getMessage()); Iterator<Entry<String, String>> infoIt = e.getProcessInformations().entrySet().iterator(); while (infoIt.hasNext()) { Entry<String, String> infoEntry = infoIt.next(); logger.debug("Process Information: {} = {}", infoEntry.getKey(), infoEntry.getValue()); } e.printStackTrace(System.err); System.exit(-1); } catch (Throwable e) { System.err.println("Unhandled PDF-AS Error: " + e.getMessage()); e.printStackTrace(System.err); System.exit(-1); } }
From source file:com.genentech.application.property.SDFCalculate.java
public static void main(String args[]) { String usage = "java SDFCalculate [options] <list of space separated properties>\n"; Options options = new Options(); // add options options.addOption("TPSA_P", false, "Count phosphorus atoms, default is false. (optional)"); options.addOption("TPSA_S", false, "Count sulfur atoms, default is false. (optional)"); options.addOption("cLogP", true, "SDtag where cLogP is stored, default is cLogP (optional)"); options.addOption("in", true, "inFile in OE formats: Ex: a.sdf or .sdf"); options.addOption("out", true, "outputfile in OE formats. Ex: a.sdf or .sdf "); try {/*from w ww .j a v a 2s. c o m*/ boolean countS = false; boolean countP = false; // append list of valid properties and their descriptions to the usage statement Iterator<Entry<String, String>> i = propsMap.entrySet().iterator(); while (i.hasNext()) { Map.Entry<String, String> me = i.next(); usage = usage + me.getKey() + ":\t" + me.getValue() + "\n"; } CommandLineParser parser = new PosixParser(); CommandLine cmd = parser.parse(options, args); if (cmd.hasOption("TPSA_P")) countP = true; if (cmd.hasOption("TPSA_S")) countS = true; // get list of properties Vector<String> propsList = new Vector<String>(Arrays.asList(cmd.getArgs())); if (propsList.isEmpty()) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp(usage, options); System.exit(1); } //make sure list of requested pros are valid props for (String p : propsList) { if (!propsMap.containsKey(p)) { System.err.println(p + " is not a valid property."); HelpFormatter formatter = new HelpFormatter(); formatter.printHelp(usage, options); System.exit(1); } } // get cLogP SD label tag option value String cLogPTag = "cLogP"; if (cmd.hasOption("cLogP")) { cLogPTag = cmd.getOptionValue("cLogP"); } String inFile = cmd.getOptionValue("in"); if (inFile == null) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp(usage, options); System.exit(1); } String outFile = cmd.getOptionValue("out"); if (outFile == null) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp(usage, options); System.exit(1); } String filename = "smarts.xml"; URL url = SDFCalculate.class.getResource(filename); Element root = XMLUtil.getRootElement(url, false); SDFCalculate test = new SDFCalculate(outFile, cLogPTag, countP, countS, root); test.calcProperties(inFile, propsList); } catch (ParseException e) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp(usage, options); System.exit(1); } catch (Exception e) { e.printStackTrace(); } }
From source file:edu.jhu.hlt.concrete.gigaword.expt.ConvertGigawordDocuments.java
/** * @param args/*from www . j a va 2s .c o m*/ */ public static void main(String... args) { Thread.setDefaultUncaughtExceptionHandler(new UncaughtExceptionHandler() { @Override public void uncaughtException(Thread t, Throwable e) { logger.error("Thread {} caught unhandled exception.", t.getName()); logger.error("Unhandled exception.", e); } }); if (args.length != 2) { logger.info("Usage: {} {} {}", GigawordConcreteConverter.class.getName(), "path/to/expt/file", "path/to/out/folder"); System.exit(1); } String exptPathStr = args[0]; String outPathStr = args[1]; // Verify path points to something. Path exptPath = Paths.get(exptPathStr); if (!Files.exists(exptPath)) { logger.error("File: {} does not exist. Re-run with the correct path to " + " the experiment 2 column file. See README.md."); System.exit(1); } logger.info("Experiment map located at: {}", exptPathStr); // Create output dir if not yet created. Path outPath = Paths.get(outPathStr); if (!Files.exists(outPath)) { logger.info("Creating directory: {}", outPath.toString()); try { Files.createDirectories(outPath); } catch (IOException e) { logger.error("Caught an IOException when creating output dir.", e); System.exit(1); } } logger.info("Output directory located at: {}", outPathStr); // Read in expt map. See README.md. Map<String, Set<String>> exptMap = null; try (Reader r = ExperimentUtils.createReader(exptPath); BufferedReader br = new BufferedReader(r)) { exptMap = ExperimentUtils.createFilenameToIdMap(br); } catch (IOException e) { logger.error("Caught an IOException when creating expt map.", e); System.exit(1); } // Start a timer. logger.info("Gigaword -> Concrete beginning."); StopWatch sw = new StopWatch(); sw.start(); // Iterate over expt map. exptMap.entrySet() // .parallelStream() .forEach(p -> { final String pathStr = p.getKey(); final Set<String> ids = p.getValue(); final Path lp = Paths.get(pathStr); logger.info("Converting path: {}", pathStr); // Get the file name and immediate folder it is under. int nElements = lp.getNameCount(); Path fileName = lp.getName(nElements - 1); Path subFolder = lp.getName(nElements - 2); String newFnStr = fileName.toString().split("\\.")[0] + ".tar"; // Mirror folders in output dir. Path localOutFolder = outPath.resolve(subFolder); Path localOutPath = localOutFolder.resolve(newFnStr); // Create output subfolders. if (!Files.exists(localOutFolder) && !Files.isDirectory(localOutFolder)) { logger.info("Creating out file: {}", localOutFolder.toString()); try { Files.createDirectories(localOutFolder); } catch (IOException e) { throw new RuntimeException("Caught an IOException when creating output dir.", e); } } // Iterate over communications. Iterator<Communication> citer; try (OutputStream os = Files.newOutputStream(localOutPath); BufferedOutputStream bos = new BufferedOutputStream(os); Archiver archiver = new TarArchiver(bos);) { citer = new ConcreteGigawordDocumentFactory().iterator(lp); while (citer.hasNext()) { Communication c = citer.next(); String cId = c.getId(); // Document ID must be in the set. Remove. boolean wasInSet = ids.remove(cId); if (!wasInSet) { // Some IDs are duplicated in Gigaword. // See ERRATA. logger.debug( "ID: {} was parsed from path: {}, but was not in the experiment map. Attempting to remove dupe.", cId, pathStr); // Attempt to create a duplicate id (append .duplicate to the id). // Then, try to remove again. String newId = RepairDuplicateIDs.repairDuplicate(cId); boolean dupeRemoved = ids.remove(newId); // There are not nested duplicates, so this should never fire. if (!dupeRemoved) { logger.info("Failed to remove dupe."); return; } else // Modify the communication ID to the unique version. c.setId(newId); } archiver.addEntry(new ArchivableCommunication(c)); } logger.info("Finished path: {}", pathStr); } catch (ConcreteException ex) { logger.error("Caught ConcreteException during Concrete mapping.", ex); logger.error("Path: {}", pathStr); } catch (IOException e) { logger.error("Error archiving communications.", e); logger.error("Path: {}", localOutPath.toString()); } }); sw.stop(); logger.info("Finished."); Minutes m = new Duration(sw.getTime()).toStandardMinutes(); logger.info("Runtime: Approximately {} minutes.", m.getMinutes()); }
From source file:asl.seedscan.SeedScan.java
public static void main(String args[]) { // Default locations of config and schema files File configFile = new File("config.xml"); File schemaFile = new File("schemas/SeedScanConfig.xsd"); boolean parseConfig = true; ArrayList<File> schemaFiles = new ArrayList<File>(); schemaFiles.add(schemaFile);/*from ww w . ja v a2 s. c om*/ // ==== Command Line Parsing ==== Options options = new Options(); Option opConfigFile = new Option("c", "config-file", true, "The config file to use for seedscan. XML format according to SeedScanConfig.xsd."); Option opSchemaFile = new Option("s", "schema-file", true, "The xsd schema file which should be used to verify the config file format. "); OptionGroup ogConfig = new OptionGroup(); ogConfig.addOption(opConfigFile); OptionGroup ogSchema = new OptionGroup(); ogConfig.addOption(opSchemaFile); options.addOptionGroup(ogConfig); options.addOptionGroup(ogSchema); PosixParser optParser = new PosixParser(); CommandLine cmdLine = null; try { cmdLine = optParser.parse(options, args, true); } catch (org.apache.commons.cli.ParseException e) { logger.error("Error while parsing command-line arguments."); System.exit(1); } Option opt; Iterator<?> iter = cmdLine.iterator(); while (iter.hasNext()) { opt = (Option) iter.next(); if (opt.getOpt().equals("c")) { configFile = new File(opt.getValue()); } else if (opt.getOpt().equals("s")) { schemaFile = new File(opt.getValue()); } } // ==== Configuration Read and Parse Actions ==== ConfigParser parser = new ConfigParser(schemaFiles); ConfigT config = parser.parseConfig(configFile); // Print out configuration file contents Formatter formatter = new Formatter(new StringBuilder(), Locale.US); // ===== CONFIG: LOCK FILE ===== File lockFile = new File(config.getLockfile()); logger.info("SeedScan lock file is '" + lockFile + "'"); LockFile lock = new LockFile(lockFile); if (!lock.acquire()) { logger.error("Could not acquire lock."); System.exit(1); } // ===== CONFIG: LOGGING ===== // MTH: This is now done in log4j.properties file // ===== CONFIG: DATABASE ===== MetricDatabase readDB = new MetricDatabase(config.getDatabase()); MetricDatabase writeDB = new MetricDatabase(config.getDatabase()); MetricReader reader = new MetricReader(readDB); MetricInjector injector = new MetricInjector(writeDB); // ===== CONFIG: SCANS ===== Hashtable<String, Scan> scans = new Hashtable<String, Scan>(); if (config.getScans().getScan() == null) { logger.error("No scans in configuration."); System.exit(1); } else { for (ScanT scanCfg : config.getScans().getScan()) { String name = scanCfg.getName(); if (scans.containsKey(name)) { logger.error("Duplicate scan name '" + name + "' encountered."); System.exit(1); } // This should really be handled by jaxb by setting it up in schemas/SeedScanConfig.xsd if (scanCfg.getStartDay() == null && scanCfg.getStartDate() == null) { logger.error( "== SeedScan Error: Must set EITHER cfg:start_day -OR- cfg:start_date in config.xml to start Scan!"); System.exit(1); } // Configure this Scan Scan scan = new Scan(scanCfg.getName()); scan.setPathPattern(scanCfg.getPath()); scan.setDatalessDir(scanCfg.getDatalessDir()); scan.setEventsDir(scanCfg.getEventsDir()); scan.setPlotsDir(scanCfg.getPlotsDir()); scan.setDaysToScan(scanCfg.getDaysToScan().intValue()); if (scanCfg.getStartDay() != null) { scan.setStartDay(scanCfg.getStartDay().intValue()); } if (scanCfg.getStartDate() != null) { scan.setStartDate(scanCfg.getStartDate().intValue()); } if (scanCfg.getNetworkSubset() != null) { logger.debug("Filter on Network Subset=[{}]", scanCfg.getNetworkSubset()); Filter filter = new Filter(false); for (String network : scanCfg.getNetworkSubset().split(",")) { logger.debug("Network =[{}]", network); filter.addFilter(network); } scan.setNetworks(filter); } if (scanCfg.getStationSubset() != null) { logger.debug("Filter on Station Subset=[{}]", scanCfg.getStationSubset()); Filter filter = new Filter(false); for (String station : scanCfg.getStationSubset().split(",")) { logger.debug("Station =[{}]", station); filter.addFilter(station); } scan.setStations(filter); } if (scanCfg.getLocationSubset() != null) { logger.debug("Filter on Location Subset=[{}]", scanCfg.getLocationSubset()); Filter filter = new Filter(false); for (String location : scanCfg.getLocationSubset().split(",")) { logger.debug("Location =[{}]", location); filter.addFilter(location); } scan.setLocations(filter); } if (scanCfg.getChannelSubset() != null) { logger.debug("Filter on Channel Subset=[{}]", scanCfg.getChannelSubset()); Filter filter = new Filter(false); for (String channel : scanCfg.getChannelSubset().split(",")) { logger.debug("Channel =[{}]", channel); filter.addFilter(channel); } scan.setChannels(filter); } for (MetricT met : scanCfg.getMetrics().getMetric()) { try { Class<?> metricClass = Class.forName(met.getClassName()); MetricWrapper wrapper = new MetricWrapper(metricClass); for (ArgumentT arg : met.getArgument()) { wrapper.add(arg.getName(), arg.getValue()); } scan.addMetric(wrapper); } catch (ClassNotFoundException ex) { logger.error("No such metric class '" + met.getClassName() + "'"); System.exit(1); } catch (InstantiationException ex) { logger.error("Could not dynamically instantiate class '" + met.getClassName() + "'"); System.exit(1); } catch (IllegalAccessException ex) { logger.error("Illegal access while loading class '" + met.getClassName() + "'"); System.exit(1); } catch (NoSuchFieldException ex) { logger.error("Invalid dynamic argument to Metric subclass '" + met.getClassName() + "'"); System.exit(1); } } scans.put(name, scan); } } // ==== Establish Database Connection ==== // TODO: State Tracking in the Database // - Record scan started in database. // - Track our progress as we go so a new process can pick up where // we left off if our process dies. // - Mark when each date-station-channel-operation is complete //LogDatabaseHandler logDB = new LogDatabaseHandler(configuration.get // For each day ((yesterday - scanDepth) to yesterday) // scan for these channel files, only process them if // they have not yet been scanned, or if changes have // occurred to the file since its last scan. Do this for // each scan type. Do not re-scan data for each type, // launch processes for each scan and use the same data set // for each. If we can pipe the data as it is read, do so. // If we need to push all of it at once, do these in sequence // in order to preserve overall system memory resources. Scan scan = null; // ==== Perform Scans ==== scan = scans.get("daily"); //MTH: This part could/should be moved up higher except that we need to know datalessDir, which, // at this point, is configured on a per scan basis ... so we need to know what scan we're doing MetaServer metaServer = null; if (config.getMetaserver() != null) { if (config.getMetaserver().getUseRemote().equals("yes") || config.getMetaserver().getUseRemote().equals("true")) { String remoteServer = config.getMetaserver().getRemoteUri(); try { metaServer = new MetaServer(new URI(remoteServer)); } catch (Exception e) { logger.error("caught URI exception:" + e.getMessage()); } } else { metaServer = new MetaServer(scan.getDatalessDir()); } } else { // Use local MetaServer metaServer = new MetaServer(scan.getDatalessDir()); } List<Station> stations = null; if (config.getStationList() == null) { // get StationList from MetaServer logger.info("Get StationList from MetaServer"); stations = metaServer.getStationList(); } else { // read StationList from config.xml logger.info("Read StationList from config.xml"); List<String> stationList = config.getStationList().getStation(); if (stationList.size() > 0) { stations = new ArrayList<Station>(); for (String station : stationList) { String[] words = station.split("_"); if (words.length != 2) { logger.warn(String.format("stationList: station=[%s] is NOT a valid station --> Skip", station)); } else { stations.add(new Station(words[0], words[1])); logger.info("config.xml: Read station:" + station); } } } else { logger.error("Error: No valid stations read from config.xml"); } } if (stations == null) { logger.error("Found NO stations to scan --> EXITTING SeedScan"); System.exit(1); } Thread readerThread = new Thread(reader); readerThread.start(); logger.info("Reader thread started."); Thread injectorThread = new Thread(injector); injectorThread.start(); logger.info("Injector thread started."); // Loop over scans and hand each one to a ScanManager logger.info("Hand scan to ScanManager"); for (String key : scans.keySet()) { scan = scans.get(key); logger.info(String.format("Scan=[%s] startDay=%d startDate=%d daysToScan=%d\n", key, scan.getStartDay(), scan.getStartDate(), scan.getDaysToScan())); ScanManager scanManager = new ScanManager(reader, injector, stations, scan, metaServer); } logger.info("ScanManager is [ FINISHED ] --> stop the injector and reader threads"); try { injector.halt(); logger.info("All stations processed. Waiting for injector thread to finish..."); synchronized (injectorThread) { //injectorThread.wait(); injectorThread.interrupt(); } logger.info("Injector thread halted."); } catch (InterruptedException ex) { logger.warn("The injector thread was interrupted while attempting to complete requests."); } try { reader.halt(); logger.info("All stations processed. Waiting for reader thread to finish..."); synchronized (readerThread) { //readerThread.wait(); readerThread.interrupt(); } logger.info("Reader thread halted."); } catch (InterruptedException ex) { logger.warn("The reader thread was interrupted while attempting to complete requests."); } try { lock.release(); } catch (IOException e) { ; } finally { logger.info("Release seedscan lock and quit metaServer"); lock = null; metaServer.quit(); } }