List of usage examples for java.util Iterator next
E next();
From source file:hepple.postag.POSTagger.java
/** * Main method. Runs the tagger using the arguments to find the resources * to be used for initialisation and the input file. *//*from w ww .j ava2s. co m*/ public static void main(String[] args) { if (args.length == 0) help(); try { LongOpt[] options = new LongOpt[] { new LongOpt("help", LongOpt.NO_ARGUMENT, null, 'h'), new LongOpt("lexicon", LongOpt.NO_ARGUMENT, null, 'l'), new LongOpt("rules", LongOpt.NO_ARGUMENT, null, 'r') }; Getopt getopt = new Getopt("HepTag", args, "hl:r:", options); String lexiconUrlString = null; String rulesUrlString = null; int opt; while ((opt = getopt.getopt()) != -1) { switch (opt) { // -h case 'h': { help(); System.exit(0); break; } // -l new lexicon case 'l': { lexiconUrlString = getopt.getOptarg(); break; } // -l new lexicon case 'r': { rulesUrlString = getopt.getOptarg(); break; } default: { System.err.println("Invalid option " + args[getopt.getOptind() - 1] + "!"); System.exit(1); } }//switch(opt) } //while( (opt = g.getopt()) != -1 ) String[] fileNames = new String[args.length - getopt.getOptind()]; for (int i = getopt.getOptind(); i < args.length; i++) { fileNames[i - getopt.getOptind()] = args[i]; } URL lexiconURL = (lexiconUrlString == null) ? POSTagger.class.getResource("/hepple/resources/sample_lexicon") : new File(lexiconUrlString).toURI().toURL(); URL rulesURL = (rulesUrlString == null) ? POSTagger.class.getResource("/hepple/resources/sample_ruleset.big") : new File(rulesUrlString).toURI().toURL(); POSTagger tagger = new POSTagger(lexiconURL, rulesURL); for (int i = 0; i < fileNames.length; i++) { String file = fileNames[i]; BufferedReader reader = null; try { reader = new BufferedReader(new FileReader(file)); String line = reader.readLine(); while (line != null) { StringTokenizer tokens = new StringTokenizer(line); List<String> sentence = new ArrayList<String>(); while (tokens.hasMoreTokens()) sentence.add(tokens.nextToken()); List<List<String>> sentences = new ArrayList<List<String>>(); sentences.add(sentence); List<List<String[]>> result = tagger.runTagger(sentences); Iterator<List<String[]>> iter = result.iterator(); while (iter.hasNext()) { List<String[]> sentenceFromTagger = iter.next(); Iterator<String[]> sentIter = sentenceFromTagger.iterator(); while (sentIter.hasNext()) { String[] tag = sentIter.next(); System.out.print(tag[0] + "/" + tag[1]); if (sentIter.hasNext()) System.out.print(" "); else System.out.println(); } //while(sentIter.hasNext()) } //while(iter.hasNext()) line = reader.readLine(); } //while(line != null) } finally { IOUtils.closeQuietly(reader); } // // // // List result = tagger.runTagger(readInput(file)); // Iterator iter = result.iterator(); // while(iter.hasNext()){ // List sentence = (List)iter.next(); // Iterator sentIter = sentence.iterator(); // while(sentIter.hasNext()){ // String[] tag = (String[])sentIter.next(); // System.out.print(tag[0] + "/" + tag[1]); // if(sentIter.hasNext()) System.out.print(" "); // else System.out.println(); // }//while(sentIter.hasNext()) // }//while(iter.hasNext()) } //for(int i = 0; i < fileNames.length; i++) } catch (Exception e) { e.printStackTrace(); } }
From source file:asl.seedscan.SeedScan.java
public static void main(String args[]) { // Default locations of config and schema files File configFile = new File("config.xml"); File schemaFile = new File("schemas/SeedScanConfig.xsd"); boolean parseConfig = true; ArrayList<File> schemaFiles = new ArrayList<File>(); schemaFiles.add(schemaFile);//www .j a v a 2s .c o m // ==== Command Line Parsing ==== Options options = new Options(); Option opConfigFile = new Option("c", "config-file", true, "The config file to use for seedscan. XML format according to SeedScanConfig.xsd."); Option opSchemaFile = new Option("s", "schema-file", true, "The xsd schema file which should be used to verify the config file format. "); OptionGroup ogConfig = new OptionGroup(); ogConfig.addOption(opConfigFile); OptionGroup ogSchema = new OptionGroup(); ogConfig.addOption(opSchemaFile); options.addOptionGroup(ogConfig); options.addOptionGroup(ogSchema); PosixParser optParser = new PosixParser(); CommandLine cmdLine = null; try { cmdLine = optParser.parse(options, args, true); } catch (org.apache.commons.cli.ParseException e) { logger.error("Error while parsing command-line arguments."); System.exit(1); } Option opt; Iterator<?> iter = cmdLine.iterator(); while (iter.hasNext()) { opt = (Option) iter.next(); if (opt.getOpt().equals("c")) { configFile = new File(opt.getValue()); } else if (opt.getOpt().equals("s")) { schemaFile = new File(opt.getValue()); } } // ==== Configuration Read and Parse Actions ==== ConfigParser parser = new ConfigParser(schemaFiles); ConfigT config = parser.parseConfig(configFile); // Print out configuration file contents Formatter formatter = new Formatter(new StringBuilder(), Locale.US); // ===== CONFIG: LOCK FILE ===== File lockFile = new File(config.getLockfile()); logger.info("SeedScan lock file is '" + lockFile + "'"); LockFile lock = new LockFile(lockFile); if (!lock.acquire()) { logger.error("Could not acquire lock."); System.exit(1); } // ===== CONFIG: LOGGING ===== // MTH: This is now done in log4j.properties file // ===== CONFIG: DATABASE ===== MetricDatabase readDB = new MetricDatabase(config.getDatabase()); MetricDatabase writeDB = new MetricDatabase(config.getDatabase()); MetricReader reader = new MetricReader(readDB); MetricInjector injector = new MetricInjector(writeDB); // ===== CONFIG: SCANS ===== Hashtable<String, Scan> scans = new Hashtable<String, Scan>(); if (config.getScans().getScan() == null) { logger.error("No scans in configuration."); System.exit(1); } else { for (ScanT scanCfg : config.getScans().getScan()) { String name = scanCfg.getName(); if (scans.containsKey(name)) { logger.error("Duplicate scan name '" + name + "' encountered."); System.exit(1); } // This should really be handled by jaxb by setting it up in schemas/SeedScanConfig.xsd if (scanCfg.getStartDay() == null && scanCfg.getStartDate() == null) { logger.error( "== SeedScan Error: Must set EITHER cfg:start_day -OR- cfg:start_date in config.xml to start Scan!"); System.exit(1); } // Configure this Scan Scan scan = new Scan(scanCfg.getName()); scan.setPathPattern(scanCfg.getPath()); scan.setDatalessDir(scanCfg.getDatalessDir()); scan.setEventsDir(scanCfg.getEventsDir()); scan.setPlotsDir(scanCfg.getPlotsDir()); scan.setDaysToScan(scanCfg.getDaysToScan().intValue()); if (scanCfg.getStartDay() != null) { scan.setStartDay(scanCfg.getStartDay().intValue()); } if (scanCfg.getStartDate() != null) { scan.setStartDate(scanCfg.getStartDate().intValue()); } if (scanCfg.getNetworkSubset() != null) { logger.debug("Filter on Network Subset=[{}]", scanCfg.getNetworkSubset()); Filter filter = new Filter(false); for (String network : scanCfg.getNetworkSubset().split(",")) { logger.debug("Network =[{}]", network); filter.addFilter(network); } scan.setNetworks(filter); } if (scanCfg.getStationSubset() != null) { logger.debug("Filter on Station Subset=[{}]", scanCfg.getStationSubset()); Filter filter = new Filter(false); for (String station : scanCfg.getStationSubset().split(",")) { logger.debug("Station =[{}]", station); filter.addFilter(station); } scan.setStations(filter); } if (scanCfg.getLocationSubset() != null) { logger.debug("Filter on Location Subset=[{}]", scanCfg.getLocationSubset()); Filter filter = new Filter(false); for (String location : scanCfg.getLocationSubset().split(",")) { logger.debug("Location =[{}]", location); filter.addFilter(location); } scan.setLocations(filter); } if (scanCfg.getChannelSubset() != null) { logger.debug("Filter on Channel Subset=[{}]", scanCfg.getChannelSubset()); Filter filter = new Filter(false); for (String channel : scanCfg.getChannelSubset().split(",")) { logger.debug("Channel =[{}]", channel); filter.addFilter(channel); } scan.setChannels(filter); } for (MetricT met : scanCfg.getMetrics().getMetric()) { try { Class<?> metricClass = Class.forName(met.getClassName()); MetricWrapper wrapper = new MetricWrapper(metricClass); for (ArgumentT arg : met.getArgument()) { wrapper.add(arg.getName(), arg.getValue()); } scan.addMetric(wrapper); } catch (ClassNotFoundException ex) { logger.error("No such metric class '" + met.getClassName() + "'"); System.exit(1); } catch (InstantiationException ex) { logger.error("Could not dynamically instantiate class '" + met.getClassName() + "'"); System.exit(1); } catch (IllegalAccessException ex) { logger.error("Illegal access while loading class '" + met.getClassName() + "'"); System.exit(1); } catch (NoSuchFieldException ex) { logger.error("Invalid dynamic argument to Metric subclass '" + met.getClassName() + "'"); System.exit(1); } } scans.put(name, scan); } } // ==== Establish Database Connection ==== // TODO: State Tracking in the Database // - Record scan started in database. // - Track our progress as we go so a new process can pick up where // we left off if our process dies. // - Mark when each date-station-channel-operation is complete //LogDatabaseHandler logDB = new LogDatabaseHandler(configuration.get // For each day ((yesterday - scanDepth) to yesterday) // scan for these channel files, only process them if // they have not yet been scanned, or if changes have // occurred to the file since its last scan. Do this for // each scan type. Do not re-scan data for each type, // launch processes for each scan and use the same data set // for each. If we can pipe the data as it is read, do so. // If we need to push all of it at once, do these in sequence // in order to preserve overall system memory resources. Scan scan = null; // ==== Perform Scans ==== scan = scans.get("daily"); //MTH: This part could/should be moved up higher except that we need to know datalessDir, which, // at this point, is configured on a per scan basis ... so we need to know what scan we're doing MetaServer metaServer = null; if (config.getMetaserver() != null) { if (config.getMetaserver().getUseRemote().equals("yes") || config.getMetaserver().getUseRemote().equals("true")) { String remoteServer = config.getMetaserver().getRemoteUri(); try { metaServer = new MetaServer(new URI(remoteServer)); } catch (Exception e) { logger.error("caught URI exception:" + e.getMessage()); } } else { metaServer = new MetaServer(scan.getDatalessDir()); } } else { // Use local MetaServer metaServer = new MetaServer(scan.getDatalessDir()); } List<Station> stations = null; if (config.getStationList() == null) { // get StationList from MetaServer logger.info("Get StationList from MetaServer"); stations = metaServer.getStationList(); } else { // read StationList from config.xml logger.info("Read StationList from config.xml"); List<String> stationList = config.getStationList().getStation(); if (stationList.size() > 0) { stations = new ArrayList<Station>(); for (String station : stationList) { String[] words = station.split("_"); if (words.length != 2) { logger.warn(String.format("stationList: station=[%s] is NOT a valid station --> Skip", station)); } else { stations.add(new Station(words[0], words[1])); logger.info("config.xml: Read station:" + station); } } } else { logger.error("Error: No valid stations read from config.xml"); } } if (stations == null) { logger.error("Found NO stations to scan --> EXITTING SeedScan"); System.exit(1); } Thread readerThread = new Thread(reader); readerThread.start(); logger.info("Reader thread started."); Thread injectorThread = new Thread(injector); injectorThread.start(); logger.info("Injector thread started."); // Loop over scans and hand each one to a ScanManager logger.info("Hand scan to ScanManager"); for (String key : scans.keySet()) { scan = scans.get(key); logger.info(String.format("Scan=[%s] startDay=%d startDate=%d daysToScan=%d\n", key, scan.getStartDay(), scan.getStartDate(), scan.getDaysToScan())); ScanManager scanManager = new ScanManager(reader, injector, stations, scan, metaServer); } logger.info("ScanManager is [ FINISHED ] --> stop the injector and reader threads"); try { injector.halt(); logger.info("All stations processed. Waiting for injector thread to finish..."); synchronized (injectorThread) { //injectorThread.wait(); injectorThread.interrupt(); } logger.info("Injector thread halted."); } catch (InterruptedException ex) { logger.warn("The injector thread was interrupted while attempting to complete requests."); } try { reader.halt(); logger.info("All stations processed. Waiting for reader thread to finish..."); synchronized (readerThread) { //readerThread.wait(); readerThread.interrupt(); } logger.info("Reader thread halted."); } catch (InterruptedException ex) { logger.warn("The reader thread was interrupted while attempting to complete requests."); } try { lock.release(); } catch (IOException e) { ; } finally { logger.info("Release seedscan lock and quit metaServer"); lock = null; metaServer.quit(); } }
From source file:com.genentech.application.property.SDFCalculate.java
public static void main(String args[]) { String usage = "java SDFCalculate [options] <list of space separated properties>\n"; Options options = new Options(); // add options options.addOption("TPSA_P", false, "Count phosphorus atoms, default is false. (optional)"); options.addOption("TPSA_S", false, "Count sulfur atoms, default is false. (optional)"); options.addOption("cLogP", true, "SDtag where cLogP is stored, default is cLogP (optional)"); options.addOption("in", true, "inFile in OE formats: Ex: a.sdf or .sdf"); options.addOption("out", true, "outputfile in OE formats. Ex: a.sdf or .sdf "); try {/*from w w w.ja v a2s . c o m*/ boolean countS = false; boolean countP = false; // append list of valid properties and their descriptions to the usage statement Iterator<Entry<String, String>> i = propsMap.entrySet().iterator(); while (i.hasNext()) { Map.Entry<String, String> me = i.next(); usage = usage + me.getKey() + ":\t" + me.getValue() + "\n"; } CommandLineParser parser = new PosixParser(); CommandLine cmd = parser.parse(options, args); if (cmd.hasOption("TPSA_P")) countP = true; if (cmd.hasOption("TPSA_S")) countS = true; // get list of properties Vector<String> propsList = new Vector<String>(Arrays.asList(cmd.getArgs())); if (propsList.isEmpty()) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp(usage, options); System.exit(1); } //make sure list of requested pros are valid props for (String p : propsList) { if (!propsMap.containsKey(p)) { System.err.println(p + " is not a valid property."); HelpFormatter formatter = new HelpFormatter(); formatter.printHelp(usage, options); System.exit(1); } } // get cLogP SD label tag option value String cLogPTag = "cLogP"; if (cmd.hasOption("cLogP")) { cLogPTag = cmd.getOptionValue("cLogP"); } String inFile = cmd.getOptionValue("in"); if (inFile == null) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp(usage, options); System.exit(1); } String outFile = cmd.getOptionValue("out"); if (outFile == null) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp(usage, options); System.exit(1); } String filename = "smarts.xml"; URL url = SDFCalculate.class.getResource(filename); Element root = XMLUtil.getRootElement(url, false); SDFCalculate test = new SDFCalculate(outFile, cLogPTag, countP, countS, root); test.calcProperties(inFile, propsList); } catch (ParseException e) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp(usage, options); System.exit(1); } catch (Exception e) { e.printStackTrace(); } }
From source file:at.gv.egiz.pdfas.cli.Main.java
public static void main(String[] args) { // create the command line parser CommandLineParser parser = new GnuParser(); ModeOfOperation mode = ModeOfOperation.INVALID; try {/*w w w . j ava 2 s .c o m*/ CommandLine cli = parser.parse(createOptions(), args); if (cli.hasOption(CLI_ARG_DEPLOY_SHORT)) { PdfAsFactory.deployDefaultConfiguration(new File(STANDARD_CONFIG_LOCATION)); System.out.println("Configuration was deployed to: " + STANDARD_CONFIG_LOCATION); } if (cli.hasOption(CLI_ARG_MODE_SHORT)) { String modevalue = cli.getOptionValue(CLI_ARG_MODE_SHORT); if (modevalue.toLowerCase().trim().equals("sign")) { mode = ModeOfOperation.SIGN; } else if (modevalue.toLowerCase().trim().equals("verify")) { mode = ModeOfOperation.VERIFY; } else { throw new ParseException("Invalid value for option " + CLI_ARG_MODE_SHORT + ": " + modevalue); } } if (cli.hasOption(CLI_ARG_HELP_SHORT)) { usage(); System.exit(0); } if (mode == ModeOfOperation.INVALID && !cli.hasOption(CLI_ARG_DEPLOY_SHORT)) { usage(); } else if (mode == ModeOfOperation.SIGN) { perform_sign(cli); } else if (mode == ModeOfOperation.VERIFY) { perform_verify(cli); } } catch (ParseException e) { System.err.println("Invalid arguments: " + e.getMessage()); usage(); System.exit(-1); } catch (PDFASError e) { System.err.println("PDF-AS Error: [" + e.getCode() + "]" + e.getMessage()); Iterator<Entry<String, String>> infoIt = e.getProcessInformations().entrySet().iterator(); while (infoIt.hasNext()) { Entry<String, String> infoEntry = infoIt.next(); logger.debug("Process Information: {} = {}", infoEntry.getKey(), infoEntry.getValue()); } e.printStackTrace(System.err); System.exit(-1); } catch (Throwable e) { System.err.println("Unhandled PDF-AS Error: " + e.getMessage()); e.printStackTrace(System.err); System.exit(-1); } }
From source file:com.zimbra.cs.lmtpserver.utils.LmtpInject.java
public static void main(String[] args) { CliUtil.toolSetup();/*w ww. ja va 2 s . c o m*/ CommandLine cl = parseArgs(args); if (cl.hasOption("h")) { usage(null); } boolean quietMode = cl.hasOption("q"); int threads = 1; if (cl.hasOption("t")) { threads = Integer.valueOf(cl.getOptionValue("t")).intValue(); } String host = null; if (cl.hasOption("a")) { host = cl.getOptionValue("a"); } else { host = "localhost"; } int port; Protocol proto = null; if (cl.hasOption("smtp")) { proto = Protocol.SMTP; port = 25; } else port = 7025; if (cl.hasOption("p")) port = Integer.valueOf(cl.getOptionValue("p")).intValue(); String[] recipients = cl.getOptionValues("r"); String sender = cl.getOptionValue("s"); boolean tracingEnabled = cl.hasOption("T"); int everyN; if (cl.hasOption("N")) { everyN = Integer.valueOf(cl.getOptionValue("N")).intValue(); } else { everyN = 100; } // Process files from the -d option. List<File> files = new ArrayList<File>(); if (cl.hasOption("d")) { File dir = new File(cl.getOptionValue("d")); if (!dir.isDirectory()) { System.err.format("%s is not a directory.\n", dir.getPath()); System.exit(1); } File[] fileArray = dir.listFiles(); if (fileArray == null || fileArray.length == 0) { System.err.format("No files found in directory %s.\n", dir.getPath()); } Collections.addAll(files, fileArray); } // Process files specified as arguments. for (String arg : cl.getArgs()) { files.add(new File(arg)); } // Validate file content. if (!cl.hasOption("noValidation")) { Iterator<File> i = files.iterator(); while (i.hasNext()) { InputStream in = null; File file = i.next(); boolean valid = false; try { in = new FileInputStream(file); if (FileUtil.isGzipped(file)) { in = new GZIPInputStream(in); } in = new BufferedInputStream(in); // Required for RFC 822 check if (!EmailUtil.isRfc822Message(in)) { System.err.format("%s does not contain a valid RFC 822 message.\n", file.getPath()); } else { valid = true; } } catch (IOException e) { System.err.format("Unable to validate %s: %s.\n", file.getPath(), e.toString()); } finally { ByteUtil.closeStream(in); } if (!valid) { i.remove(); } } } if (files.size() == 0) { System.err.println("No files to inject."); System.exit(1); } if (!quietMode) { System.out.format( "Injecting %d message(s) to %d recipient(s). Server %s, port %d, using %d thread(s).\n", files.size(), recipients.length, host, port, threads); } int totalFailed = 0; int totalSucceeded = 0; long startTime = System.currentTimeMillis(); boolean verbose = cl.hasOption("v"); boolean skipTLSCertValidation = cl.hasOption("skipTLSCertValidation"); LmtpInject injector = null; try { injector = new LmtpInject(threads, sender, recipients, files, host, port, proto, quietMode, tracingEnabled, verbose, skipTLSCertValidation); } catch (Exception e) { mLog.error("Unable to initialize LmtpInject", e); System.exit(1); } injector.setReportEvery(everyN); injector.markStartTime(); try { injector.run(); } catch (IOException e) { e.printStackTrace(); System.exit(1); } int succeeded = injector.getSuccessCount(); int failedThisTime = injector.getFailureCount(); long elapsedMS = System.currentTimeMillis() - startTime; double elapsed = elapsedMS / 1000.0; double msPerMsg = 0.0; double msgSizeKB = 0.0; if (succeeded > 0) { msPerMsg = elapsedMS; msPerMsg /= succeeded; msgSizeKB = injector.mFileSizeTotal / 1024.0; msgSizeKB /= succeeded; } double msgPerSec = ((double) succeeded / (double) elapsedMS) * 1000; if (!quietMode) { System.out.println(); System.out.printf( "LmtpInject Finished\n" + "submitted=%d failed=%d\n" + "%.2fs, %.2fms/msg, %.2fmsg/s\n" + "average message size = %.2fKB\n", succeeded, failedThisTime, elapsed, msPerMsg, msgPerSec, msgSizeKB); } totalFailed += failedThisTime; totalSucceeded += succeeded; if (totalFailed != 0) System.exit(1); }
From source file:edu.jhu.hlt.concrete.gigaword.expt.ConvertGigawordDocuments.java
/** * @param args//w w w . j ava 2 s.c o m */ public static void main(String... args) { Thread.setDefaultUncaughtExceptionHandler(new UncaughtExceptionHandler() { @Override public void uncaughtException(Thread t, Throwable e) { logger.error("Thread {} caught unhandled exception.", t.getName()); logger.error("Unhandled exception.", e); } }); if (args.length != 2) { logger.info("Usage: {} {} {}", GigawordConcreteConverter.class.getName(), "path/to/expt/file", "path/to/out/folder"); System.exit(1); } String exptPathStr = args[0]; String outPathStr = args[1]; // Verify path points to something. Path exptPath = Paths.get(exptPathStr); if (!Files.exists(exptPath)) { logger.error("File: {} does not exist. Re-run with the correct path to " + " the experiment 2 column file. See README.md."); System.exit(1); } logger.info("Experiment map located at: {}", exptPathStr); // Create output dir if not yet created. Path outPath = Paths.get(outPathStr); if (!Files.exists(outPath)) { logger.info("Creating directory: {}", outPath.toString()); try { Files.createDirectories(outPath); } catch (IOException e) { logger.error("Caught an IOException when creating output dir.", e); System.exit(1); } } logger.info("Output directory located at: {}", outPathStr); // Read in expt map. See README.md. Map<String, Set<String>> exptMap = null; try (Reader r = ExperimentUtils.createReader(exptPath); BufferedReader br = new BufferedReader(r)) { exptMap = ExperimentUtils.createFilenameToIdMap(br); } catch (IOException e) { logger.error("Caught an IOException when creating expt map.", e); System.exit(1); } // Start a timer. logger.info("Gigaword -> Concrete beginning."); StopWatch sw = new StopWatch(); sw.start(); // Iterate over expt map. exptMap.entrySet() // .parallelStream() .forEach(p -> { final String pathStr = p.getKey(); final Set<String> ids = p.getValue(); final Path lp = Paths.get(pathStr); logger.info("Converting path: {}", pathStr); // Get the file name and immediate folder it is under. int nElements = lp.getNameCount(); Path fileName = lp.getName(nElements - 1); Path subFolder = lp.getName(nElements - 2); String newFnStr = fileName.toString().split("\\.")[0] + ".tar"; // Mirror folders in output dir. Path localOutFolder = outPath.resolve(subFolder); Path localOutPath = localOutFolder.resolve(newFnStr); // Create output subfolders. if (!Files.exists(localOutFolder) && !Files.isDirectory(localOutFolder)) { logger.info("Creating out file: {}", localOutFolder.toString()); try { Files.createDirectories(localOutFolder); } catch (IOException e) { throw new RuntimeException("Caught an IOException when creating output dir.", e); } } // Iterate over communications. Iterator<Communication> citer; try (OutputStream os = Files.newOutputStream(localOutPath); BufferedOutputStream bos = new BufferedOutputStream(os); Archiver archiver = new TarArchiver(bos);) { citer = new ConcreteGigawordDocumentFactory().iterator(lp); while (citer.hasNext()) { Communication c = citer.next(); String cId = c.getId(); // Document ID must be in the set. Remove. boolean wasInSet = ids.remove(cId); if (!wasInSet) { // Some IDs are duplicated in Gigaword. // See ERRATA. logger.debug( "ID: {} was parsed from path: {}, but was not in the experiment map. Attempting to remove dupe.", cId, pathStr); // Attempt to create a duplicate id (append .duplicate to the id). // Then, try to remove again. String newId = RepairDuplicateIDs.repairDuplicate(cId); boolean dupeRemoved = ids.remove(newId); // There are not nested duplicates, so this should never fire. if (!dupeRemoved) { logger.info("Failed to remove dupe."); return; } else // Modify the communication ID to the unique version. c.setId(newId); } archiver.addEntry(new ArchivableCommunication(c)); } logger.info("Finished path: {}", pathStr); } catch (ConcreteException ex) { logger.error("Caught ConcreteException during Concrete mapping.", ex); logger.error("Path: {}", pathStr); } catch (IOException e) { logger.error("Error archiving communications.", e); logger.error("Path: {}", localOutPath.toString()); } }); sw.stop(); logger.info("Finished."); Minutes m = new Duration(sw.getTime()).toStandardMinutes(); logger.info("Runtime: Approximately {} minutes.", m.getMinutes()); }
From source file:ListAlgorithms.java
public static void main(String[] args) { Security.addProvider(new org.bouncycastle.jce.provider.BouncyCastleProvider()); Provider[] providers = Security.getProviders(); Set<String> ciphers = new HashSet<String>(); Set<String> keyAgreements = new HashSet<String>(); Set<String> macs = new HashSet<String>(); Set<String> messageDigests = new HashSet<String>(); Set<String> signatures = new HashSet<String>(); Set<String> keyFactory = new HashSet<String>(); Set<String> keyPairGenerator = new HashSet<String>(); Set<String> keyGenerator = new HashSet<String>(); for (int i = 0; i != providers.length; i++) { Iterator it = providers[i].keySet().iterator(); while (it.hasNext()) { String entry = (String) it.next(); if (entry.startsWith("Alg.Alias.")) { entry = entry.substring("Alg.Alias.".length()); }//from w ww. ja v a2s . co m if (entry.startsWith("Cipher.")) { ciphers.add(entry.substring("Cipher.".length())); } else if (entry.startsWith("KeyAgreement.")) { keyAgreements.add(entry.substring("KeyAgreement.".length())); } else if (entry.startsWith("Mac.")) { macs.add(entry.substring("Mac.".length())); } else if (entry.startsWith("MessageDigest.")) { messageDigests.add(entry.substring("MessageDigest.".length())); } else if (entry.startsWith("Signature.")) { signatures.add(entry.substring("Signature.".length())); } else if (entry.startsWith("KeyPairGenerator.")) { keyPairGenerator.add(entry.substring("KeyPairGenerator.".length())); } else if (entry.startsWith("KeyFactory.")) { keyFactory.add(entry.substring("KeyFactory.".length())); } else if (entry.startsWith("KeyGenerator.")) { keyGenerator.add(entry.substring("KeyGenerator.".length())); } else { System.out.println(entry); } } } printSet("KeyGenerator", keyGenerator); printSet("KeyFactory", keyFactory); printSet("KeyPairGenerator", keyPairGenerator); printSet("Ciphers", ciphers); printSet("KeyAgreeents", keyAgreements); printSet("Macs", macs); printSet("MessageDigests", messageDigests); printSet("Signatures", signatures); }
From source file:io.werval.cli.DamnSmallDevShell.java
public static void main(String[] args) { Options options = declareOptions();/*from w w w. j ava2 s .c o m*/ CommandLineParser parser = new PosixParser(); try { CommandLine cmd = parser.parse(options, args); // Handle --help if (cmd.hasOption("help")) { PrintWriter out = new PrintWriter(System.out); printHelp(options, out); out.flush(); System.exit(0); } // Handle --version if (cmd.hasOption("version")) { System.out.print(String.format( "Werval CLI v%s\n" + "Git commit: %s%s, built on: %s\n" + "Java version: %s, vendor: %s\n" + "Java home: %s\n" + "Default locale: %s, platform encoding: %s\n" + "OS name: %s, version: %s, arch: %s\n", VERSION, COMMIT, (DIRTY ? " (DIRTY)" : ""), DATE, System.getProperty("java.version"), System.getProperty("java.vendor"), System.getProperty("java.home"), Locale.getDefault().toString(), System.getProperty("file.encoding"), System.getProperty("os.name"), System.getProperty("os.version"), System.getProperty("os.arch"))); System.out.flush(); System.exit(0); } // Debug final boolean debug = cmd.hasOption('d'); // Temporary directory final File tmpDir = new File(cmd.getOptionValue('t', "build" + separator + "devshell.tmp")); if (debug) { System.out.println("Temporary directory set to '" + tmpDir.getAbsolutePath() + "'."); } // Handle commands @SuppressWarnings("unchecked") List<String> commands = cmd.getArgList(); if (commands.isEmpty()) { commands = Collections.singletonList("start"); } if (debug) { System.out.println("Commands to be executed: " + commands); } Iterator<String> commandsIterator = commands.iterator(); while (commandsIterator.hasNext()) { String command = commandsIterator.next(); switch (command) { case "new": System.out.println(LOGO); newCommand(commandsIterator.hasNext() ? commandsIterator.next() : "werval-application", cmd); break; case "clean": cleanCommand(debug, tmpDir); break; case "devshell": System.out.println(LOGO); devshellCommand(debug, tmpDir, cmd); break; case "start": System.out.println(LOGO); startCommand(debug, tmpDir, cmd); break; case "secret": secretCommand(); break; default: PrintWriter out = new PrintWriter(System.err); System.err.println("Unknown command: '" + command + "'"); printHelp(options, out); out.flush(); System.exit(1); break; } } } catch (IllegalArgumentException | ParseException | IOException ex) { PrintWriter out = new PrintWriter(System.err); printHelp(options, out); out.flush(); System.exit(1); } catch (WervalException ex) { ex.printStackTrace(System.err); System.err.flush(); System.exit(1); } }
From source file:json_cmp.Comparer.java
public static void main(String[] args) { System.out.println("Testing Begin"); try {/*from w w w.j a v a 2s.c om*/ String accessLogFolder = "/Users/herizhao/workspace/accessLog/"; // String yqlFileName = "json_cmp/test1.log"; // String yqlpFileName = "json_cmp/test2.log"; String yqlFileName = "tempLog/0812_yql.res"; String yqlpFileName = "tempLog/0812_yqlp.res"; ReadResults input1 = new ReadResults(accessLogFolder + yqlFileName); ReadResults input2 = new ReadResults(accessLogFolder + yqlpFileName); Integer diffNum = 0; Integer errorCount = 0; Integer totalIDNum1 = 0; Integer totalIDNum2 = 0; Integer equalIDwithDuplicate = 0; Integer beacons = 0; Integer lineNum = 0; Integer tempCount = 0; HashMap<String, IDclass> IDarray = new HashMap<String, IDclass>(); FileOutputStream fos = new FileOutputStream( "/Users/herizhao/workspace/accessLog/json_cmp/cmp_result.txt"); OutputStreamWriter osw = new OutputStreamWriter(fos); BufferedWriter bw = new BufferedWriter(osw); FileOutputStream consoleStream = new FileOutputStream( "/Users/herizhao/workspace/accessLog/json_cmp/console"); OutputStreamWriter consoleOSW = new OutputStreamWriter(consoleStream); BufferedWriter console = new BufferedWriter(consoleOSW); while (true) { input1.ReadNextLine(); if (input1.line == null) break; input2.ReadNextLine(); if (input2.line == null) break; while (input1.line.equals("")) { lineNum++; input1.ReadNextLine(); input2.ReadNextLine(); } if (input2.line == null) break; if (input1.line == null) break; lineNum++; System.out.println("lineNum = " + lineNum); String str1 = input1.line; String str2 = input2.line; ObjectMapper mapper1 = new ObjectMapper(); ObjectMapper mapper2 = new ObjectMapper(); JsonNode root1 = mapper1.readTree(str1); JsonNode root2 = mapper2.readTree(str2); JsonNode mediaNode1 = root1.path("query").path("results").path("mediaObj"); JsonNode mediaNode2 = root2.path("query").path("results").path("mediaObj"); if (mediaNode2.isMissingNode() && !mediaNode1.isMissingNode()) tempCount += mediaNode1.size(); //For yqlp if (mediaNode2.isArray()) { totalIDNum2 += mediaNode2.size(); for (int i = 0; i < mediaNode2.size(); i++) { ObjectNode mediaObj = (ObjectNode) mediaNode2.get(i); mediaObj.put("yvap", ""); JsonNode streamsNode = mediaObj.path("streams"); //streams if (streamsNode.isArray()) { for (int j = 0; j < streamsNode.size(); j++) { ObjectNode streamsObj = (ObjectNode) streamsNode.get(j); changeStreamsPath(streamsObj); ChangedHost(streamsObj); //if(streamsObj.path("h264_profile").isMissingNode()) streamsObj.put("h264_profile", ""); if (streamsObj.path("is_primary").isMissingNode()) streamsObj.put("is_primary", false); } } //meta if (!mediaObj.path("meta").isMissingNode()) { ObjectNode metaObj = (ObjectNode) mediaObj.path("meta"); changeMetaThumbnail(metaObj); if (metaObj.path("show_name").isMissingNode()) metaObj.put("show_name", ""); if (metaObj.path("event_start").isMissingNode()) metaObj.put("event_start", ""); if (metaObj.path("event_stop").isMissingNode()) metaObj.put("event_stop", ""); //if(metaObj.path("credits").path("label").isMissingNode()) ((ObjectNode) metaObj.path("credits")).put("label", ""); } //Metrics -> plidl & isrc changeMetrics(mediaObj); } } //For yql if (mediaNode1.isArray()) { totalIDNum1 += mediaNode1.size(); for (int i = 0; i < mediaNode1.size(); i++) { JsonNode mediaObj = mediaNode1.get(i); ((ObjectNode) mediaObj).put("yvap", ""); //Meta //System.out.println("meta: "); if (!mediaObj.path("meta").isMissingNode()) { ObjectNode metaObj = (ObjectNode) mediaObj.path("meta"); changeMetaThumbnail(metaObj); metaObj.put("event_start", ""); metaObj.put("event_stop", ""); FloatingtoInt(metaObj, "duration"); if (metaObj.path("show_name").isMissingNode()) metaObj.put("show_name", ""); //System.out.println("thub_dem: "); if (!metaObj.path("thumbnail_dimensions").isMissingNode()) { ObjectNode thub_demObj = (ObjectNode) metaObj.path("thumbnail_dimensions"); FloatingtoInt(thub_demObj, "height"); FloatingtoInt(thub_demObj, "width"); } ((ObjectNode) metaObj.path("credits")).put("label", ""); } //Visualseek //System.out.println("visualseek: "); if (!mediaObj.path("visualseek").isMissingNode()) { ObjectNode visualseekObj = (ObjectNode) mediaObj.path("visualseek"); FloatingtoInt(visualseekObj, "frequency"); FloatingtoInt(visualseekObj, "width"); FloatingtoInt(visualseekObj, "height"); //visualseek -> images, float to int JsonNode imagesNode = visualseekObj.path("images"); if (imagesNode.isArray()) { for (int j = 0; j < imagesNode.size(); j++) { ObjectNode imageObj = (ObjectNode) imagesNode.get(j); FloatingtoInt(imageObj, "start_index"); FloatingtoInt(imageObj, "count"); } } } //Streams //System.out.println("streams: "); JsonNode streamsNode = mediaObj.path("streams"); if (streamsNode.isArray()) { for (int j = 0; j < streamsNode.size(); j++) { ObjectNode streamsObj = (ObjectNode) streamsNode.get(j); FloatingtoInt(streamsObj, "height"); FloatingtoInt(streamsObj, "bitrate"); FloatingtoInt(streamsObj, "duration"); FloatingtoInt(streamsObj, "width"); changeStreamsPath(streamsObj); ChangedHost(streamsObj); // if(streamsObj.path("h264_profile").isMissingNode()) streamsObj.put("h264_profile", ""); if (streamsObj.path("is_primary").isMissingNode()) streamsObj.put("is_primary", false); } } //Metrics -> plidl & isrc changeMetrics(mediaObj); } } //Compare if (mediaNode2.isArray() && mediaNode1.isArray()) { for (int i = 0; i < mediaNode2.size() && i < mediaNode1.size(); i++) { JsonNode mediaObj1 = mediaNode1.get(i); JsonNode mediaObj2 = mediaNode2.get(i); if (!mediaObj1.equals(mediaObj2)) { if (!mediaObj1.path("id").toString().equals(mediaObj2.path("id").toString())) { errorCount++; } else { Integer IFdiffStreams = 0; Integer IFdiffMeta = 0; Integer IFdiffvisualseek = 0; Integer IFdiffMetrics = 0; Integer IFdifflicense = 0; Integer IFdiffclosedcaptions = 0; String statusCode = ""; MetaClass tempMeta = new MetaClass(); if (!mediaObj1.path("status").equals(mediaObj2.path("status"))) { JsonNode statusNode1 = mediaObj1.path("status"); JsonNode statusNode2 = mediaObj2.path("status"); if (statusNode2.path("code").toString().equals("\"100\"") || (statusNode1.path("code").toString().equals("\"400\"") && statusNode1.path("code").toString().equals("\"404\"")) || (statusNode1.path("code").toString().equals("\"200\"") && statusNode1.path("code").toString().equals("\"200\"")) || (statusNode1.path("code").toString().equals("\"200\"") && statusNode1.path("code").toString().equals("\"403\""))) statusCode = ""; else statusCode = "yql code: " + mediaObj1.path("status").toString() + " yqlp code:" + mediaObj2.path("status").toString(); } else {//Status code is 100 if (!mediaObj1.path("streams").equals(mediaObj2.path("streams"))) IFdiffStreams = 1; if (!tempMeta.CompareMeta(mediaObj1.path("meta"), mediaObj2.path("meta"), lineNum)) IFdiffMeta = 1; if (!mediaObj1.path("visualseek").equals(mediaObj2.path("visualseek"))) IFdiffvisualseek = 1; if (!mediaObj1.path("metrics").equals(mediaObj2.path("metrics"))) { IFdiffMetrics = 1; JsonNode metrics1 = mediaObj1.path("metrics"); JsonNode metrics2 = mediaObj2.path("metrics"); if (!metrics1.path("beacons").equals(metrics2.path("beacons"))) beacons++; } if (!mediaObj1.path("license").equals(mediaObj2.path("license"))) IFdifflicense = 1; if (!mediaObj1.path("closedcaptions").equals(mediaObj2.path("closedcaptions"))) IFdiffclosedcaptions = 1; } if (IFdiffStreams + IFdiffMeta + IFdiffvisualseek + IFdiffMetrics + IFdifflicense + IFdiffclosedcaptions != 0 || !statusCode.equals("")) { String ID_str = mediaObj1.path("id").toString(); if (!IDarray.containsKey(ID_str)) { IDclass temp_IDclass = new IDclass(ID_str); temp_IDclass.addNum(IFdiffStreams, IFdiffMeta, IFdiffvisualseek, IFdiffMetrics, IFdifflicense, IFdiffclosedcaptions, lineNum); if (!statusCode.equals("")) temp_IDclass.statusCode = statusCode; IDarray.put(ID_str, temp_IDclass); } else { IDarray.get(ID_str).addNum(IFdiffStreams, IFdiffMeta, IFdiffvisualseek, IFdiffMetrics, IFdifflicense, IFdiffclosedcaptions, lineNum); if (!statusCode.equals("")) IDarray.get(ID_str).statusCode = statusCode; } IDarray.get(ID_str).stream.CompareStream(IFdiffStreams, mediaObj1.path("streams"), mediaObj2.path("streams"), lineNum); if (!IDarray.get(ID_str).metaDone) { IDarray.get(ID_str).meta = tempMeta; IDarray.get(ID_str).metaDone = true; } } else equalIDwithDuplicate++; } } else equalIDwithDuplicate++; } } bw.flush(); console.flush(); } //while System.out.println("done"); bw.write("Different ID" + " " + "num "); bw.write(PrintStreamsTitle()); bw.write(PrintMetaTitle()); bw.write(PrintTitle()); bw.newLine(); Iterator<String> iter = IDarray.keySet().iterator(); while (iter.hasNext()) { String key = iter.next(); bw.write(key + " "); bw.write(IDarray.get(key).num.toString() + " "); bw.write(IDarray.get(key).stream.print()); bw.write(IDarray.get(key).meta.print()); bw.write(IDarray.get(key).print()); bw.newLine(); //System.out.println(key); } //System.out.println("different log num = " + diffNum); //System.out.println("same log num = " + sameLogNum); System.out.println("Different ID size = " + IDarray.size()); // System.out.println("streamEqual = " + streamEqual); // System.out.println("metaEqual = " + metaEqual); // System.out.println("metricsEqual = " + metricsEqual); // System.out.println("visualseekEqual = " + visualseekEqual); // System.out.println("licenseEqual = " + licenseEqual); // System.out.println("closedcaptionsEqualEqual = " + closedcaptionsEqual); System.out.println(tempCount); System.out.println("beacons = " + beacons); System.out.println("equalIDwithDuplicate = " + equalIDwithDuplicate); System.out.println("Total ID num yql (including duplicates) = " + totalIDNum1); System.out.println("Total ID num yqpl (including duplicates) = " + totalIDNum2); System.out.println("Error " + errorCount); bw.close(); console.close(); } catch (IOException e) { } }
From source file:com.cloud.utils.crypt.EncryptionSecretKeyChanger.java
public static void main(String[] args) { List<String> argsList = Arrays.asList(args); Iterator<String> iter = argsList.iterator(); String oldMSKey = null;/* w w w .j a v a 2 s. com*/ String oldDBKey = null; String newMSKey = null; String newDBKey = null; //Parse command-line args while (iter.hasNext()) { String arg = iter.next(); // Old MS Key if (arg.equals("-m")) { oldMSKey = iter.next(); } // Old DB Key if (arg.equals("-d")) { oldDBKey = iter.next(); } // New MS Key if (arg.equals("-n")) { newMSKey = iter.next(); } // New DB Key if (arg.equals("-e")) { newDBKey = iter.next(); } } if (oldMSKey == null || oldDBKey == null) { System.out.println("Existing MS secret key or DB secret key is not provided"); usage(); return; } if (newMSKey == null && newDBKey == null) { System.out.println("New MS secret key and DB secret are both not provided"); usage(); return; } final File dbPropsFile = PropertiesUtil.findConfigFile("db.properties"); final Properties dbProps; EncryptionSecretKeyChanger keyChanger = new EncryptionSecretKeyChanger(); StandardPBEStringEncryptor encryptor = new StandardPBEStringEncryptor(); keyChanger.initEncryptor(encryptor, oldMSKey); dbProps = new EncryptableProperties(encryptor); PropertiesConfiguration backupDBProps = null; System.out.println("Parsing db.properties file"); try { dbProps.load(new FileInputStream(dbPropsFile)); backupDBProps = new PropertiesConfiguration(dbPropsFile); } catch (FileNotFoundException e) { System.out.println("db.properties file not found while reading DB secret key" + e.getMessage()); } catch (IOException e) { System.out.println("Error while reading DB secret key from db.properties" + e.getMessage()); } catch (ConfigurationException e) { e.printStackTrace(); } String dbSecretKey = null; try { dbSecretKey = dbProps.getProperty("db.cloud.encrypt.secret"); } catch (EncryptionOperationNotPossibleException e) { System.out.println("Failed to decrypt existing DB secret key from db.properties. " + e.getMessage()); return; } if (!oldDBKey.equals(dbSecretKey)) { System.out.println("Incorrect MS Secret Key or DB Secret Key"); return; } System.out.println("Secret key provided matched the key in db.properties"); final String encryptionType = dbProps.getProperty("db.cloud.encryption.type"); if (newMSKey == null) { System.out.println("No change in MS Key. Skipping migrating db.properties"); } else { if (!keyChanger.migrateProperties(dbPropsFile, dbProps, newMSKey, newDBKey)) { System.out.println("Failed to update db.properties"); return; } else { //db.properties updated successfully if (encryptionType.equals("file")) { //update key file with new MS key try { FileWriter fwriter = new FileWriter(keyFile); BufferedWriter bwriter = new BufferedWriter(fwriter); bwriter.write(newMSKey); bwriter.close(); } catch (IOException e) { System.out.println("Failed to write new secret to file. Please update the file manually"); } } } } boolean success = false; if (newDBKey == null || newDBKey.equals(oldDBKey)) { System.out.println("No change in DB Secret Key. Skipping Data Migration"); } else { EncryptionSecretKeyChecker.initEncryptorForMigration(oldMSKey); try { success = keyChanger.migrateData(oldDBKey, newDBKey); } catch (Exception e) { System.out.println("Error during data migration"); e.printStackTrace(); success = false; } } if (success) { System.out.println("Successfully updated secret key(s)"); } else { System.out.println("Data Migration failed. Reverting db.properties"); //revert db.properties try { backupDBProps.save(); } catch (ConfigurationException e) { e.printStackTrace(); } if (encryptionType.equals("file")) { //revert secret key in file try { FileWriter fwriter = new FileWriter(keyFile); BufferedWriter bwriter = new BufferedWriter(fwriter); bwriter.write(oldMSKey); bwriter.close(); } catch (IOException e) { System.out.println("Failed to revert to old secret to file. Please update the file manually"); } } } }