List of usage examples for java.nio.file Paths get
public static Path get(URI uri)
From source file:com.sludev.mssqlapplylog.MSSQLApplyLogMain.java
public static void main(String[] args) { CommandLineParser parser = new DefaultParser(); Options options = new Options(); // Most of the following defaults should be changed in // the --conf or "conf.properties" file String sqlURL = null;/* w w w .j ava 2 s . co m*/ String sqlUser = null; String sqlPass = null; String sqlDb = null; String sqlHost = "127.0.0.1"; String backupDirStr = null; String laterThanStr = ""; String fullBackupPathStr = null; String fullBackupPatternStr = "(?:[\\w_-]+?)(\\d+)\\.bak"; String fullBackupDatePatternStr = "yyyyMMddHHmm"; String sqlProcessUser = null; String logBackupPatternStr = "(.*)\\.trn"; String logBackupDatePatternStr = "yyyyMMddHHmmss"; boolean doFullRestore = false; Boolean useLogFileLastMode = null; Boolean monitorLogBackupDir = null; options.addOption(Option.builder().longOpt("conf").desc("Configuration file.").hasArg().build()); options.addOption(Option.builder().longOpt("laterthan").desc("'Later Than' file filter.").hasArg().build()); options.addOption(Option.builder().longOpt("restore-full") .desc("Restore the full backup before continuing.").build()); options.addOption(Option.builder().longOpt("use-lastmod") .desc("Sort/filter the log backups using their File-System 'Last Modified' date.").build()); options.addOption(Option.builder().longOpt("monitor-backup-dir") .desc("Monitor the backup directory for new log backups, and apply them.").build()); CommandLine line = null; try { try { line = parser.parse(options, args); } catch (ParseException ex) { throw new MSSQLApplyLogException(String.format("Error parsing command line.'%s'", ex.getMessage()), ex); } String confFile = null; // Process the command line arguments Iterator cmdI = line.iterator(); while (cmdI.hasNext()) { Option currOpt = (Option) cmdI.next(); String currOptName = currOpt.getLongOpt(); switch (currOptName) { case "conf": // Parse the configuration file confFile = currOpt.getValue(); break; case "laterthan": // "Later Than" file date filter laterThanStr = currOpt.getValue(); break; case "restore-full": // Do a full backup restore before restoring logs doFullRestore = true; break; case "monitor-backup-dir": // Monitor the backup directory for new logs monitorLogBackupDir = true; break; case "use-lastmod": // Use the last-modified date on Log Backup files for sorting/filtering useLogFileLastMode = true; break; } } Properties confProperties = null; if (StringUtils.isBlank(confFile) || Files.isReadable(Paths.get(confFile)) == false) { throw new MSSQLApplyLogException( "Missing or unreadable configuration file. Please specify --conf"); } else { // Process the conf.properties file confProperties = new Properties(); try { confProperties.load(Files.newBufferedReader(Paths.get(confFile))); } catch (IOException ex) { throw new MSSQLApplyLogException("Error loading properties file", ex); } sqlURL = confProperties.getProperty("sqlURL", ""); sqlUser = confProperties.getProperty("sqlUser", ""); sqlPass = confProperties.getProperty("sqlPass", ""); sqlDb = confProperties.getProperty("sqlDb", ""); sqlHost = confProperties.getProperty("sqlHost", ""); backupDirStr = confProperties.getProperty("backupDir", ""); if (StringUtils.isBlank(laterThanStr)) { laterThanStr = confProperties.getProperty("laterThan", ""); } fullBackupPathStr = confProperties.getProperty("fullBackupPath", fullBackupPathStr); fullBackupPatternStr = confProperties.getProperty("fullBackupPattern", fullBackupPatternStr); fullBackupDatePatternStr = confProperties.getProperty("fullBackupDatePattern", fullBackupDatePatternStr); sqlProcessUser = confProperties.getProperty("sqlProcessUser", ""); logBackupPatternStr = confProperties.getProperty("logBackupPattern", logBackupPatternStr); logBackupDatePatternStr = confProperties.getProperty("logBackupDatePattern", logBackupDatePatternStr); if (useLogFileLastMode == null) { String useLogFileLastModeStr = confProperties.getProperty("useLogFileLastMode", "false"); useLogFileLastMode = Boolean .valueOf(StringUtils.lowerCase(StringUtils.trim(useLogFileLastModeStr))); } if (monitorLogBackupDir == null) { String monitorBackupDirStr = confProperties.getProperty("monitorBackupDir", "false"); monitorLogBackupDir = Boolean .valueOf(StringUtils.lowerCase(StringUtils.trim(monitorBackupDirStr))); } } } catch (MSSQLApplyLogException ex) { try (StringWriter sw = new StringWriter(); PrintWriter pw = new PrintWriter(sw)) { pw.append(String.format("Error : '%s'\n\n", ex.getMessage())); HelpFormatter formatter = new HelpFormatter(); formatter.printHelp(pw, 80, "\njava -jar mssqlapplylog.jar ", "\nThe MSSQLApplyLog application can be used in a variety of options and modes.\n", options, 0, 2, " All Rights Reserved.", true); System.out.println(sw.toString()); } catch (IOException iex) { LOGGER.debug("Error processing usage", iex); } System.exit(1); } MSSQLApplyLogConfig config = MSSQLApplyLogConfig.from(backupDirStr, fullBackupPathStr, fullBackupDatePatternStr, laterThanStr, fullBackupPatternStr, logBackupPatternStr, logBackupDatePatternStr, sqlHost, sqlDb, sqlUser, sqlPass, sqlURL, sqlProcessUser, useLogFileLastMode, doFullRestore, monitorLogBackupDir); MSSQLApplyLog logProc = MSSQLApplyLog.from(config); BasicThreadFactory thFactory = new BasicThreadFactory.Builder().namingPattern("restoreThread-%d").build(); ExecutorService mainThreadExe = Executors.newSingleThreadExecutor(thFactory); Future<Integer> currRunTask = mainThreadExe.submit(logProc); mainThreadExe.shutdown(); Integer resp = 0; try { resp = currRunTask.get(); } catch (InterruptedException ex) { LOGGER.error("Application 'main' thread was interrupted", ex); } catch (ExecutionException ex) { LOGGER.error("Application 'main' thread execution error", ex); } finally { // If main leaves for any reason, shutdown all threads mainThreadExe.shutdownNow(); } System.exit(resp); }
From source file:com.google.cloud.runtimes.builder.Application.java
/** * Main method for invocation from the command line. Handles parsing of command line options. *//*from w ww. j a va 2 s . co m*/ public static void main(String[] args) throws BuildStepException, IOException { addCliOptions(CLI_OPTIONS); CommandLine cmd = parse(args); String[] jdkMappings = cmd.getOptionValues("j"); String[] serverMappings = cmd.getOptionValues("s"); String compatImage = cmd.getOptionValue("c"); String mavenImage = cmd.getOptionValue("m"); String gradleImage = cmd.getOptionValue("g"); boolean disableSourceBuild = cmd.hasOption("n"); Injector injector = Guice .createInjector(new RootModule(mergeSettingsWithDefaults(jdkMappings, serverMappings), compatImage == null ? DEFAULT_COMPAT_RUNTIME_IMAGE : compatImage, mavenImage == null ? DEFAULT_MAVEN_DOCKER_IMAGE : mavenImage, gradleImage == null ? DEFAULT_GRADLE_DOCKER_IMAGE : gradleImage, disableSourceBuild, getAppYamlOverrideSettings(cmd))); // Perform dependency injection and run the application Path workspaceDir = Paths.get(System.getProperty("user.dir")); injector.getInstance(BuildPipelineConfigurator.class).generateDockerResources(workspaceDir); }
From source file:ch.bfh.evoting.verifier.Verifier.java
/** * This program is a verifier for MobiVote application with HKRS12 protocol * You have to indicate the XML file exported from the application as argument for this program. * You can also indicate the files of different participants. In this case, there will be checked if all files contain the same values. * @param args the file(s) containing the values of the protocol *//*from ww w. ja v a 2 s. c o m*/ public static void main(String[] args) { /* * Reading files given as parameters */ if (args.length < 1) { //Not enough arguments System.out.println( "You must indicate the location of the XML file containing the data to verify! You can also indicate the files of different participants."); return; } else if (args.length > 1) { //Make a digest of all files and compare them List<byte[]> digests = new ArrayList<byte[]>(); for (String s : args) { try { MessageDigest md = MessageDigest.getInstance("SHA-256"); InputStream is = Files.newInputStream(Paths.get(s)); DigestInputStream dis = new DigestInputStream(is, md); @SuppressWarnings("unused") int numBytes = -1; while ((numBytes = dis.read()) != -1) { dis.read(); } byte[] digest = md.digest(); digests.add(digest); } catch (IOException e) { System.out.println("One of the given files does not exists!"); return; } catch (NoSuchAlgorithmException e1) { System.out.println("Unable to compute the digest of the file!"); return; } } System.out.print("Checking the similarity of the files..."); for (byte[] b : digests) { for (byte[] b2 : digests) { if (!Arrays.equals(b, b2)) { System.out.print("\t\t\t\t\t\t\t\t\t\tWRONG\n"); System.out.println( "The given files are not all identical! This means the content received by the participants was not the same."); return; } } } System.out.print("\t\t\t\t\t\t\t\t\t\tOK\n\n"); } else if (args.length == 1) { //Print a warning System.out.println( "BE CAREFUL: checking only one file ensure that the cryptographic values of the protocol are correct.\n" + "But, it does NOT ensure that all users have the same result. This could happen when a participant missed some messages.\n" + "This case is only covered by this verifier by giving multiple files as argument.\n"); } Serializer serializer = new Persister(); File source = new File(args[0]); poll = null; System.out.print("Reading file..."); try { poll = serializer.read(XMLPoll.class, source); System.out.print("\t\t\t\t\t\t\t\t\t\t\t\t\tOK\n"); } catch (Exception e) { System.out.print("\t\t\t\t\t\t\t\t\t\t\t\t\tFAILED\n"); e.printStackTrace(); return; } /* * Create the initializations needed by the protocol */ G_q = GStarModSafePrime.getInstance(new BigInteger(poll.getP())); Z_q = G_q.getZModOrder(); Zgroup = Z.getInstance(); generator = poll.getGenerator().getValue(G_q); representations = new Element[poll.getOptions().size()]; int i = 0; for (XMLOption op : poll.getOptions()) { representations[i] = op.getRepresentation().getValue(Z_q); i++; } System.out.println(); System.out.println("Verifying vote: \"" + poll.getQuestion() + "\""); /* * Verify the dependence between cryptographic values and the vote properties */ //Not used anymore, implicitly done with otherInputs of proofs //verifyDependencyTextCrypto(); /* * Verify the proofs for the user */ System.out.println(); System.out.println("Verifying the proof for the participants..."); a = new Element[poll.getParticipants().size()]; b = new Element[poll.getParticipants().size()]; h = new Element[poll.getParticipants().size()]; hHat = new Element[poll.getParticipants().size()]; hHatPowX = new Element[poll.getParticipants().size()]; proofForX = new Element[poll.getParticipants().size()]; validityProof = new Element[poll.getParticipants().size()]; equalityProof = new Element[poll.getParticipants().size()]; Tuple pollTuple = preparePollOtherInput(poll, representations, generator); int k = 0; for (XMLParticipant p : poll.getParticipants()) { System.out.println("\tParticipant " + p.getIdentification() + " (" + p.getUniqueId() + ")"); if (p.getAi() != null) { a[k] = p.getAi().getValue(G_q); if (DEBUG) System.out.println("Value a: " + a[k]); } if (p.getHi() != null) { h[k] = p.getHi().getValue(G_q); if (DEBUG) System.out.println("Value h: " + h[k]); } if (p.getBi() != null) { b[k] = p.getBi().getValue(G_q); if (DEBUG) System.out.println("Value b: " + b[k]); } if (p.getHiHat() != null) { hHat[k] = p.getHiHat().getValue(G_q); if (DEBUG) System.out.println("Value h hat: " + hHat[k]); } if (p.getHiHatPowXi() != null) { hHatPowX[k] = p.getHiHatPowXi().getValue(G_q); if (DEBUG) System.out.println("Value h hat ^ x: " + hHatPowX[k]); } if (p.getProofForXi() != null) { ProductGroup group = ProductGroup.getInstance(G_q, Zgroup, Z_q); proofForX[k] = p.getProofForXi().getValue(group); } if (p.getProofValidVote() != null) { Group[] tripleElement1Groups = new Group[poll.getOptions().size()]; Group[] tripleElement2Groups = new Group[poll.getOptions().size()]; Group[] tripleElement3Groups = new Group[poll.getOptions().size()]; for (i = 0; i < poll.getOptions().size(); i++) { tripleElement1Groups[i] = ProductGroup.getInstance(G_q, G_q); tripleElement2Groups[i] = Zgroup; tripleElement3Groups[i] = Z_q; } ProductGroup tripleElement1 = ProductGroup.getInstance(tripleElement1Groups); ProductGroup tripleElement2 = ProductGroup.getInstance(tripleElement2Groups); ProductGroup tripleElement3 = ProductGroup.getInstance(tripleElement3Groups); ProductGroup group = ProductGroup.getInstance(tripleElement1, tripleElement2, tripleElement3); validityProof[k] = p.getProofValidVote().getValue(group); if (DEBUG) System.out.println("Proof of validity: " + validityProof[k]); } if (p.getProofForHiHat() != null) { recoveryNeeded = true; ProductGroup group = ProductGroup.getInstance(ProductGroup.getInstance(G_q, G_q), Zgroup, Z_q); equalityProof[k] = p.getProofForHiHat().getValue(group); } Tuple participantTuple = prepareParticipantOtherInput(p); otherInput = Tuple.getInstance(participantTuple, pollTuple); /* * Verify proof of knowledge */ System.out.print("\t\tVerifying proof of knowledge of x value..."); verifyProofOfKnowledgeOfX(k); /* * Verify proof of validity */ if (validityProof[k] != null) { System.out.print("\t\tVerifying proof of valid vote..."); verifyValidityProof(k); } /* * Verify proof of equality */ if (equalityProof[k] != null) { System.out.print("\t\tVerifying proof of equality between discrete logarithm g^x and h_hat^x..."); verifyEqualityProof(k); } k++; } /** * Computing the result */ System.out.println(); System.out.print("Computing the result..."); computeResult(); }
From source file:com.chenshu.compress.CompressSizeTest.java
public static void main(String[] args) throws Exception { byte[] data = Files.readAllBytes(Paths.get("./THIRDPARTYLICENSEREADME.txt")); for (int level = 1; level < 10; level++) { System.out.println("JDK Gzip\t" + level + " " + compress(new JdkGzipCompress(level), data)); System.out.println("COMMONS Gzip\t" + level + " " + compress(new CommonsGzipCompress(level), data)); System.out.println("COMMONS BZip2\t" + level + " " + compress(new CommonsBZip2Compress(level), data)); System.out/* www .jav a2s .c o m*/ .println("COMMONS Deflate\t" + level + " " + compress(new CommonsDeflateCompress(level), data)); } }
From source file:de.kaixo.mubi.lists.store.MubiListsToElasticSearchLoader.java
public static void main(String arg[]) { if (arg.length < 1) { logger.error("Usage: MubiListsToElasticSearchLoader <directory>"); System.exit(1);/*from w ww . j a va2 s. c o m*/ } Path path = Paths.get(arg[0]); if (!Files.isDirectory(path)) { logger.error("Does not exist or is not a directory: " + path.toString()); System.exit(1); } MubiListsToElasticSearchLoader loader = new MubiListsToElasticSearchLoader(); if (loader.prepareIndex(true)) loader.loadAllFromDir(path); loader.close(); }
From source file:edu.jhu.hlt.concrete.ingesters.simple.DoubleLineBreakFileIngester.java
/** * See usage string.//from ww w . jav a2 s . c o m * * @param args */ public static void main(String[] args) { if (args.length != 4) { System.err.println("This program converts a character-based file to a .concrete file."); System.err.println("The text file must contain UTF-8 encoded characters."); System.err.println( "If the file contains any double-newlines, the file will be split into sections where those double-newlines occur."); System.err.println( "The .concrete file will share the same name as the input file, including the extension."); System.err.println("This program takes 4 arguments."); System.err.println("Argument 1: path/to/a/character/based/file"); System.err.println("Argument 2: type of Communication to generate [e.g., tweet]"); System.err.println("Argument 3: type of Sections to generate [e.g., passage]"); System.err.println("Argument 4: path/to/out/concrete/file"); System.err.println("Example usage: " + CompleteFileIngester.class.getName() + " /my/text/file story passage /my/output/folder"); System.exit(1); } String inPathStr = args[0]; Path inPath = Paths.get(inPathStr); try { ExistingNonDirectoryFile ef = new ExistingNonDirectoryFile(inPath); Optional<String> commType = Optional.ofNullable(args[1]); Optional<String> sectionType = Optional.ofNullable(args[2]); Optional<String> outPathStr = Optional.ofNullable(args[3]); Path ep = ef.getPath(); String fn = ef.getName(); Path outPath = Paths.get(outPathStr.get()); Path outFile = outPath.resolve(fn + ".concrete"); // Output directory exists, or it doesn't. // Try to create if it does not. if (!Files.exists(outPath)) { try { Files.createDirectories(outPath); } catch (IOException e) { logger.error("Caught exception when making output directories.", e); } // if it does, check to make sure it's a directory. } else { if (!Files.isDirectory(outPath)) { logger.error("Output path exists but is not a directory."); System.exit(1); } else { // check to make sure the output file won't be overwritten. if (Files.exists(outFile)) { logger.warn("Output file {} exists; not overwriting.", outFile.toString()); System.exit(1); } } } try { UTF8FileIngester ing = new DoubleLineBreakFileIngester(commType.get(), sectionType.get()); Communication comm = ing.fromCharacterBasedFile(ep); new WritableCommunication(comm).writeToFile(outFile, false); } catch (IngestException e) { logger.error("Caught exception during ingest.", e); System.exit(1); } catch (ConcreteException e) { logger.error("Caught exception writing output.", e); } } catch (NoSuchFileException e) { logger.error("Path {} does not exist.", inPathStr); System.exit(1); } catch (NotFileException e) { logger.error("Path {} is a directory.", inPathStr); System.exit(1); } }
From source file:lucene.IndexFiles.java
/** Index all text files under a directory. */ public static void main(String[] args) { String usage = "java org.apache.lucene.demo.IndexFiles" + " [-index INDEX_PATH] [-docs DOCS_PATH] [-update]\n\n" + "This indexes the documents in DOCS_PATH, creating a Lucene index" + "in INDEX_PATH that can be searched with SearchFiles"; String indexPath = "E:/index26"; // is DFR// ww w. j a v a 2 s .c o m //2 is normal //3 is ib with h3 //4 is ib with h2 with porter stemmer //5 is ib with h2 with s stemmer //6 is ib with h2 without stemmer //7 is without stemmer without <p //8 is basic with all tags //9 is ib with h2 and stopwords without stemmer //10 like without ib, lower tags //11 like 10 with lower tags p, br and hr //12 like 11 with tags closed //13 is closed tags with punctuation replace and whitespace tokenizer with hyphen cared for //14 std tokenizer with hyphen taken cared for with stemmer //15 like 14 without stemming //16 like 15 with LMD //17 like 11 with LMD //18 with count of lower and upper delimiters of split //19 like 18 with (?i) to ignore case in all and valipas > 9 //20 with (?i) in all //21 is fresh 19 //22 is legalspans with LMD //23 is fresh 19 without 0 pass //24 is legalspans with InB2 //25 is 23 //26 is 25 with s stemmer and 0 //27 is legalspans demo of 50 passages //28 is modified legal span and fast //29 is 28 with s-stemming //30 is 28 with porter stemming String docsPath = "E:/documents/text"; boolean create = true; for (int i = 0; i < args.length; i++) { if ("-index".equals(args[i])) { indexPath = args[i + 1]; i++; } else if ("-docs".equals(args[i])) { docsPath = args[i + 1]; i++; } else if ("-update".equals(args[i])) { create = false; } } if (docsPath == null) { System.err.println("Usage: " + usage); System.exit(1); } final Path docDir = Paths.get(docsPath); if (!Files.isReadable(docDir)) { System.out.println("Document directory dhfndk '" + docDir.toAbsolutePath() + "' does not exist or is not readable, please check the path"); System.exit(1); } Date start = new Date(); try { System.out.println("Indexing to directory '" + indexPath + "'..."); Directory dir = FSDirectory.open(Paths.get(indexPath)); //Analyzer analyzer = new StandardAnalyzer(); //IndexWriterConfig iwc = new IndexWriterConfig(analyzer); StandardAnalyzer analyzer = new StandardAnalyzer(); //Directory dir = new RAMDirectory(); IndexWriterConfig iwc = new IndexWriterConfig(analyzer); /*IBSimilarity similarity = new IBSimilarity( new DistributionLL(),//1 //new DistributionSPL(),//2 new LambdaDF(),//1 //new LambdaTTF(), //2 new NormalizationH2());*/ /*DFRSimilarity similarity = new DFRSimilarity( ///////INB2 Similarity new BasicModelIn(), new AfterEffectL(), new NormalizationH1());*/ LMDirichletSimilarity similarity = new LMDirichletSimilarity();//////// LMD Model iwc.setSimilarity(similarity); IndexWriter writer = new IndexWriter(dir, iwc); if (create) { // Create a new index in the directory, removing any // previously indexed documents: iwc.setOpenMode(OpenMode.CREATE); } else { // Add new documents to an existing index: iwc.setOpenMode(OpenMode.CREATE_OR_APPEND); } System.out.println("Test 1"); // Optional: for better indexing performance, if you // are indexing many documents, increase the RAM // buffer. But if you do this, increase the max heap // size to the JVM (eg add -Xmx512m or -Xmx1g): // // iwc.setRAMBufferSizeMB(256.0); //IndexWriter writer = new IndexWriter(dir, iwc); System.out.println("Test 2"); indexDocs(writer, docDir); System.out.println("Test 3"); // NOTE: if you want to maximize search performance, // you can optionally call forceMerge here. This can be // a terribly costly operation, so generally it's only // worth it when your index is relatively static (ie // you're done adding documents to it): // // writer.forceMerge(1); writer.close(); Date end = new Date(); System.out.println(end.getTime() - start.getTime() + " total milliseconds"); } catch (IOException e) { System.out.println(" caught a " + e.getClass() + "\n with message: " + e.getMessage()); } }
From source file:cloudeventbus.cli.Certs.java
public static void main(String[] args) throws Exception { final JCommander commander = new JCommander(); final DefaultOptions options = new DefaultOptions(); final CreateAuthorityCommand createAuthorityCommand = new CreateAuthorityCommand(); final CreateClientCommand createClientCommand = new CreateClientCommand(); final CreateServerCommand createServerCommand = new CreateServerCommand(); final ChainCertificateCommand chainCertificateCommand = new ChainCertificateCommand(); final ShowCertificateCommand showCertificateCommand = new ShowCertificateCommand(); final ImportCertificatesCommand importCertificatesCommand = new ImportCertificatesCommand(); final RemoveAuthorityCommand removeAuthorityCommand = new RemoveAuthorityCommand(); final ValidateCommand validateCommand = new ValidateCommand(); commander.addObject(options);// w w w .j ava 2 s. c om commander.addCommand(createAuthorityCommand); commander.addCommand(createClientCommand); commander.addCommand(createServerCommand); commander.addCommand(chainCertificateCommand); commander.addCommand(new ListAuthorities()); commander.addCommand(showCertificateCommand); commander.addCommand(importCertificatesCommand); commander.addCommand(removeAuthorityCommand); commander.addCommand(validateCommand); commander.setProgramName("eventbus-certs"); try { commander.parse(args); final String command = commander.getParsedCommand(); if (command == null) { commander.usage(); } else { final TrustStore trustStore = CertificateUtils.loadTrustStore(options.trustStore); switch (command) { case CREATE_AUTHORITY: { final KeyPair keyPair = CertificateUtils.generateKeyPair(); CertificateUtils.savePrivateKey(keyPair.getPrivate(), createAuthorityCommand.privateKey); final Certificate certificate = CertificateUtils.generateSelfSignedCertificate(keyPair, getExpirationDate(createAuthorityCommand.expirationDate), Subject.list(createAuthorityCommand.subscribePermissions), Subject.list(createAuthorityCommand.publishPermissions), createAuthorityCommand.comment); trustStore.add(certificate); CertificateUtils.saveCertificates(options.trustStore, trustStore); System.out.println("Created authority certificate."); break; } case LIST_AUTHORITIES: { displayCertificates(trustStore); break; } case CREATE_CLIENT: createCertificate(trustStore, Certificate.Type.CLIENT, createClientCommand); break; case CREATE_SERVER: createCertificate(trustStore, Certificate.Type.SERVER, createServerCommand); break; case SHOW_CERTIFICATE: { final CertificateChain certificates = CertificateUtils .loadCertificateChain(showCertificateCommand.certificate); displayCertificates(certificates); break; } case CHAIN_CERTIFICATE: chainCertificate(chainCertificateCommand); break; case VALIDATE_CERTIFICATE: { final CertificateChain certificates = CertificateUtils .loadCertificateChain(validateCommand.certificate); trustStore.validateCertificateChain(certificates); System.out.println(validateCommand.certificate + " is valid."); break; } case IMPORT_CERTIFICATES: { final Path path = Paths.get(importCertificatesCommand.certificate); try (final InputStream fileIn = Files.newInputStream(path); final InputStream in = new Base64InputStream(fileIn)) { final Collection<Certificate> certificates = new ArrayList<>(); CertificateStoreLoader.load(in, certificates); for (Certificate certificate : certificates) { trustStore.add(certificate); } CertificateUtils.saveCertificates(options.trustStore, trustStore); } break; } case REMOVE_AUTHORITY: if (trustStore.remove(removeAuthorityCommand.serialNumber)) { System.err.println("Removed certificate from trust store."); CertificateUtils.saveCertificates(options.trustStore, trustStore); } else { System.err.println("Certificate with serial number " + removeAuthorityCommand.serialNumber + " not found."); } } } } catch (ParameterException e) { System.err.println(e.getMessage()); commander.usage(commander.getParsedCommand()); System.exit(1); } }
From source file:io.anserini.index.IndexTweets.java
@SuppressWarnings("static-access") public static void main(String[] args) throws Exception { Options options = new Options(); options.addOption(new Option(HELP_OPTION, "show help")); options.addOption(new Option(OPTIMIZE_OPTION, "merge indexes into a single segment")); options.addOption(new Option(STORE_TERM_VECTORS_OPTION, "store term vectors")); options.addOption(OptionBuilder.withArgName("dir").hasArg().withDescription("source collection directory") .create(COLLECTION_OPTION)); options.addOption(//from w ww .ja v a 2 s .c o m OptionBuilder.withArgName("dir").hasArg().withDescription("index location").create(INDEX_OPTION)); options.addOption(OptionBuilder.withArgName("file").hasArg().withDescription("file with deleted tweetids") .create(DELETES_OPTION)); options.addOption(OptionBuilder.withArgName("id").hasArg().withDescription("max id").create(MAX_ID_OPTION)); CommandLine cmdline = null; CommandLineParser parser = new GnuParser(); try { cmdline = parser.parse(options, args); } catch (ParseException exp) { System.err.println("Error parsing command line: " + exp.getMessage()); System.exit(-1); } if (cmdline.hasOption(HELP_OPTION) || !cmdline.hasOption(COLLECTION_OPTION) || !cmdline.hasOption(INDEX_OPTION)) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp(IndexTweets.class.getName(), options); System.exit(-1); } String collectionPath = cmdline.getOptionValue(COLLECTION_OPTION); String indexPath = cmdline.getOptionValue(INDEX_OPTION); final FieldType textOptions = new FieldType(); textOptions.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); textOptions.setStored(true); textOptions.setTokenized(true); if (cmdline.hasOption(STORE_TERM_VECTORS_OPTION)) { textOptions.setStoreTermVectors(true); } LOG.info("collection: " + collectionPath); LOG.info("index: " + indexPath); LongOpenHashSet deletes = null; if (cmdline.hasOption(DELETES_OPTION)) { deletes = new LongOpenHashSet(); File deletesFile = new File(cmdline.getOptionValue(DELETES_OPTION)); if (!deletesFile.exists()) { System.err.println("Error: " + deletesFile + " does not exist!"); System.exit(-1); } LOG.info("Reading deletes from " + deletesFile); FileInputStream fin = new FileInputStream(deletesFile); byte[] ignoreBytes = new byte[2]; fin.read(ignoreBytes); // "B", "Z" bytes from commandline tools BufferedReader br = new BufferedReader(new InputStreamReader(new CBZip2InputStream(fin))); String s; while ((s = br.readLine()) != null) { if (s.contains("\t")) { deletes.add(Long.parseLong(s.split("\t")[0])); } else { deletes.add(Long.parseLong(s)); } } br.close(); fin.close(); LOG.info("Read " + deletes.size() + " tweetids from deletes file."); } long maxId = Long.MAX_VALUE; if (cmdline.hasOption(MAX_ID_OPTION)) { maxId = Long.parseLong(cmdline.getOptionValue(MAX_ID_OPTION)); LOG.info("index: " + maxId); } long startTime = System.currentTimeMillis(); File file = new File(collectionPath); if (!file.exists()) { System.err.println("Error: " + file + " does not exist!"); System.exit(-1); } StatusStream stream = new JsonStatusCorpusReader(file); Directory dir = FSDirectory.open(Paths.get(indexPath)); final IndexWriterConfig config = new IndexWriterConfig(ANALYZER); config.setOpenMode(IndexWriterConfig.OpenMode.CREATE); IndexWriter writer = new IndexWriter(dir, config); int cnt = 0; Status status; try { while ((status = stream.next()) != null) { if (status.getText() == null) { continue; } // Skip deletes tweetids. if (deletes != null && deletes.contains(status.getId())) { continue; } if (status.getId() > maxId) { continue; } cnt++; Document doc = new Document(); doc.add(new LongPoint(StatusField.ID.name, status.getId())); doc.add(new StoredField(StatusField.ID.name, status.getId())); doc.add(new LongPoint(StatusField.EPOCH.name, status.getEpoch())); doc.add(new StoredField(StatusField.EPOCH.name, status.getEpoch())); doc.add(new TextField(StatusField.SCREEN_NAME.name, status.getScreenname(), Store.YES)); doc.add(new Field(StatusField.TEXT.name, status.getText(), textOptions)); doc.add(new IntPoint(StatusField.FRIENDS_COUNT.name, status.getFollowersCount())); doc.add(new StoredField(StatusField.FRIENDS_COUNT.name, status.getFollowersCount())); doc.add(new IntPoint(StatusField.FOLLOWERS_COUNT.name, status.getFriendsCount())); doc.add(new StoredField(StatusField.FOLLOWERS_COUNT.name, status.getFriendsCount())); doc.add(new IntPoint(StatusField.STATUSES_COUNT.name, status.getStatusesCount())); doc.add(new StoredField(StatusField.STATUSES_COUNT.name, status.getStatusesCount())); long inReplyToStatusId = status.getInReplyToStatusId(); if (inReplyToStatusId > 0) { doc.add(new LongPoint(StatusField.IN_REPLY_TO_STATUS_ID.name, inReplyToStatusId)); doc.add(new StoredField(StatusField.IN_REPLY_TO_STATUS_ID.name, inReplyToStatusId)); doc.add(new LongPoint(StatusField.IN_REPLY_TO_USER_ID.name, status.getInReplyToUserId())); doc.add(new StoredField(StatusField.IN_REPLY_TO_USER_ID.name, status.getInReplyToUserId())); } String lang = status.getLang(); if (!lang.equals("unknown")) { doc.add(new TextField(StatusField.LANG.name, status.getLang(), Store.YES)); } long retweetStatusId = status.getRetweetedStatusId(); if (retweetStatusId > 0) { doc.add(new LongPoint(StatusField.RETWEETED_STATUS_ID.name, retweetStatusId)); doc.add(new StoredField(StatusField.RETWEETED_STATUS_ID.name, retweetStatusId)); doc.add(new LongPoint(StatusField.RETWEETED_USER_ID.name, status.getRetweetedUserId())); doc.add(new StoredField(StatusField.RETWEETED_USER_ID.name, status.getRetweetedUserId())); doc.add(new IntPoint(StatusField.RETWEET_COUNT.name, status.getRetweetCount())); doc.add(new StoredField(StatusField.RETWEET_COUNT.name, status.getRetweetCount())); if (status.getRetweetCount() < 0 || status.getRetweetedStatusId() < 0) { LOG.warn("Error parsing retweet fields of " + status.getId()); } } writer.addDocument(doc); if (cnt % 100000 == 0) { LOG.info(cnt + " statuses indexed"); } } LOG.info(String.format("Total of %s statuses added", cnt)); if (cmdline.hasOption(OPTIMIZE_OPTION)) { LOG.info("Merging segments..."); writer.forceMerge(1); LOG.info("Done!"); } LOG.info("Total elapsed time: " + (System.currentTimeMillis() - startTime) + "ms"); } catch (Exception e) { e.printStackTrace(); } finally { writer.close(); dir.close(); stream.close(); } }
From source file:io.anserini.index.IndexTweetsUpdatePlace.java
@SuppressWarnings("static-access") public static void main(String[] args) throws Exception { Options options = new Options(); options.addOption(new Option(HELP_OPTION, "show help")); options.addOption(new Option(OPTIMIZE_OPTION, "merge indexes into a single segment")); options.addOption(new Option(STORE_TERM_VECTORS_OPTION, "store term vectors")); options.addOption(OptionBuilder.withArgName("collection").hasArg() .withDescription("source collection directory").create(COLLECTION_OPTION)); options.addOption(// w ww .j a va2 s . c o m OptionBuilder.withArgName("dir").hasArg().withDescription("index location").create(INDEX_OPTION)); options.addOption(OptionBuilder.withArgName("file").hasArg().withDescription("file with deleted tweetids") .create(DELETES_OPTION)); options.addOption(OptionBuilder.withArgName("id").hasArg().withDescription("max id").create(MAX_ID_OPTION)); CommandLine cmdline = null; CommandLineParser parser = new GnuParser(); try { cmdline = parser.parse(options, args); } catch (ParseException exp) { System.err.println("Error parsing command line: " + exp.getMessage()); System.exit(-1); } if (cmdline.hasOption(HELP_OPTION) || !cmdline.hasOption(COLLECTION_OPTION) || !cmdline.hasOption(INDEX_OPTION)) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp(IndexTweetsUpdatePlace.class.getName(), options); System.exit(-1); } String collectionPath = cmdline.getOptionValue(COLLECTION_OPTION); String indexPath = cmdline.getOptionValue(INDEX_OPTION); System.out.println(collectionPath + " " + indexPath); LOG.info("collection: " + collectionPath); LOG.info("index: " + indexPath); long startTime = System.currentTimeMillis(); File file = new File(collectionPath); if (!file.exists()) { System.err.println("Error: " + file + " does not exist!"); System.exit(-1); } final FieldType textOptions = new FieldType(); textOptions.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); textOptions.setStored(true); textOptions.setTokenized(true); if (cmdline.hasOption(STORE_TERM_VECTORS_OPTION)) { textOptions.setStoreTermVectors(true); } final StatusStream stream = new JsonStatusCorpusReader(file); final Directory dir = new SimpleFSDirectory(Paths.get(cmdline.getOptionValue(INDEX_OPTION))); final IndexWriterConfig config = new IndexWriterConfig(ANALYZER); config.setOpenMode(IndexWriterConfig.OpenMode.APPEND); final IndexWriter writer = new IndexWriter(dir, config); System.out.print("Original # of docs " + writer.numDocs()); int updateCount = 0; Runtime.getRuntime().addShutdownHook(new Thread() { public void run() { try { stream.close(); } catch (IOException e1) { // TODO Auto-generated catch block e1.printStackTrace(); } try { writer.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } try { dir.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } System.out.println("Shutting down"); } }); int cnt = 0; Status status; try { while ((status = stream.next()) != null) { if (status.getPlace() != null) { // Query q = NumericRangeQuery.newLongRange(TweetStreamReader.StatusField.ID.name, status.getId(), // status.getId(), true, true); // System.out.print("Deleting docCount="+writer.numDocs()); // writer.deleteDocuments(q); // writer.commit(); // System.out.print(" Deleted docCount="+writer.numDocs()); Document doc = new Document(); doc.add(new LongField(StatusField.ID.name, status.getId(), Field.Store.YES)); doc.add(new LongField(StatusField.EPOCH.name, status.getEpoch(), Field.Store.YES)); doc.add(new TextField(StatusField.SCREEN_NAME.name, status.getScreenname(), Store.YES)); doc.add(new Field(StatusField.TEXT.name, status.getText(), textOptions)); doc.add(new IntField(StatusField.FRIENDS_COUNT.name, status.getFollowersCount(), Store.YES)); doc.add(new IntField(StatusField.FOLLOWERS_COUNT.name, status.getFriendsCount(), Store.YES)); doc.add(new IntField(StatusField.STATUSES_COUNT.name, status.getStatusesCount(), Store.YES)); doc.add(new DoubleField(StatusField.LONGITUDE.name, status.getLongitude(), Store.YES)); doc.add(new DoubleField(StatusField.LATITUDE.name, status.getlatitude(), Store.YES)); doc.add(new StringField(StatusField.PLACE.name, status.getPlace(), Store.YES)); long inReplyToStatusId = status.getInReplyToStatusId(); if (inReplyToStatusId > 0) { doc.add(new LongField(StatusField.IN_REPLY_TO_STATUS_ID.name, inReplyToStatusId, Field.Store.YES)); doc.add(new LongField(StatusField.IN_REPLY_TO_USER_ID.name, status.getInReplyToUserId(), Field.Store.YES)); } String lang = status.getLang(); if (!lang.equals("unknown")) { doc.add(new TextField(StatusField.LANG.name, status.getLang(), Store.YES)); } long retweetStatusId = status.getRetweetedStatusId(); if (retweetStatusId > 0) { doc.add(new LongField(StatusField.RETWEETED_STATUS_ID.name, retweetStatusId, Field.Store.YES)); doc.add(new LongField(StatusField.RETWEETED_USER_ID.name, status.getRetweetedUserId(), Field.Store.YES)); doc.add(new IntField(StatusField.RETWEET_COUNT.name, status.getRetweetCount(), Store.YES)); if (status.getRetweetCount() < 0 || status.getRetweetedStatusId() < 0) { LOG.warn("Error parsing retweet fields of " + status.getId()); } } long id = status.getId(); BytesRefBuilder brb = new BytesRefBuilder(); NumericUtils.longToPrefixCodedBytes(id, 0, brb); Term term = new Term(StatusField.ID.name, brb.get()); writer.updateDocument(term, doc); // writer.addDocument(doc); updateCount += 1; if (updateCount % 10000 == 0) { LOG.info(updateCount + " statuses updated"); writer.commit(); System.out.println("Updated docCount=" + writer.numDocs()); } } } LOG.info("Total elapsed time: " + (System.currentTimeMillis() - startTime) + "ms"); } catch (Exception e) { e.printStackTrace(); } finally { writer.close(); dir.close(); stream.close(); } }