List of usage examples for java.util HashSet HashSet
public HashSet()
From source file:org.switchyard.quickstarts.demo.policy.security.basic.propagate.WorkServiceMain.java
public static void main(String... args) throws Exception { Set<String> policies = new HashSet<String>(); for (String arg : args) { arg = Strings.trimToNull(arg);/*from w ww . j a v a2s . c om*/ if (arg != null) { if (arg.equals(CONFIDENTIALITY) || arg.equals(CLIENT_AUTHENTICATION) || arg.equals(HELP)) { policies.add(arg); } else { LOGGER.error(MAVEN_USAGE); throw new Exception(MAVEN_USAGE); } } } if (policies.contains(HELP)) { LOGGER.info(MAVEN_USAGE); } else { final String scheme; final int port; if (policies.contains(CONFIDENTIALITY)) { scheme = "https"; port = 8443; SSLContext sslcontext = SSLContext.getInstance("TLS"); sslcontext.init(null, null, null); SSLSocketFactory sf = new SSLSocketFactory(sslcontext, SSLSocketFactory.STRICT_HOSTNAME_VERIFIER); Scheme https = new Scheme(scheme, port, sf); SchemeRegistry sr = new SchemeRegistry(); sr.register(https); } else { scheme = "http"; port = 8080; } String[] userPass = policies.contains(CLIENT_AUTHENTICATION) ? new String[] { "kermit", "the-frog-1" } : null; invokeWorkService(scheme, port, userPass); } }
From source file:dk.netarkivet.harvester.tools.CreateIndex.java
/** * The main method that does the parsing of the commandline, and makes the actual index request. * * @param args the arguments//from w ww. j a va 2s . co m */ public static void main(String[] args) { Options options = new Options(); CommandLineParser parser = new PosixParser(); CommandLine cmd = null; Option indexType = new Option("t", "type", true, "Type of index"); Option jobList = new Option("l", "jobids", true, "list of jobids"); indexType.setRequired(true); jobList.setRequired(true); options.addOption(indexType); options.addOption(jobList); try { // parse the command line arguments cmd = parser.parse(options, args); } catch (MissingOptionException e) { System.err.println("Some of the required parameters are missing: " + e.getMessage()); dieWithUsage(); } catch (ParseException exp) { System.err.println("Parsing of parameters failed: " + exp.getMessage()); dieWithUsage(); } String typeValue = cmd.getOptionValue(INDEXTYPE_OPTION); String jobidsValue = cmd.getOptionValue(JOBIDS_OPTION); String[] jobidsAsStrings = jobidsValue.split(","); Set<Long> jobIDs = new HashSet<Long>(); for (String idAsString : jobidsAsStrings) { jobIDs.add(Long.valueOf(idAsString)); } JobIndexCache cache = null; String indexTypeAstring = ""; if (typeValue.equalsIgnoreCase("CDX")) { indexTypeAstring = "CDX"; cache = IndexClientFactory.getCDXInstance(); } else if (typeValue.equalsIgnoreCase("DEDUP")) { indexTypeAstring = "DEDUP"; cache = IndexClientFactory.getDedupCrawllogInstance(); } else if (typeValue.equalsIgnoreCase("CRAWLLOG")) { indexTypeAstring = "CRAWLLOG"; cache = IndexClientFactory.getFullCrawllogInstance(); } else { System.err.println("Unknown indextype '" + typeValue + "' requested."); dieWithUsage(); } System.out.println("Creating " + indexTypeAstring + " index for ids: " + jobIDs); Index<Set<Long>> index = cache.getIndex(jobIDs); JMSConnectionFactory.getInstance().cleanup(); }
From source file:edu.upenn.egricelab.AlignerBoost.FilterSAMAlignPE.java
public static void main(String[] args) { if (args.length == 0) { printUsage();//from ww w. j ava2 s .c o m return; } try { parseOptions(args); } catch (IllegalArgumentException e) { System.err.println("Error: " + e.getMessage()); printUsage(); return; } // Read in chrList, if specified if (chrFile != null) { chrFilter = new HashSet<String>(); try { BufferedReader chrFilterIn = new BufferedReader(new FileReader(chrFile)); String chr = null; while ((chr = chrFilterIn.readLine()) != null) chrFilter.add(chr); chrFilterIn.close(); if (verbose > 0) System.err.println( "Only looking at alignments on " + chrFilter.size() + " specified chromosomes"); } catch (IOException e) { System.err.println("Error: " + e.getMessage()); return; } } if (verbose > 0) { // Start the processMonitor processMonitor = new Timer(); // Start the ProcessStatusTask statusTask = new ProcessStatusTask(); // Schedule to show the status every 1 second processMonitor.scheduleAtFixedRate(statusTask, 0, statusFreq); } // Read in known SNP file, if specified if (knownSnpFile != null) { if (verbose > 0) System.err.println("Checking known SNPs from user specified VCF file"); knownVCF = new VCFFileReader(new File(knownSnpFile)); } SamReaderFactory readerFac = SamReaderFactory.makeDefault(); SAMFileWriterFactory writerFac = new SAMFileWriterFactory(); if (!isSilent) readerFac.validationStringency(ValidationStringency.LENIENT); // use LENIENT stringency else readerFac.validationStringency(ValidationStringency.SILENT); // use SILENT stringency SamReader in = readerFac.open(new File(inFile)); SAMFileHeader inHeader = in.getFileHeader(); if (inHeader.getGroupOrder() == GroupOrder.reference && inHeader.getSortOrder() == SortOrder.coordinate) System.err.println("Warning: Input file '" + inFile + "' might be sorted by coordinate and cannot be correctly processed!"); SAMFileHeader header = inHeader.clone(); // copy the inFile header as outFile header // Add new programHeader SAMProgramRecord progRec = new SAMProgramRecord(progName); progRec.setProgramName(progName); progRec.setProgramVersion(progVer); progRec.setCommandLine(StringUtils.join(" ", args)); header.addProgramRecord(progRec); //System.err.println(inFile + " groupOrder: " + in.getFileHeader().getGroupOrder() + " sortOrder: " + in.getFileHeader().getSortOrder()); // reset the orders header.setGroupOrder(groupOrder); header.setSortOrder(sortOrder); // write SAMHeader String prevID = null; SAMRecord prevRecord = null; List<SAMRecord> alnList = new ArrayList<SAMRecord>(); List<SAMRecordPair> alnPEList = null; // Estimate fragment length distribution by scan one-pass through the alignments SAMRecordIterator results = in.iterator(); if (!NO_ESTIMATE) { if (verbose > 0) { System.err.println("Estimating insert fragment size distribution ..."); statusTask.reset(); statusTask.setInfo("alignments scanned"); } long N = 0; double fragL_S = 0; // fragLen sum double fragL_SS = 0; // fragLen^2 sum while (results.hasNext()) { SAMRecord record = results.next(); if (verbose > 0) statusTask.updateStatus(); if (record.getFirstOfPairFlag() && !record.isSecondaryOrSupplementary()) { double fragLen = Math.abs(record.getInferredInsertSize()); if (fragLen != 0 && fragLen >= MIN_FRAG_LEN && fragLen <= MAX_FRAG_LEN) { // only consider certain alignments N++; fragL_S += fragLen; fragL_SS += fragLen * fragLen; } // stop estimate if already enough if (MAX_ESTIMATE_SCAN > 0 && N >= MAX_ESTIMATE_SCAN) break; } } if (verbose > 0) statusTask.finish(); // estimate fragment size if (N >= MIN_ESTIMATE_BASE) { // override command line values MEAN_FRAG_LEN = fragL_S / N; SD_FRAG_LEN = Math.sqrt((N * fragL_SS - fragL_S * fragL_S) / (N * (N - 1))); String estStr = String.format("Estimated fragment size distribution: N(%.1f, %.1f)", MEAN_FRAG_LEN, SD_FRAG_LEN); if (verbose > 0) System.err.println(estStr); // also add the estimation to comment header.addComment(estStr); } else { System.err.println( "Unable to estimate the fragment size distribution due to too few observed alignments"); System.err.println( "You have to specify the '--mean-frag-len' and '--sd-frag-len' on the command line and re-run this step"); statusTask.cancel(); processMonitor.cancel(); return; } // Initiate the normal model normModel = new NormalDistribution(MEAN_FRAG_LEN, SD_FRAG_LEN); // reset the iterator, if necessary if (in.type() == SamReader.Type.SAM_TYPE) { try { in.close(); } catch (IOException e) { System.err.println(e.getMessage()); } in = readerFac.open(new File(inFile)); } results.close(); results = in.iterator(); } // end of NO_ESTIMATE SAMFileWriter out = OUT_IS_SAM ? writerFac.makeSAMWriter(header, false, new File(outFile)) : writerFac.makeBAMWriter(header, false, new File(outFile)); // check each alignment again if (verbose > 0) { System.err.println("Filtering alignments ..."); statusTask.reset(); statusTask.setInfo("alignments processed"); } while (results.hasNext()) { SAMRecord record = results.next(); if (verbose > 0) statusTask.updateStatus(); String ID = record.getReadName(); // fix read and quality string for this read, if is a secondary hit from multiple hits, used for BWA alignment if (ID.equals(prevID) && record.getReadLength() == 0) SAMAlignFixer.fixSAMRecordRead(record, prevRecord); if (chrFilter != null && !chrFilter.contains(record.getReferenceName())) { prevID = ID; prevRecord = record; continue; } // fix MD:Z string for certain aligners with invalid format (i.e. seqAlto) if (fixMD) SAMAlignFixer.fixMisStr(record); // fix alignment, ignore if failed (unmapped or empty) if (!SAMAlignFixer.fixSAMRecord(record, knownVCF, DO_1DP)) { prevID = ID; prevRecord = record; continue; } if (!record.getReadPairedFlag()) { System.err.println("Error: alignment is not from a paired-end read at\n" + record.getSAMString()); out.close(); statusTask.cancel(); processMonitor.cancel(); return; } if (!ID.equals(prevID) && prevID != null || !results.hasNext()) { // a non-first new ID meet, or end of alignments // create alnPEList from filtered alnList alnPEList = createAlnPEListFromAlnList(alnList); //System.err.printf("%d alignments for %s transformed to %d alnPairs%n", alnList.size(), prevID, alnPEList.size()); int totalPair = alnPEList.size(); // filter highly unlikely PEhits filterPEHits(alnPEList, MIN_ALIGN_RATE, MIN_IDENTITY); // calculate posterior mapQ for each pair calcPEHitPostP(alnPEList, totalPair, MAX_HIT); // filter hits by mapQ if (MIN_MAPQ > 0) filterPEHits(alnPEList, MIN_MAPQ); // sort the list first with an anonymous class of comparator, with DESCREASING order Collections.sort(alnPEList, Collections.reverseOrder()); // control max-best if (MAX_BEST != 0 && alnPEList.size() > MAX_BEST) { // potential too much best hits int nBestStratum = 0; int bestMapQ = alnPEList.get(0).getPEMapQ(); // best mapQ from first PE for (SAMRecordPair pr : alnPEList) if (pr.getPEMapQ() == bestMapQ) nBestStratum++; else break; // stop searching for sorted list if (nBestStratum > MAX_BEST) alnPEList.clear(); } // filter alignments with auxiliary filters if (!MAX_SENSITIVITY) filterPEHits(alnPEList, MAX_SEED_MIS, MAX_SEED_INDEL, MAX_ALL_MIS, MAX_ALL_INDEL); // report remaining secondary alignments, up-to MAX_REPORT for (int i = 0; i < alnPEList.size() && (MAX_REPORT == 0 || i < MAX_REPORT); i++) { SAMRecordPair repPair = alnPEList.get(i); if (doUpdateBit) repPair.setNotPrimaryAlignmentFlags(i != 0); int nReport = MAX_REPORT == 0 ? Math.min(alnPEList.size(), MAX_REPORT) : alnPEList.size(); int nFiltered = alnPEList.size(); if (repPair.fwdRecord != null) { repPair.fwdRecord.setAttribute("NH", nReport); repPair.fwdRecord.setAttribute("XN", nFiltered); out.addAlignment(repPair.fwdRecord); } if (repPair.revRecord != null) { repPair.revRecord.setAttribute("NH", nReport); repPair.revRecord.setAttribute("XN", nFiltered); out.addAlignment(repPair.revRecord); } } // reset list alnList.clear(); alnPEList.clear(); } // update if (!ID.equals(prevID)) { prevID = ID; prevRecord = record; } alnList.add(record); } // end while try { in.close(); out.close(); } catch (IOException e) { System.err.println(e.getMessage()); } // Terminate the monitor task and monitor if (verbose > 0) { statusTask.cancel(); statusTask.finish(); processMonitor.cancel(); } }
From source file:metaTile.Main.java
/** * @param args//from w w w .ja v a2s.co m * @throws IOException */ public static void main(String[] args) throws IOException { try { /* parse the command line arguments */ // create the command line parser CommandLineParser parser = new PosixParser(); // create the Options Options options = new Options(); options.addOption("i", "input", true, "File to read original tile list from."); options.addOption("o", "output", true, "File to write shorter meta-tile list to."); options.addOption("m", "metatiles", true, "Number of tiles in x and y direction to group into one meta-tile."); // parse the command line arguments CommandLine commandLine = parser.parse(options, args); if (!commandLine.hasOption("input") || !commandLine.hasOption("output") || !commandLine.hasOption("metatiles")) printUsage(options); String inputFileName = commandLine.getOptionValue("input"); String outputFileName = commandLine.getOptionValue("output"); int metaTileSize = Integer.parseInt(commandLine.getOptionValue("metatiles")); ArrayList<RenderingTile> tiles = new ArrayList<RenderingTile>(); BufferedReader tileListReader = new BufferedReader(new FileReader(new File(inputFileName))); BufferedWriter renderMetatileListWriter = new BufferedWriter(new FileWriter(new File(outputFileName))); String line = tileListReader.readLine(); while (line != null) { String[] columns = line.split("/"); if (columns.length == 3) tiles.add(new RenderingTile(Integer.parseInt(columns[0]), Integer.parseInt(columns[1]), Integer.parseInt(columns[2]))); line = tileListReader.readLine(); } tileListReader.close(); int hits = 0; // tiles which we are already rendering as the top left corner of 4x4 metatiles HashSet<RenderingTile> whitelist = new HashSet<RenderingTile>(); // for each tile in the list see if it has a meta-tile in the whitelist already for (int i = 0; i < tiles.size(); i++) { boolean hit = false; // by default we aren't already rendering this tile as part of another metatile for (int dx = 0; dx < metaTileSize; dx++) { for (int dy = 0; dy < metaTileSize; dy++) { RenderingTile candidate = new RenderingTile(tiles.get(i).z, tiles.get(i).x - dx, tiles.get(i).y - dy); if (whitelist.contains(candidate)) { hit = true; // now exit the two for loops iterating over tiles inside a meta-tile dx = metaTileSize; dy = metaTileSize; } } } // if this tile doesn't already have a meta-tile in the whitelist, add it if (hit == false) { hits++; renderMetatileListWriter.write(tiles.get(i).toString() + "/" + metaTileSize + "\n"); whitelist.add(tiles.get(i)); } } renderMetatileListWriter.close(); System.out.println( "Reduced " + tiles.size() + " tiles into " + hits + " metatiles of size " + metaTileSize); } catch (Exception e) { e.printStackTrace(); } }
From source file:squash.tools.FakeBookingCreator.java
public static void main(String[] args) throws IOException { int numberOfDays = 21; int numberOfCourts = 5; int maxCourtSpan = 5; int numberOfSlots = 16; int maxSlotSpan = 3; int minSurnameLength = 2; int maxSurnameLength = 20; int minBookingsPerDay = 0; int maxBookingsPerDay = 8; LocalDate startDate = LocalDate.of(2016, 7, 5); DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); List<Booking> bookings = new ArrayList<>(); for (LocalDate date = startDate; date.isBefore(startDate.plusDays(numberOfDays)); date = date.plusDays(1)) { int numBookings = ThreadLocalRandom.current().nextInt(minBookingsPerDay, maxBookingsPerDay + 1); List<Booking> daysBookings = new ArrayList<>(); for (int bookingIndex = 0; bookingIndex < numBookings; bookingIndex++) { String player1 = RandomStringUtils.randomAlphabetic(1) + "." + RandomStringUtils.randomAlphabetic( ThreadLocalRandom.current().nextInt(minSurnameLength, maxSurnameLength + 1)); String player2 = RandomStringUtils.randomAlphabetic(1) + "." + RandomStringUtils.randomAlphabetic( ThreadLocalRandom.current().nextInt(minSurnameLength, maxSurnameLength + 1)); Set<ImmutablePair<Integer, Integer>> bookedCourts = new HashSet<>(); daysBookings.forEach((booking) -> { addBookingToSet(booking, bookedCourts); });//w ww .j ava2s. c o m Booking booking; Set<ImmutablePair<Integer, Integer>> courtsToBook = new HashSet<>(); do { // Loop until we create a booking of free courts int court = ThreadLocalRandom.current().nextInt(1, numberOfCourts + 1); int courtSpan = ThreadLocalRandom.current().nextInt(1, Math.min(maxCourtSpan + 1, numberOfCourts - court + 2)); int slot = ThreadLocalRandom.current().nextInt(1, numberOfSlots + 1); int slotSpan = ThreadLocalRandom.current().nextInt(1, Math.min(maxSlotSpan + 1, numberOfSlots - slot + 2)); booking = new Booking(court, courtSpan, slot, slotSpan, player1 + "/" + player2); booking.setDate(date.format(formatter)); courtsToBook.clear(); addBookingToSet(booking, courtsToBook); } while (Boolean.valueOf(Sets.intersection(courtsToBook, bookedCourts).size() > 0)); daysBookings.add(booking); } bookings.addAll(daysBookings); } // Encode bookings as JSON // Create the node factory that gives us nodes. JsonNodeFactory factory = new JsonNodeFactory(false); // Create a json factory to write the treenode as json. JsonFactory jsonFactory = new JsonFactory(); ObjectNode rootNode = factory.objectNode(); ArrayNode bookingsNode = rootNode.putArray("bookings"); for (int i = 0; i < bookings.size(); i++) { Booking booking = bookings.get(i); ObjectNode bookingNode = factory.objectNode(); bookingNode.put("court", booking.getCourt()); bookingNode.put("courtSpan", booking.getCourtSpan()); bookingNode.put("slot", booking.getSlot()); bookingNode.put("slotSpan", booking.getSlotSpan()); bookingNode.put("name", booking.getName()); bookingNode.put("date", booking.getDate()); bookingsNode.add(bookingNode); } // Add empty booking rules array - just so restore works rootNode.putArray("bookingRules"); rootNode.put("clearBeforeRestore", true); try (JsonGenerator generator = jsonFactory.createGenerator(new File("FakeBookings.json"), JsonEncoding.UTF8)) { ObjectMapper mapper = new ObjectMapper(); mapper.setSerializationInclusion(Include.NON_EMPTY); mapper.setSerializationInclusion(Include.NON_NULL); mapper.writeTree(generator, rootNode); } }
From source file:com.khartec.waltz.jobs.sample.FlowGenerator.java
public static void main(String[] args) { AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(DIConfiguration.class); AuthoritativeSourceDao authSourceDao = ctx.getBean(AuthoritativeSourceDao.class); ApplicationService applicationDao = ctx.getBean(ApplicationService.class); DataTypeService dataTypesDao = ctx.getBean(DataTypeService.class); DataFlowService dataFlowDao = ctx.getBean(DataFlowService.class); OrganisationalUnitService orgUnitDao = ctx.getBean(OrganisationalUnitService.class); DataSource dataSource = ctx.getBean(DataSource.class); DSLContext dsl = ctx.getBean(DSLContext.class); List<AuthoritativeSource> authSources = authSourceDao.findByEntityKind(EntityKind.ORG_UNIT); List<String> dataTypes = dataTypesDao.getAll().stream().map(dt -> dt.code()).collect(toList()); List<Application> apps = applicationDao.findAll(); List<OrganisationalUnit> orgUnits = orgUnitDao.findAll(); Set<DataFlow> expectedFlows = authSources.stream().flatMap(a -> { long orgUnitId = a.parentReference().id(); return IntStream.range(0, rnd.nextInt(40)) .mapToObj(i -> ImmutableDataFlow.builder().dataType(a.dataType()) .source(a.applicationReference()).target(randomAppPick(apps, orgUnitId)).build()); }).collect(Collectors.toSet()); Set<DataFlow> probableFlows = authSources.stream().flatMap(a -> IntStream.range(0, rnd.nextInt(30)) .mapToObj(i -> ImmutableDataFlow.builder().dataType(a.dataType()).source(a.applicationReference()) .target(randomAppPick(apps, randomPick(orgUnits).id().get())).build())) .collect(Collectors.toSet()); Set<DataFlow> randomFlows = apps.stream() .map(a -> ImmutableDataFlow.builder().source(a.toEntityReference())) .map(b -> b.target(randomAppPick(apps, randomPick(orgUnits).id().get()))) .map(b -> b.dataType(randomPick(dataTypes)).build()).collect(Collectors.toSet()); dsl.deleteFrom(DATA_FLOW).execute(); Set<DataFlow> all = new HashSet<>(); all.addAll(randomFlows);// w w w. jav a 2s . c o m all.addAll(expectedFlows); all.addAll(probableFlows); dataFlowDao.addFlows(new ArrayList<>(all)); System.out.println("Done"); }
From source file:HSqlPrimerDesign.java
public static void main(String[] args) throws NoSuchFieldException, IllegalAccessException, ClassNotFoundException, InstantiationException, SQLException, FileNotFoundException { Class.forName(JDBC_DRIVER_HSQL).newInstance(); conn = DriverManager.getConnection(DB_SERVER_URL, USER, PASS); PrintWriter log = new PrintWriter(new File("javalog.log")); DpalLoad.main(args);/*from w w w.j a v a 2 s . c o m*/ Dpal_Inst = DpalLoad.INSTANCE_WIN64; // int[][] arr =calcHairpin("GGGGGGCCCCCCCCCCCCGGGGGGG",4); // if(arr.length<=1){ // System.out.println("No Hairpin's found"); // }else{ // System.out.println("Hairpin(s) found"); // } Statement stat = conn.createStatement(); ResultSet call = stat.executeQuery( "Select * From " + "Primerdb.primers where Cluster ='A1' and UniqueP =True and Bp = 20"); Set<CharSequence> primers = new HashSet<>(); while (call.next()) { primers.add(call.getString("Sequence")); } // primers.stream().forEach(x->{ // log.println(x); // log.println(complementarity(x, x, Dpal_Inst)); // int[][] arr =calcHairpin((String)x,4); // if(arr.length<=1){ // log.println("No Hairpin's found"); // }else{ // log.println("Hairpin(s) found"); // } // log.println(); // log.flush(); // }); }
From source file:edu.wpi.margrave.MCommunicator.java
public static void main(String[] args) { ArrayList<String> foo = new ArrayList<String>(); Set<String> argsSet = new HashSet<String>(); for (int ii = 0; ii < args.length; ii++) { argsSet.add(args[ii].toLowerCase()); }//ww w . j a va2 s. c o m if (argsSet.contains("-log")) { // parser is in racket now. instead, require -log switch for logging //MEnvironment.debugParser = true; bDoLogging = true; } if (argsSet.contains("-min")) { bMinimalModels = true; } // Re-direct all System.err input to our custom buffer // Uses Apache Commons IO for WriterOutputStream. System.setErr(new PrintStream(new WriterOutputStream(MEnvironment.errorWriter), true)); // Re-direct all System.out input to our custom buffer. // (We have already saved System.out.) // This is useful in case we start getting GC messages from SAT4j. System.setOut(new PrintStream(new WriterOutputStream(MEnvironment.outWriter), true)); initializeLog(); writeToLog("\n\n"); // Inform the caller that we are ready to receive commands sendReadyReply(); while (true) { // Block until a command is received, handle it, and then return the result. handleXMLCommand(in); } // outLog will be closed as it goes out of scope }
From source file:de.zib.vold.userInterface.ABI.java
public static void main(String[] args) { ABI abi = new ABI(); while (true) { try {//from ww w . j a v a 2s . c om InputStreamReader isr = new InputStreamReader(System.in); BufferedReader br = new BufferedReader(isr); System.out.print("#: "); String s = br.readLine(); if (null == s) break; String[] a = s.split("\\s+"); if (a.length < 1) { System.out.println("ERROR: The following commands are valid:"); System.out.println("ERROR: insert <source> <scope> <type> <keyname> {<value> }*"); System.out.println("ERROR: lookup <scope> <type> <keyname>"); System.out.println("ERROR: exit"); continue; } else if (a[0].equals("lookup") || a[0].equals("l")) { if (a.length < 4) { System.out.println("ERROR: Syntax for lookup is:"); System.out.println("ERROR: lookup <scope> <type> <keyname>"); continue; } Map<Key, Set<String>> result; try { result = abi.frontend.lookup(new Key(a[1], a[2], a[3])); } catch (VoldException e) { System.out.println( "An internal error occured: " + e.getClass().getName() + ": " + e.getMessage()); continue; } for (Map.Entry<Key, Set<String>> entry : result.entrySet()) { Key k = entry.getKey(); System.out.println("+Found ('" + k.get_scope() + "', '" + k.get_type() + "', '" + k.get_keyname() + "')"); for (String v : entry.getValue()) { System.out.println("-" + v); } } } else if (a[0].equals("insert") || a[0].equals("i")) { if (a.length < 5) { System.out.println("ERROR: Syntax for insert is:"); System.out.println("ERROR: insert <source> <scope> <type> <keyname> {<value> }*"); continue; } Key k = new Key(a[2], a[3], a[4]); Set<String> values = new HashSet<String>(); for (int i = 5; i < a.length; ++i) { values.add(a[i]); } try { abi.frontend.insert(a[1], k, values, DateTime.now().getMillis()); } catch (VoldException e) { System.out.println( "An internal error occured: " + e.getClass().getName() + ": " + e.getMessage()); continue; } } else if (a[0].equals("exit") || a[0].equals("x")) { break; } else { System.out.println("ERROR: Unknown command!"); System.out.println("ERROR: The following commands are valid:"); System.out.println("ERROR: insert <source> <scope> <type> <keyname> {<value> }*"); System.out.println("ERROR: lookup <scope> <type> <keyname>"); System.out.println("ERROR: exit"); } } catch (IOException e) { e.printStackTrace(); } } System.exit(0); }
From source file:cu.uci.gws.sdlcrawler.PdfCrawlController.java
public static void main(String[] args) throws Exception { Properties cm = PdfCrawlerConfigManager.getInstance().loadConfigFile(); long startTime = System.currentTimeMillis(); DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); Date date = new Date(); System.out.println(dateFormat.format(date)); int numberOfCrawlers = Integer.parseInt(cm.getProperty("sdlcrawler.NumberOfCrawlers")); String pdfFolder = cm.getProperty("sdlcrawler.CrawlPdfFolder"); CrawlConfig config = new CrawlConfig(); config.setCrawlStorageFolder(cm.getProperty("sdlcrawler.CrawlStorageFolder")); config.setProxyHost(cm.getProperty("sdlcrawler.ProxyHost")); if (!"".equals(cm.getProperty("sdlcrawler.ProxyPort"))) { config.setProxyPort(Integer.parseInt(cm.getProperty("sdlcrawler.ProxyPort"))); }//from www. j av a 2s.c om config.setProxyUsername(cm.getProperty("sdlcrawler.ProxyUser")); config.setProxyPassword(cm.getProperty("sdlcrawler.ProxyPass")); config.setMaxDownloadSize(Integer.parseInt(cm.getProperty("sdlcrawler.MaxDownloadSize"))); config.setIncludeBinaryContentInCrawling( Boolean.parseBoolean(cm.getProperty("sdlcrawler.IncludeBinaryContent"))); config.setFollowRedirects(Boolean.parseBoolean(cm.getProperty("sdlcrawler.Redirects"))); config.setUserAgentString(cm.getProperty("sdlcrawler.UserAgent")); config.setMaxDepthOfCrawling(Integer.parseInt(cm.getProperty("sdlcrawler.MaxDepthCrawl"))); config.setMaxConnectionsPerHost(Integer.parseInt(cm.getProperty("sdlcrawler.MaxConnectionsPerHost"))); config.setSocketTimeout(Integer.parseInt(cm.getProperty("sdlcrawler.SocketTimeout"))); config.setMaxOutgoingLinksToFollow(Integer.parseInt(cm.getProperty("sdlcrawler.MaxOutgoingLinks"))); config.setResumableCrawling(Boolean.parseBoolean(cm.getProperty("sdlcrawler.ResumableCrawling"))); config.setIncludeHttpsPages(Boolean.parseBoolean(cm.getProperty("sdlcrawler.IncludeHttpsPages"))); config.setMaxTotalConnections(Integer.parseInt(cm.getProperty("sdlcrawler.MaxTotalConnections"))); config.setMaxPagesToFetch(Integer.parseInt(cm.getProperty("sdlcrawler.MaxPagesToFetch"))); config.setPolitenessDelay(Integer.parseInt(cm.getProperty("sdlcrawler.PolitenessDelay"))); config.setConnectionTimeout(Integer.parseInt(cm.getProperty("sdlcrawler.ConnectionTimeout"))); System.out.println(config.toString()); Collection<BasicHeader> defaultHeaders = new HashSet<>(); defaultHeaders .add(new BasicHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")); defaultHeaders.add(new BasicHeader("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.3")); defaultHeaders.add(new BasicHeader("Accept-Language", "en-US,en,es-ES,es;q=0.8")); defaultHeaders.add(new BasicHeader("Connection", "keep-alive")); config.setDefaultHeaders(defaultHeaders); List<String> list = Files.readAllLines(Paths.get("config/" + cm.getProperty("sdlcrawler.SeedFile")), StandardCharsets.UTF_8); String[] crawlDomains = list.toArray(new String[list.size()]); PageFetcher pageFetcher = new PageFetcher(config); RobotstxtConfig robotstxtConfig = new RobotstxtConfig(); RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig, pageFetcher); CrawlController controller = new CrawlController(config, pageFetcher, robotstxtServer); for (String domain : crawlDomains) { controller.addSeed(domain); } PdfCrawler.configure(crawlDomains, pdfFolder); controller.start(PdfCrawler.class, numberOfCrawlers); DateFormat dateFormat1 = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); Date date1 = new Date(); System.out.println(dateFormat1.format(date1)); long endTime = System.currentTimeMillis(); long totalTime = endTime - startTime; System.out.println("Total time:" + totalTime); }