List of usage examples for java.util List size
int size();
From source file:com.cloudera.recordbreaker.schemadict.SchemaSuggest.java
/** * SchemaSuggest takes an avro file where schema elements may be anonymous. It then attempts to * compute good labels for the anonymous elts. By default, this tool simply prints out the * suggested labels, if any. The user may include a flag to rewrite the input data using * the new labels.//from w w w. j a v a 2 s . c o m * * schemaSuggest avroFile * */ public static void main(String argv[]) throws IOException { CommandLine cmd = null; boolean debug = false; Options options = new Options(); options.addOption("?", false, "Help for command-line"); options.addOption("f", true, "Accept suggestions and rewrite input to a new Avro file"); options.addOption("d", false, "Debug mode"); options.addOption("k", true, "How many matches to emit."); try { CommandLineParser parser = new PosixParser(); cmd = parser.parse(options, argv); } catch (ParseException e) { HelpFormatter fmt = new HelpFormatter(); fmt.printHelp("SchemaSuggest", options, true); System.err.println("Required inputs: <schemadictionary> <anonymousAvro>"); System.exit(-1); } if (cmd.hasOption("?")) { HelpFormatter fmt = new HelpFormatter(); fmt.printHelp("SchemaSuggest", options, true); System.err.println("Required inputs: <schemadictionary> <anonymousAvro>"); System.exit(0); } if (cmd.hasOption("d")) { debug = true; } int k = 1; if (cmd.hasOption("k")) { try { k = Integer.parseInt(cmd.getOptionValue("k")); } catch (NumberFormatException nfe) { } } String[] argArray = cmd.getArgs(); if (argArray.length < 2) { HelpFormatter fmt = new HelpFormatter(); fmt.printHelp("SchemaSuggest", options, true); System.err.println("Required inputs: <schemadictionary> <anonymousAvro>"); System.exit(0); } File dataDir = new File(argArray[0]).getCanonicalFile(); File inputData = new File(argArray[1]).getCanonicalFile(); SchemaSuggest ss = new SchemaSuggest(dataDir); List<DictionaryMapping> mappings = ss.inferSchemaMapping(inputData, k); if (!cmd.hasOption("f")) { System.out.println("Ranking of closest known data types, with match-distance (smaller is better):"); int counter = 1; for (DictionaryMapping mapping : mappings) { SchemaMapping sm = mapping.getMapping(); List<SchemaMappingOp> bestOps = sm.getMapping(); System.err.println(); System.err.println(); System.err.println("-------------------------------------------------------------"); System.out.println( counter + ". '" + mapping.getDictEntry().getInfo() + "', with distance: " + sm.getDist()); List<SchemaMappingOp> renames = new ArrayList<SchemaMappingOp>(); List<SchemaMappingOp> extraInTarget = new ArrayList<SchemaMappingOp>(); List<SchemaMappingOp> extraInSource = new ArrayList<SchemaMappingOp>(); for (SchemaMappingOp op : bestOps) { if (op.opcode == SchemaMappingOp.CREATE_OP) { extraInTarget.add(op); } else if (op.opcode == SchemaMappingOp.DELETE_OP) { if (op.getS1DatasetLabel().compareTo("input") == 0) { extraInSource.add(op); } else { extraInTarget.add(op); } } else if (op.opcode == SchemaMappingOp.TRANSFORM_OP) { renames.add(op); } } System.err.println(); System.err.println(" DISCOVERED LABELS"); int counterIn = 1; if (renames.size() == 0) { System.err.println(" (None)"); } else { for (SchemaMappingOp op : renames) { System.err.println(" " + counterIn + ". " + "In '" + op.getS1DatasetLabel() + "', label '" + op.getS1FieldLabel() + "' AS " + op.getS2FieldLabel()); if (debug) { if (op.getS1DocStr() != null && op.getS1DocStr().length() > 0) { System.err.println( " '" + op.getS1DocStr() + "' ==> '" + op.getS2DocStr() + "'"); } } counterIn++; } } System.err.println(); System.err.println(" UNMATCHED ITEMS IN TARGET DATA TYPE"); counterIn = 1; if (extraInTarget.size() == 0) { System.err.println(" (None)"); } else { for (SchemaMappingOp op : extraInTarget) { System.err.println(" " + counterIn + ". " + op.getS1FieldLabel()); if (debug) { if (op.getS1DocStr() != null && op.getS1DocStr().length() > 0) { System.err.println(" " + op.getS1DocStr()); } } counterIn++; } } System.err.println(); System.err.println(" UNMATCHED ITEMS IN SOURCE DATA"); counterIn = 1; if (extraInSource.size() == 0) { System.err.println(" (None)"); } else { for (SchemaMappingOp op : extraInSource) { System.err.println(" " + counterIn + ". " + op.getS1FieldLabel()); if (debug) { if (op.getS1DocStr() != null && op.getS1DocStr().length() > 0) { System.err.println(" " + op.getS1DocStr()); } } counterIn++; } } counter++; } } }
From source file:com.roncoo.pay.app.reconciliation.ReconciliationTask.java
public static void main(String[] args) { SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd"); try {//from w ww .jav a 2s . c om // Spring? ClassPathXmlApplicationContext context = new ClassPathXmlApplicationContext( new String[] { "spring-context.xml" }); // ?SpringContextUtil final SpringContextUtil ctxUtil = new SpringContextUtil(); ctxUtil.setApplicationContext(context); @SuppressWarnings("rawtypes") // ??(???????) List reconciliationInterList = ReconciliationInterface.getInterface(); // ?biz ReconciliationFileDownBiz fileDownBiz = (ReconciliationFileDownBiz) SpringContextUtil .getBean("reconciliationFileDownBiz"); ReconciliationFileParserBiz parserBiz = (ReconciliationFileParserBiz) SpringContextUtil .getBean("reconciliationFileParserBiz"); ReconciliationCheckBiz checkBiz = (ReconciliationCheckBiz) SpringContextUtil .getBean("reconciliationCheckBiz"); ReconciliationValidateBiz validateBiz = (ReconciliationValidateBiz) SpringContextUtil .getBean("reconciliationValidateBiz"); RpAccountCheckBatchService batchService = (RpAccountCheckBatchService) SpringContextUtil .getBean("rpAccountCheckBatchService"); BuildNoService buildNoService = (BuildNoService) SpringContextUtil.getBean("buildNoService"); // ????? for (int num = 0; num < reconciliationInterList.size(); num++) { // ?? ReconciliationInterface reconciliationInter = (ReconciliationInterface) reconciliationInterList .get(num); if (reconciliationInter == null) { LOG.info("??" + reconciliationInter + ""); continue; } // ??? Date billDate = DateUtil.addDay(new Date(), -reconciliationInter.getBillDay()); // ?? String interfaceCode = reconciliationInter.getInterfaceCode(); /** step1:? **/ RpAccountCheckBatch batch = new RpAccountCheckBatch(); Boolean checked = validateBiz.isChecked(interfaceCode, billDate); if (checked) { LOG.info("?[" + sdf.format(billDate) + "],?[" + interfaceCode + "],????"); continue; } // batch.setCreater("reconciliationSystem"); batch.setCreateTime(new Date()); batch.setBillDate(billDate); batch.setBatchNo(buildNoService.buildReconciliationNo()); batch.setBankType(interfaceCode); /** step2: **/ File file = null; try { LOG.info("ReconciliationFileDownBiz,"); file = fileDownBiz.downReconciliationFile(interfaceCode, billDate); if (file == null) { continue; } LOG.info("?"); } catch (Exception e) { LOG.error(":", e); batch.setStatus(BatchStatusEnum.FAIL.name()); batch.setRemark(""); batchService.saveData(batch); continue; } /** step3:? **/ List<ReconciliationEntityVo> bankList = null; try { LOG.info("=ReconciliationFileParserBiz=>?>>>"); // ? bankList = parserBiz.parser(batch, file, billDate, interfaceCode); // if (BatchStatusEnum.ERROR.name().equals(batch.getStatus())) { continue; } LOG.info("??"); } catch (Exception e) { LOG.error("?:", e); batch.setStatus(BatchStatusEnum.FAIL.name()); batch.setRemark("?"); batchService.saveData(batch); continue; } /** step4:? **/ try { checkBiz.check(bankList, interfaceCode, batch); } catch (Exception e) { LOG.error(":", e); batch.setStatus(BatchStatusEnum.FAIL.name()); batch.setRemark(""); batchService.saveData(batch); continue; } } /** step5:? **/ // ??? validateBiz.validateScratchPool(); } catch (Exception e) { LOG.error("roncoo-app-reconciliation error:", e); } }
From source file:org.objectrepository.MessageConsumerDaemon.java
/** * main/*from w ww. j ava 2 s. c om*/ * <p/> * Accepts one folder as argument: -messageQueues * That folder ought to contain one or more folders ( or symbolic links ) to the files * The folder has the format: [foldername] or [foldername].[maxTasks] * MaxTasks is to indicate the total number of jobs being able to run. * * long * * @param argv */ public static void main(String[] argv) { if (instance == null) { final Properties properties = new Properties(); if (argv.length > 0) { for (int i = 0; i < argv.length; i += 2) { try { properties.put(argv[i], argv[i + 1]); } catch (ArrayIndexOutOfBoundsException arr) { System.out.println("Missing value after parameter " + argv[i]); System.exit(-1); } } } else { log.fatal("Usage: pmq-agent.jar -messageQueues [queues] -heartbeatInterval [interval in ms]\n" + "The queues is a folder that contains symbolic links to the startup scripts."); System.exit(-1); } if (log.isInfoEnabled()) { log.info("Arguments set: "); for (String key : properties.stringPropertyNames()) { log.info("'" + key + "'='" + properties.getProperty(key) + "'"); } } if (!properties.containsKey("-messageQueues")) { log.fatal("Expected case sensitive parameter: -messageQueues"); System.exit(-1); } final File messageQueues = new File((String) properties.get("-messageQueues")); if (!messageQueues.exists()) { log.fatal("Cannot find folder for messageQueues: " + messageQueues.getAbsolutePath()); System.exit(-1); } if (messageQueues.isFile()) { log.fatal( "-messageQueues should point to a folder, not a file: " + messageQueues.getAbsolutePath()); System.exit(-1); } long heartbeatInterval = 600000; if (properties.containsKey("-heartbeatInterval")) { heartbeatInterval = Long.parseLong((String) properties.get("heartbeatInterval")); } String identifier = null; if (properties.containsKey("-id")) { identifier = (String) properties.get("-id"); } else if (properties.containsKey("-identifier")) { identifier = (String) properties.get("-identifier"); } final File[] files = messageQueues.listFiles(); final String[] scriptNames = (properties.containsKey("-startup")) ? new String[] { properties.getProperty("-startup") } : new String[] { "/startup.sh", "\\startup.bat" }; final List<Queue> queues = new ArrayList<Queue>(); for (File file : files) { final String name = file.getName(); final String[] split = name.split("\\.", 2); final String queueName = split[0]; for (String scriptName : scriptNames) { final String shellScript = file.getAbsolutePath() + scriptName; final int maxTask = (split.length == 1) ? 1 : Integer.parseInt(split[1]); log.info("Candidate mq client for " + queueName + " maxTasks " + maxTask); if (new File(shellScript).exists()) { final Queue queue = new Queue(queueName, shellScript, false); queue.setCorePoolSize(1); queue.setMaxPoolSize(maxTask); queue.setQueueCapacity(1); queues.add(queue); break; } else { log.warn("... skipping, because no startup script found at " + shellScript); } } } if (queues.size() == 0) { log.fatal("No queue folders seen in " + messageQueues.getAbsolutePath()); System.exit(-1); } // Add the system queue queues.add(new Queue("Connection", null, true)); getInstance(queues, identifier, heartbeatInterval).run(); } System.exit(0); }
From source file:com.act.analysis.surfactant.AnalysisDriver.java
public static void main(String[] args) throws Exception { Options opts = new Options(); for (Option.Builder b : OPTION_BUILDERS) { opts.addOption(b.build());/*w w w.ja v a 2 s . c om*/ } CommandLine cl = null; try { CommandLineParser parser = new DefaultParser(); cl = parser.parse(opts, args); } catch (ParseException e) { System.err.format("Argument parsing failed: %s\n", e.getMessage()); HELP_FORMATTER.printHelp(LoadPlateCompositionIntoDB.class.getCanonicalName(), HELP_MESSAGE, opts, null, true); System.exit(1); } if (cl.hasOption("help")) { HELP_FORMATTER.printHelp(LoadPlateCompositionIntoDB.class.getCanonicalName(), HELP_MESSAGE, opts, null, true); return; } Set<String> seenOutputIds = new HashSet<>(); TSVWriter<String, String> tsvWriter = null; if (cl.hasOption(OPTION_OUTPUT_FILE)) { File outputFile = new File(cl.getOptionValue(OPTION_OUTPUT_FILE)); List<Map<String, String>> oldResults = null; if (outputFile.exists()) { System.err.format( "Output file already exists, reading old results and skipping processed molecules.\n"); TSVParser outputParser = new TSVParser(); outputParser.parse(outputFile); oldResults = outputParser.getResults(); for (Map<String, String> row : oldResults) { // TODO: verify that the last row was written cleanly/completely. seenOutputIds.add(row.get("id")); } } List<String> header = new ArrayList<>(); header.add("name"); header.add("id"); header.add("inchi"); header.add("label"); for (SurfactantAnalysis.FEATURES f : SurfactantAnalysis.FEATURES.values()) { header.add(f.toString()); } // TODO: make this API more auto-closable friendly. tsvWriter = new TSVWriter<>(header); tsvWriter.open(outputFile); if (oldResults != null) { System.out.format("Re-writing %d existing result rows\n", oldResults.size()); tsvWriter.append(oldResults); } } try { Map<SurfactantAnalysis.FEATURES, Double> analysisFeatures; LicenseManager.setLicenseFile(cl.getOptionValue(OPTION_LICENSE_FILE)); if (cl.hasOption(OPTION_INCHI)) { analysisFeatures = SurfactantAnalysis.performAnalysis(cl.getOptionValue(OPTION_INCHI), cl.hasOption(OPTION_DISPLAY)); Map<String, String> tsvFeatures = new HashMap<>(); // Convert features to strings to avoid some weird formatting issues. It's ugly, but it works. for (Map.Entry<SurfactantAnalysis.FEATURES, Double> entry : analysisFeatures.entrySet()) { tsvFeatures.put(entry.getKey().toString(), String.format("%.6f", entry.getValue())); } tsvFeatures.put("name", "direct-inchi-input"); if (tsvWriter != null) { tsvWriter.append(tsvFeatures); } } else if (cl.hasOption(OPTION_INPUT_FILE)) { TSVParser parser = new TSVParser(); parser.parse(new File(cl.getOptionValue(OPTION_INPUT_FILE))); int i = 0; List<Map<String, String>> inputRows = parser.getResults(); for (Map<String, String> row : inputRows) { i++; // Just for warning messages. if (!row.containsKey("name") || !row.containsKey("id") || !row.containsKey("inchi")) { System.err.format( "WARNING: TSV rows must contain at least name, id, and inchi, skipping row %d\n", i); continue; } if (seenOutputIds.contains(row.get("id"))) { System.out.format("Skipping input row with id already in output: %s\n", row.get("id")); continue; } System.out.format("Analysis for chemical %s\n", row.get("name")); try { analysisFeatures = SurfactantAnalysis.performAnalysis(row.get("inchi"), false); } catch (Exception e) { // Ignore exceptions for now. Sometimes the regression analysis or Chemaxon processing chokes unexpectedly. System.err.format("ERROR caught exception while processing '%s':\n", row.get("name")); System.err.format("%s\n", e.getMessage()); e.printStackTrace(System.err); System.err.println("Skipping..."); continue; } System.out.format("--- Done analysis for chemical %s\n", row.get("name")); // This is a duplicate of the OPTION_INCHI block code, but it's inside of a tight loop, so... Map<String, String> tsvFeatures = new HashMap<>(); for (Map.Entry<SurfactantAnalysis.FEATURES, Double> entry : analysisFeatures.entrySet()) { tsvFeatures.put(entry.getKey().toString(), String.format("%.6f", entry.getValue())); } tsvFeatures.put("name", row.get("name")); tsvFeatures.put("id", row.get("id")); tsvFeatures.put("inchi", row.get("inchi")); tsvFeatures.put("label", row.containsKey("label") ? row.get("label") : "?"); if (tsvWriter != null) { tsvWriter.append(tsvFeatures); // Flush every time in case we crash or get interrupted. The features must flow! tsvWriter.flush(); } } } else { throw new RuntimeException("Must specify inchi or input file"); } } finally { if (tsvWriter != null) { tsvWriter.close(); } } }
From source file:com.github.checkstyle.Main.java
/** * Entry point.//w ww .j av a2 s .c o m * @param args command line arguments. */ public static void main(String... args) { int errorCounter; List<String> publicationErrors = null; try { final CliProcessor cliProcessor = new CliProcessor(args); cliProcessor.process(); if (cliProcessor.hasErrors()) { printListOf(cliProcessor.getErrorMessages()); errorCounter = cliProcessor.getErrorMessages().size(); } else { final CliOptions cliOptions = cliProcessor.getCliOptions(); final Result notesBuilderResult = runNotesBuilder(cliOptions); errorCounter = notesBuilderResult.getErrorMessages().size(); if (errorCounter == 0) { runPostGeneration(notesBuilderResult.getReleaseNotes(), cliOptions); publicationErrors = runPostPublication(cliOptions); } } } catch (ParseException | GitAPIException | IOException | TemplateException ex) { errorCounter = 1; System.out.println(ex.getMessage()); CliProcessor.printUsage(); } if (errorCounter == 0) { if (publicationErrors != null && !publicationErrors.isEmpty()) { System.out.println(String.format("%nPublication ends with %d errors:", publicationErrors.size())); printListOf(publicationErrors); } else { System.out.println(String.format("%nExecution succeeded!")); } } else { System.out.println(String.format("%nGeneration ends with %d errors.", errorCounter)); System.exit(ERROR_EXIT_CODE); } }
From source file:com.glaf.core.util.StringTools.java
public static void main(String[] args) { String text = "123,234,345,567,789"; List<String> rows = StringTools.split(text, ","); System.out.println(rows.get(2)); System.out.println(rows);/*w ww. ja v a 2s . co m*/ for (int i = 0, len = rows.size(); i < len; i++) { System.out.println(rows.get(i)); } Set<String> x = new HashSet<String>(); for (int i = 0; i < 9999; i++) { String str = StringTools.random(); x.add(str); System.out.println(str); } System.out.println("size=" + x.size()); Random random = new Random(); for (int i = 1; i < 100; i++) { int k = Math.abs(random.nextInt(9999)); int a = String.valueOf(k).length(); int b = String.valueOf(9999).length(); System.out.println("a=" + a); System.out.println("b=" + b); String xx = ""; if (a != b) { for (int j = 0; j < (b - a); j++) { xx = "0" + xx; } } xx = xx + k; System.out.println(xx); } System.out.println(toUnderLineName("ISOCertifiedStaff")); System.out.println(toUnderLineName("CertifiedStaff")); System.out.println(toUnderLineName("UserID")); System.out.println(toCamelCase("iso_certified_staff")); System.out.println(toCamelCase("certified_staff")); System.out.println(toCamelCase("user_id")); }
From source file:de.codesourcery.geoip.Main.java
public static void main(String[] args) throws Exception { final IGeoLocator<StringSubject> locator = createGeoLocator(); final JFrame frame = new JFrame("GeoIP"); frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE); frame.addWindowListener(new WindowAdapter() { public void windowClosing(java.awt.event.WindowEvent e) { try { locator.dispose();/* w ww .j a va 2 s . c o m*/ } catch (Exception e1) { e1.printStackTrace(); } }; }); final MapImage image = MapImage.getRobinsonWorldMap(); // MapImage.getMillerWorldMap(); final MapCanvas canvas = new MapCanvas(image); for (GeoLocation<StringSubject> loc : locator.locate(getSpammers())) { if (loc.hasValidCoordinates()) { canvas.addCoordinate(PointRenderer.createPoint(loc, Color.YELLOW)); } } // canvas.addCoordinate( PointRenderer.createPoint( ZERO , Color.YELLOW ) ); // canvas.addCoordinate( PointRenderer.createPoint( WELLINGTON , Color.RED ) ); // canvas.addCoordinate( PointRenderer.createPoint( MELBOURNE , Color.RED ) ); // canvas.addCoordinate( PointRenderer.createPoint( HAMBURG , Color.RED ) ); final double heightToWidth = image.height() / (double) image.width(); // preserve aspect ratio of map canvas.setPreferredSize(new Dimension(640, (int) Math.round(640 * heightToWidth))); JPanel panel = new JPanel(); panel.setLayout(new FlowLayout()); panel.add(new JLabel("Scale-X")); final JTextField scaleX = new JTextField(Double.toString(image.getScaleX())); scaleX.setColumns(5); final JTextField scaleY = new JTextField(Double.toString(image.getScaleY())); scaleY.setColumns(5); final ActionListener listener = new ActionListener() { @Override public void actionPerformed(ActionEvent e) { double x = Double.parseDouble(scaleX.getText()); double y = Double.parseDouble(scaleY.getText()); image.setScale(x, y); canvas.repaint(); } }; scaleX.addActionListener(listener); scaleY.addActionListener(listener); panel.add(new JLabel("Scale-X")); panel.add(scaleX); panel.add(new JLabel("Scale-Y")); panel.add(scaleY); final JTextField ipAddress = new JTextField("www.kickstarter.com"); ipAddress.setColumns(20); final ActionListener ipListener = new ActionListener() { @Override public void actionPerformed(ActionEvent e) { final String destinationIP = ipAddress.getText(); if (StringUtils.isBlank(destinationIP)) { return; } /* * Perform traceroute. */ final List<String> hops; try { if (TracePath.isPathTracingAvailable()) { hops = TracePath.trace(destinationIP); } else { System.err.println("tracepath not available."); if (TracePath.isValidAddress(destinationIP)) { hops = new ArrayList<>(); hops.add(destinationIP); } else { System.err.println(destinationIP + " is no valid IP"); return; } } } catch (Exception ex) { System.err.println("Failed to trace " + destinationIP); ex.printStackTrace(); return; } System.out.println("Trace contains " + hops.size() + " IPs"); /* * Gather locations. */ final List<StringSubject> subjects = new ArrayList<>(); for (String ip : hops) { subjects.add(new StringSubject(ip)); } final List<GeoLocation<StringSubject>> locations; try { long time = -System.currentTimeMillis(); locations = locator.locate(subjects); time += System.currentTimeMillis(); System.out.println("Locating hops for " + destinationIP + " returned " + locations.size() + " valid locations ( time: " + time + " ms)"); System.out.flush(); } catch (Exception e2) { e2.printStackTrace(); return; } /* * Weed-out invalid/unknown locations. */ { GeoLocation<StringSubject> previous = null; for (Iterator<GeoLocation<StringSubject>> it = locations.iterator(); it.hasNext();) { final GeoLocation<StringSubject> location = it.next(); if (!location.hasValidCoordinates() || (previous != null && previous.coordinate().equals(location.coordinate()))) { it.remove(); System.err.println("Ignoring invalid/duplicate location for " + location); } else { previous = location; } } } /* * Populate chart. */ System.out.println("Adding " + locations.size() + " hops to chart"); System.out.flush(); canvas.removeAllCoordinates(); if (locations.size() == 1) { canvas.addCoordinate( PointRenderer.createPoint(locations.get(0), getLabel(locations.get(0)), Color.BLACK)); } else if (locations.size() > 1) { GeoLocation<StringSubject> previous = locations.get(0); MapPoint previousPoint = PointRenderer.createPoint(previous, getLabel(previous), Color.BLACK); final int len = locations.size(); for (int i = 1; i < len; i++) { final GeoLocation<StringSubject> current = locations.get(i); // final MapPoint currentPoint = PointRenderer.createPoint( current , getLabel( current ) , Color.BLACK ); final MapPoint currentPoint = PointRenderer.createPoint(current, Color.BLACK); // canvas.addCoordinate( LineRenderer.createLine( previousPoint , currentPoint , Color.RED ) ); canvas.addCoordinate(CurvedLineRenderer.createLine(previousPoint, currentPoint, Color.RED)); previous = locations.get(i); previousPoint = currentPoint; } } System.out.println("Finished adding"); System.out.flush(); canvas.repaint(); } }; ipAddress.addActionListener(ipListener); panel.add(new JLabel("IP")); panel.add(ipAddress); frame.getContentPane().setLayout(new BorderLayout()); frame.getContentPane().add(panel, BorderLayout.NORTH); frame.getContentPane().add(canvas, BorderLayout.CENTER); frame.pack(); frame.setVisible(true); }
From source file:co.mitro.core.util.RPCLogReplayer.java
public static void main(String[] args) throws IOException, InterruptedException, KeyManagementException, UnrecoverableKeyException, NoSuchAlgorithmException, KeyStoreException { List<Request> requests = new ArrayList<>(); ExecutorService executor = Executors.newFixedThreadPool(5); for (int i = 0; i < args.length; ++i) { String filename = args[i]; System.err.println("Reading file: " + filename); JsonRecordReader rr = JsonRecordReader.MakeFromFilename(filename); JsonRecordReader.JsonLog log;// ww w . j av a 2 s .co m try { while (null != (log = rr.readJson())) { if (Strings.isNullOrEmpty(log.payload.transactionId) && !log.payload.implicitBeginTransaction) { // read only transaction requests.add(new Request(log.metadata.endpoint, log.payload)); } } } catch (EOFException e) { System.err.println("unexpected end of file; skipping"); } } // run the simulation for a while. long scaling = 1000; double requestsPerMs = 353. / 9805199; long START_MS = 0; // run for 20 min long END_MS = 20 * 60 * 1000; long now = START_MS; int count = 0; while (now < END_MS) { double toSleep = nextExp(requestsPerMs * scaling); now += toSleep; ++count; Thread.sleep((long) toSleep); executor.execute(new SendQuery(requests.get(RNG.nextInt(requests.size())))); System.out.println("count: " + count + "\t time:" + now + "\t rate:" + (double) count / now); } executor.awaitTermination(1, TimeUnit.MINUTES); }
From source file:com.sds.acube.ndisc.xadmin.XNDiscAdminUtil.java
public static void main(String args[]) { if (args.length == 0) { XNDiscAdminUtil.printAdminUsage(null, null); // System.exit(0); }/*from w w w .j av a 2 s. c o m*/ XNDiscAdminFile file = new XNDiscAdminFile(printlog, out, logger); XNDiscAdminMedia media = new XNDiscAdminMedia(printlog, out, logger); XNDiscAdminVolume volume = new XNDiscAdminVolume(printlog, out, logger); XNDiscAdminEnDecrypt endecrypt = new XNDiscAdminEnDecrypt(printlog, out, logger); Scanner in = new Scanner(System.in); String input = ""; List<String> options = null; Scanner part = null; do { input = in.nextLine(); part = new Scanner(input).useDelimiter(" "); options = new ArrayList<String>(); while (part.hasNext()) { String val = part.next().trim(); if (StringUtils.isEmpty(val)) { continue; } options.add(val); } if (options.size() < 2 || input.equalsIgnoreCase("q")) { if (input != null && input.trim().equalsIgnoreCase("q")) { System.out.println(LINE_SEPERATOR + "XNDiscAdminUtil quit!!!" + LINE_SEPERATOR); } else { System.out .println(LINE_SEPERATOR + "XNDiscAdminUtil are invalid parameters!!!" + LINE_SEPERATOR); } System.exit(0); } String main_op = (StringUtils.isEmpty(options.get(0))) ? "" : options.get(0); String process_op = (StringUtils.isEmpty(options.get(1))) ? "" : options.get(1); if (main_op.equals("clear")) { if (process_op.equals("screen")) { clearConsoleOutput(); } } else if (main_op.equals("file")) { if (process_op.equals("ls")) { String fileid = ""; if (options.size() > 2) { fileid = options.get(2); } if (StringUtils.isEmpty(fileid)) { file.selectFileList(); } else { file.selectFileById(fileid); } } else if (process_op.equals("reg")) { String host = ""; int port = -1; String filepath = ""; int vol = -1; if (options.size() == 4) { host = XNDiscAdminConfig.getString(XNDiscAdminConfig.HOST); port = XNDiscAdminConfig.getInt(XNDiscAdminConfig.PORT); filepath = options.get(2); if (isInteger(options.get(3))) { vol = Integer.parseInt(options.get(3)); } } else if (options.size() > 5) { host = options.get(2); port = Integer.parseInt(options.get(3)); filepath = options.get(4); if (isInteger(options.get(5))) { vol = Integer.parseInt(options.get(5)); } } if (!StringUtils.isEmpty(host) && !StringUtils.isEmpty(filepath) && port > 0 && vol > 0) { file.regFile(host, port, filepath, vol, "0"); } else { XNDiscAdminUtil.printAdminUsage(main_op, process_op); } } else if (process_op.equals("get")) { String host = ""; int port = -1; String fileid = ""; String filepath = ""; if (options.size() == 4) { host = XNDiscAdminConfig.getString(XNDiscAdminConfig.HOST); port = XNDiscAdminConfig.getInt(XNDiscAdminConfig.PORT); fileid = options.get(2); filepath = options.get(3); } else if (options.size() > 5) { host = options.get(2); if (isInteger(options.get(3))) { port = Integer.parseInt(options.get(3)); } fileid = options.get(4); filepath = options.get(5); } if (!StringUtils.isEmpty(host) && !StringUtils.isEmpty(fileid) && !StringUtils.isEmpty(filepath) && port > 0) { file.getFile(host, port, fileid, filepath); } else { XNDiscAdminUtil.printAdminUsage(main_op, process_op); } } else if (process_op.equals("wh")) { String fileid = ""; if (options.size() > 2) { fileid = options.get(2); } if (!StringUtils.isEmpty(fileid)) { file.getFilePathByFileId(fileid); } else { XNDiscAdminUtil.printAdminUsage(main_op, process_op); } } else if (process_op.equals("rm")) { String host = ""; int port = -1; String fileid = ""; if (options.size() == 3) { host = XNDiscAdminConfig.getString(XNDiscAdminConfig.HOST); port = XNDiscAdminConfig.getInt(XNDiscAdminConfig.PORT); fileid = options.get(2); } else if (options.size() > 4) { host = options.get(2); if (isInteger(options.get(3))) { port = Integer.parseInt(options.get(3)); } fileid = options.get(4); } if (!StringUtils.isEmpty(host) && !StringUtils.isEmpty(fileid) && port > 0) { file.removeFile(host, port, fileid); } else { XNDiscAdminUtil.printAdminUsage(main_op, process_op); } } } else if (main_op.equals("media")) { if (process_op.equals("mk")) { String host = ""; int port = -1; String name = ""; int type = 0; String path = ""; String desc = " "; int maxsize = -1; int vol = -1; if (options.size() == 7) { host = XNDiscAdminConfig.getString(XNDiscAdminConfig.HOST); port = XNDiscAdminConfig.getInt(XNDiscAdminConfig.PORT); name = options.get(2); if (isInteger(options.get(3))) { type = Integer.parseInt(options.get(3)); } path = options.get(4); if (isInteger(options.get(5))) { maxsize = Integer.parseInt(options.get(5)); } if (isInteger(options.get(6))) { vol = Integer.parseInt(options.get(6)); } } else if (options.size() == 8) { host = XNDiscAdminConfig.getString(XNDiscAdminConfig.HOST); port = XNDiscAdminConfig.getInt(XNDiscAdminConfig.PORT); name = options.get(2); if (isInteger(options.get(3))) { type = Integer.parseInt(options.get(3)); } path = options.get(4); desc = options.get(5) + " "; if (isInteger(options.get(6))) { maxsize = Integer.parseInt(options.get(6)); } if (isInteger(options.get(7))) { vol = Integer.parseInt(options.get(7)); } } else if (options.size() == 9) { host = options.get(2); if (isInteger(options.get(3))) { port = Integer.parseInt(options.get(3)); } name = options.get(4); if (isInteger(options.get(5))) { type = Integer.parseInt(options.get(5)); } path = options.get(6); if (isInteger(options.get(7))) { maxsize = Integer.parseInt(options.get(7)); } if (isInteger(options.get(8))) { vol = Integer.parseInt(options.get(8)); } } else if (options.size() > 9) { host = options.get(2); if (isInteger(options.get(3))) { port = Integer.parseInt(options.get(3)); } name = options.get(4); if (isInteger(options.get(5))) { type = Integer.parseInt(options.get(5)); } path = options.get(6); desc = options.get(7) + " "; if (isInteger(options.get(8))) { maxsize = Integer.parseInt(options.get(8)); } if (isInteger(options.get(9))) { vol = Integer.parseInt(options.get(9)); } } if (!StringUtils.isEmpty(host) && !StringUtils.isEmpty(name) && !StringUtils.isEmpty(path) && port > 0 && vol > 0 && maxsize > 0) { media.makeMedia(host, port, name, type, path, desc, maxsize, vol); } else { XNDiscAdminUtil.printAdminUsage(main_op, process_op); } } else if (process_op.equals("ls")) { String mediaid = "-1"; if (options.size() > 2) { mediaid = options.get(2); } if (StringUtils.isEmpty(mediaid) || mediaid.equals("-1")) { media.selectMediaList(); } else { int id = -1; if (isInteger(mediaid)) { id = Integer.parseInt(mediaid); } if (id > 0) { media.selectMediaById(id); } else { XNDiscAdminUtil.printAdminUsage(main_op, process_op); } } } else if (process_op.equals("rm")) { String mediaid = "-1"; if (options.size() > 2) { mediaid = options.get(2); } if (!StringUtils.isEmpty(mediaid)) { int id = -1; if (isInteger(mediaid)) { id = Integer.parseInt(options.get(2)); } if (id > 0) { media.removeMedia(id); } else { XNDiscAdminUtil.printAdminUsage(main_op, process_op); } } } else if (process_op.equals("ch")) { int mediaid = -1; String name = ""; int type = 0; String path = ""; String desc = ""; int maxsize = -1; int vol = -1; if (options.size() == 8) { if (isInteger(options.get(2))) { mediaid = Integer.parseInt(options.get(2)); } name = options.get(3); if (isInteger(options.get(4))) { type = Integer.parseInt(options.get(4)); } path = options.get(5); if (isInteger(options.get(6))) { maxsize = Integer.parseInt(options.get(6)); } if (isInteger(options.get(7))) { vol = Integer.parseInt(options.get(7)); } } else if (options.size() > 8) { if (isInteger(options.get(2))) { mediaid = Integer.parseInt(options.get(2)); } name = options.get(3); if (isInteger(options.get(4))) { type = Integer.parseInt(options.get(4)); } path = options.get(5); desc = options.get(6); if (isInteger(options.get(7))) { maxsize = Integer.parseInt(options.get(7)); } if (isInteger(options.get(8))) { vol = Integer.parseInt(options.get(8)); } } if (!StringUtils.isEmpty(name) && !StringUtils.isEmpty(path) && mediaid > 0 && maxsize > 0 && vol > 0) { media.changeMedia(mediaid, name, type, path, desc, maxsize, vol); } else { XNDiscAdminUtil.printAdminUsage(main_op, process_op); } } } else if (main_op.equals("vol")) { if (process_op.equals("mk")) { String name = ""; String access = ""; String desc = " "; if (options.size() == 4) { name = options.get(2); access = options.get(3); } else if (options.size() > 4) { name = options.get(2); access = options.get(3); desc = options.get(4) + " "; } if (!StringUtils.isEmpty(name) && !StringUtils.isEmpty(access)) { volume.makeVolume(name, access, desc); } else { XNDiscAdminUtil.printAdminUsage(main_op, process_op); } } else if (process_op.equals("ls")) { String volid = "-1"; if (options.size() > 2) { volid = options.get(2); } if (StringUtils.isEmpty(volid) || volid.equals("-1")) { volume.selectVolumeList(); } else { int volumeid = -1; if (isInteger(volid)) { volumeid = Integer.parseInt(volid); } if (volumeid > 0) { volume.selectVolumeById(volumeid); } else { XNDiscAdminUtil.printAdminUsage(main_op, process_op); } } } else if (process_op.equals("rm")) { String volid = "-1"; if (options.size() > 2) { volid = options.get(2); } if (!StringUtils.isEmpty(volid)) { int volumeid = -1; if (isInteger(volid)) { volumeid = Integer.parseInt(volid); } if (volumeid > 0) { volume.removeVolume(volumeid); } } else { XNDiscAdminUtil.printAdminUsage(main_op, process_op); } } else if (process_op.equals("ch")) { int volumeid = -1; String name = ""; String access = ""; String desc = ""; if (options.size() == 5) { if (isInteger(options.get(2))) { volumeid = Integer.parseInt(options.get(2)); } name = options.get(3); access = options.get(4); } else if (options.size() > 5) { if (isInteger(options.get(2))) { volumeid = Integer.parseInt(options.get(2)); } name = options.get(3); access = options.get(4); desc = options.get(5) + " "; } if (!StringUtils.isEmpty(name) && !StringUtils.isEmpty(access) && volumeid > 0) { volume.changeVolume(volumeid, name, access, desc); } else { XNDiscAdminUtil.printAdminUsage(main_op, process_op); } } } else if (main_op.equals("id")) { String id = ""; if (options.size() > 2) { id = options.get(2); } if (process_op.equals("enc")) { if (!StringUtils.isEmpty(id)) { endecrypt.encrypt(id); } else { XNDiscAdminUtil.printAdminUsage(main_op, process_op); } } else if (process_op.equals("dec")) { if (!StringUtils.isEmpty(id)) { endecrypt.decrypt(id); } else { XNDiscAdminUtil.printAdminUsage(main_op, process_op); } } } part.close(); } while (!input.equalsIgnoreCase("q")); in.close(); }
From source file:com.finderbots.miner2.pinterest.PinterestCrawlAndMinerTool.java
public static void main(String[] args) { Options options = new Options(); CmdLineParser parser = new CmdLineParser(options); try {/* www . ja v a 2 s. co m*/ parser.parseArgument(args); } catch (CmdLineException e) { System.err.println(e.getMessage()); printUsageAndExit(parser); } // Before we get too far along, see if the domain looks valid. String domain = options.getDomain(); String urlsFile = options.getUrlsFile(); if (domain != null) { validateDomain(domain, parser); } else { if (urlsFile == null) { System.err.println( "Either a target domain should be specified or a file with a list of urls needs to be provided"); printUsageAndExit(parser); } } if (domain != null && urlsFile != null) { System.out.println("Warning: Both domain and urls file list provided - using domain"); } String outputDirName = options.getOutputDir(); if (options.isDebugLogging()) { System.setProperty("bixo.root.level", "DEBUG"); } else { System.setProperty("bixo.root.level", "INFO"); } if (options.getLoggingAppender() != null) { // Set console vs. DRFA vs. something else System.setProperty("bixo.appender", options.getLoggingAppender()); } String logsDir = options.getLogsDir(); if (!logsDir.endsWith("/")) { logsDir = logsDir + "/"; } try { JobConf conf = new JobConf(); Path outputPath = new Path(outputDirName); FileSystem fs = outputPath.getFileSystem(conf); // First check if the user wants to clean if (options.isCleanOutputDir()) { if (fs.exists(outputPath)) { fs.delete(outputPath, true); } } // See if the user isn't starting from scratch then set up the // output directory and create an initial urls subdir. if (!fs.exists(outputPath)) { fs.mkdirs(outputPath); // Create a "0-<timestamp>" sub-directory with just a /crawldb subdir // In the /crawldb dir the input file will have a single URL for the target domain. Path curLoopDir = CrawlDirUtils.makeLoopDir(fs, outputPath, 0); String curLoopDirName = curLoopDir.getName(); setLoopLoggerFile(logsDir + curLoopDirName, 0); Path crawlDbPath = new Path(curLoopDir, CrawlConfig.CRAWLDB_SUBDIR_NAME); if (domain != null) { importOneDomain(domain, crawlDbPath, conf); } else { importUrls(urlsFile, crawlDbPath); } } Path latestDirPath = CrawlDirUtils.findLatestLoopDir(fs, outputPath); if (latestDirPath == null) { System.err.println("No previous cycle output dirs exist in " + outputDirName); printUsageAndExit(parser); } Path crawlDbPath = new Path(latestDirPath, CrawlConfig.CRAWLDB_SUBDIR_NAME); // Set up the start and end loop counts. int startLoop = CrawlDirUtils.extractLoopNumber(latestDirPath); int endLoop = startLoop + options.getNumLoops(); // Set up the UserAgent for the fetcher. UserAgent userAgent = new UserAgent(options.getAgentName(), CrawlConfig.EMAIL_ADDRESS, CrawlConfig.WEB_ADDRESS); // You also get to customize the FetcherPolicy FetcherPolicy defaultPolicy; if (options.getCrawlDuration() != 0) { defaultPolicy = new AdaptiveFetcherPolicy(options.getEndCrawlTime(), options.getCrawlDelay()); } else { defaultPolicy = new FetcherPolicy(); } defaultPolicy.setMaxContentSize(CrawlConfig.MAX_CONTENT_SIZE); defaultPolicy.setRequestTimeout(10L * 1000L);//10 seconds // COMPLETE for crawling a single site, EFFICIENT for many sites if (options.getCrawlPolicy().equals(Options.IMPOLITE_CRAWL_POLICY)) { defaultPolicy.setFetcherMode(FetcherPolicy.FetcherMode.IMPOLITE); } else if (options.getCrawlPolicy().equals(Options.EFFICIENT_CRAWL_POLICY)) { defaultPolicy.setFetcherMode(FetcherPolicy.FetcherMode.EFFICIENT); } else if (options.getCrawlPolicy().equals(Options.COMPLETE_CRAWL_POLICY)) { defaultPolicy.setFetcherMode(FetcherPolicy.FetcherMode.COMPLETE); } // It is a good idea to set up a crawl duration when running long crawls as you may // end up in situations where the fetch slows down due to a 'long tail' and by // specifying a crawl duration you know exactly when the crawl will end. int crawlDurationInMinutes = options.getCrawlDuration(); boolean hasEndTime = crawlDurationInMinutes != Options.NO_CRAWL_DURATION; long targetEndTime = hasEndTime ? System.currentTimeMillis() + (crawlDurationInMinutes * CrawlConfig.MILLISECONDS_PER_MINUTE) : FetcherPolicy.NO_CRAWL_END_TIME; // By setting up a url filter we only deal with urls that we want to // instead of all the urls that we extract. BaseUrlFilter urlFilter = null; List<String> patterns = null; String regexUrlFiltersFile = options.getRegexUrlFiltersFile(); if (regexUrlFiltersFile != null) { patterns = RegexUrlDatumFilter.getUrlFilterPatterns(regexUrlFiltersFile); } else { patterns = RegexUrlDatumFilter.getDefaultUrlFilterPatterns(); if (domain != null) { String domainPatterStr = "+(?i)^(http|https)://([a-z0-9]*\\.)*" + domain; patterns.add(domainPatterStr); } else { String protocolPatterStr = "+(?i)^(http|https)://*"; patterns.add(protocolPatterStr); //Log.warn("Defaulting to basic url regex filtering (just suffix and protocol"); } } urlFilter = new RegexUrlDatumFilter(patterns.toArray(new String[patterns.size()])); // get a list of patterns which tell the miner which URLs to include or exclude. patterns.clear(); RegexUrlStringFilter urlsToMineFilter = null; String regexUrlsToMineFiltersFile = options.getRegexUrlToMineFile(); AnalyzeHtml analyzer = null; if (regexUrlsToMineFiltersFile != null) { patterns = RegexUrlDatumFilter.getUrlFilterPatterns(regexUrlsToMineFiltersFile); urlsToMineFilter = new RegexUrlStringFilter(patterns.toArray(new String[patterns.size()])); analyzer = new AnalyzeHtml(urlsToMineFilter); } // OK, now we're ready to start looping, since we've got our current // settings for (int curLoop = startLoop + 1; curLoop <= endLoop; curLoop++) { // Adjust target end time, if appropriate. if (hasEndTime) { int remainingLoops = (endLoop - curLoop) + 1; long now = System.currentTimeMillis(); long perLoopTime = (targetEndTime - now) / remainingLoops; defaultPolicy.setCrawlEndTime(now + perLoopTime); } Path curLoopDirPath = CrawlDirUtils.makeLoopDir(fs, outputPath, curLoop); String curLoopDirName = curLoopDirPath.getName(); setLoopLoggerFile(logsDir + curLoopDirName, curLoop); Flow flow = PinterestCrawlAndMinerWorkflow.createFlow(curLoopDirPath, crawlDbPath, defaultPolicy, userAgent, urlFilter, analyzer, options); flow.complete(); // Writing out .dot files is a good way to verify your flows. flow.writeDOT("valid-flow.dot"); // Update crawlDbPath to point to the latest crawl db crawlDbPath = new Path(curLoopDirPath, CrawlConfig.CRAWLDB_SUBDIR_NAME); } } catch (PlannerException e) { e.writeDOT("failed-flow.dot"); System.err.println("PlannerException: " + e.getMessage()); e.printStackTrace(System.err); System.exit(-1); } catch (Throwable t) { System.err.println("Exception running tool: " + t.getMessage()); t.printStackTrace(System.err); System.exit(-1); } }