List of usage examples for java.lang Thread Thread
public Thread()
From source file:com.johnson.grab.browser.HttpClientUtil.java
public static void main(String[] args) throws IOException, CrawlException { // final String url = "http://192.168.24.248:8080/HbaseDb/youku/"; // String url = "http://business.sohu.com/20131021/n388557348.shtml?pvid=tc_business&a=&b=%E6%A5%BC%E5%B8%82%E6%B3%A1%E6%B2%AB%E7%A0%B4%E7%81%AD%E5%B0%86%E5%9C%A82015%E5%B9%B4%E5%BA%95%E4%B9%8B%E5%89%8D"; final String url = "http://www.sohu.com"; final int threadNum = 20; final int loop = 100; Thread[] threads = new Thread[threadNum]; final List<Integer> times = new ArrayList<Integer>(); final long s = System.currentTimeMillis(); for (int i = 0; i < threads.length; i++) { threads[i] = new Thread() { public void run() { for (int i = 0; i < loop; i++) { try { getContent(url); } catch (CrawlException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } catch (Throwable t) { t.printStackTrace(); }// w w w. j ava 2s . c o m long e = System.currentTimeMillis(); times.add((int) (e - s)); } } }; threads[i].start(); } while (times.size() < threadNum * loop) { int current = times.size(); System.out.println("total: " + threadNum * loop + ", current: " + current + ", left: " + (threadNum * loop - current)); try { Thread.sleep(1000); } catch (InterruptedException e) { e.printStackTrace(); } } long e = System.currentTimeMillis(); int totalTime = 0; for (Integer time : times) { totalTime += time; } System.out.println("------------------------------------------------------"); System.out.println("thread num: " + threadNum + ", loop: " + loop); System.out.println("totalTime: " + totalTime + ", averTime: " + totalTime / (threadNum * loop)); System.out.println("finalTime: " + (e - s) + ", throughput: " + (e - s) / (threadNum * loop)); }
From source file:com.amazonaws.services.kinesis.leases.impl.LeaseCoordinatorExerciser.java
public static void main(String[] args) throws InterruptedException, DependencyException, InvalidStateException, ProvisionedThroughputException, IOException { int numCoordinators = 9; int numLeases = 73; int leaseDurationMillis = 10000; int epsilonMillis = 100; AWSCredentialsProvider creds = new DefaultAWSCredentialsProviderChain(); AmazonDynamoDBClient ddb = new AmazonDynamoDBClient(creds); ILeaseManager<KinesisClientLease> leaseManager = new KinesisClientLeaseManager("nagl_ShardProgress", ddb); if (leaseManager.createLeaseTableIfNotExists(10L, 50L)) { LOG.info("Waiting for newly created lease table"); if (!leaseManager.waitUntilLeaseTableExists(10, 300)) { LOG.error("Table was not created in time"); return; }//w w w. j a v a2 s . co m } CWMetricsFactory metricsFactory = new CWMetricsFactory(creds, "testNamespace", 30 * 1000, 1000); final List<LeaseCoordinator<KinesisClientLease>> coordinators = new ArrayList<LeaseCoordinator<KinesisClientLease>>(); for (int i = 0; i < numCoordinators; i++) { String workerIdentifier = "worker-" + Integer.toString(i); LeaseCoordinator<KinesisClientLease> coord = new LeaseCoordinator<KinesisClientLease>(leaseManager, workerIdentifier, leaseDurationMillis, epsilonMillis, metricsFactory); coordinators.add(coord); } leaseManager.deleteAll(); for (int i = 0; i < numLeases; i++) { KinesisClientLease lease = new KinesisClientLease(); lease.setLeaseKey(Integer.toString(i)); lease.setCheckpoint(new ExtendedSequenceNumber("checkpoint")); leaseManager.createLeaseIfNotExists(lease); } final JFrame frame = new JFrame("Test Visualizer"); frame.setPreferredSize(new Dimension(800, 600)); final JPanel panel = new JPanel(new GridLayout(coordinators.size() + 1, 0)); final JLabel ticker = new JLabel("tick"); panel.add(ticker); frame.getContentPane().add(panel); final Map<String, JLabel> labels = new HashMap<String, JLabel>(); for (final LeaseCoordinator<KinesisClientLease> coord : coordinators) { JPanel coordPanel = new JPanel(); coordPanel.setLayout(new BoxLayout(coordPanel, BoxLayout.X_AXIS)); final Button button = new Button("Stop " + coord.getWorkerIdentifier()); button.setMaximumSize(new Dimension(200, 50)); button.addActionListener(new ActionListener() { @Override public void actionPerformed(ActionEvent arg0) { if (coord.isRunning()) { coord.stop(); button.setLabel("Start " + coord.getWorkerIdentifier()); } else { try { coord.start(); } catch (LeasingException e) { LOG.error(e); } button.setLabel("Stop " + coord.getWorkerIdentifier()); } } }); coordPanel.add(button); JLabel label = new JLabel(); coordPanel.add(label); labels.put(coord.getWorkerIdentifier(), label); panel.add(coordPanel); } frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE); new Thread() { // Key is lease key, value is green-ness as a value from 0 to 255. // Great variable name, huh? private Map<String, Integer> greenNesses = new HashMap<String, Integer>(); // Key is lease key, value is last owning worker private Map<String, String> lastOwners = new HashMap<String, String>(); @Override public void run() { while (true) { for (LeaseCoordinator<KinesisClientLease> coord : coordinators) { String workerIdentifier = coord.getWorkerIdentifier(); JLabel label = labels.get(workerIdentifier); List<KinesisClientLease> asgn = new ArrayList<KinesisClientLease>(coord.getAssignments()); Collections.sort(asgn, new Comparator<KinesisClientLease>() { @Override public int compare(KinesisClientLease arg0, KinesisClientLease arg1) { return arg0.getLeaseKey().compareTo(arg1.getLeaseKey()); } }); StringBuilder builder = new StringBuilder(); builder.append("<html>"); builder.append(workerIdentifier).append(":").append(asgn.size()).append(" "); for (KinesisClientLease lease : asgn) { String leaseKey = lease.getLeaseKey(); String lastOwner = lastOwners.get(leaseKey); // Color things green when they switch owners, decay the green-ness over time. Integer greenNess = greenNesses.get(leaseKey); if (greenNess == null || lastOwner == null || !lastOwner.equals(lease.getLeaseOwner())) { greenNess = 200; } else { greenNess = Math.max(0, greenNess - 20); } greenNesses.put(leaseKey, greenNess); lastOwners.put(leaseKey, lease.getLeaseOwner()); builder.append(String.format("<font color=\"%s\">%03d</font>", String.format("#00%02x00", greenNess), Integer.parseInt(leaseKey))).append(" "); } builder.append("</html>"); label.setText(builder.toString()); label.revalidate(); label.repaint(); } if (ticker.getText().equals("tick")) { ticker.setText("tock"); } else { ticker.setText("tick"); } try { Thread.sleep(200); } catch (InterruptedException e) { } } } }.start(); frame.pack(); frame.setVisible(true); for (LeaseCoordinator<KinesisClientLease> coord : coordinators) { coord.start(); } }
From source file:de.tbuchloh.kiskis.KisKis.java
public static void main(final String[] args) { logSystemInfo();/* w w w .ja v a2s . co m*/ final StopWatch w = new StopWatch(); initThreadMonitors(); initPreferences(); final MainFrame wnd = MainFrame.getInstance(); LOG.info("Window created in: " + w.getDuration()); final LongOpt[] longs = { new LongOpt("help", LongOpt.NO_ARGUMENT, null, '?'), new LongOpt("file", LongOpt.REQUIRED_ARGUMENT, null, 'f'), new LongOpt("validate", LongOpt.NO_ARGUMENT, null, 'v'), new LongOpt("hide", LongOpt.NO_ARGUMENT, null, 'h'), new LongOpt("installCracklibDict", LongOpt.NO_ARGUMENT, null, 'i'), new LongOpt("reset", LongOpt.NO_ARGUMENT, null, 'r'), new LongOpt("selftest", LongOpt.NO_ARGUMENT, null, 's'), new LongOpt("lastFile", LongOpt.NO_ARGUMENT, null, 'l') }; final Getopt g = new Getopt("KisKis", args, "", longs, true); int c = 0; boolean show = true; while ((c = g.getopt()) != -1) { switch (c) { case '?': showHelp(); System.exit(1); break; case 'f': // we need to show the window wnd.setVisible(true); wnd.loadFile(new File(g.getOptarg())); break; case 'h': show = false; break; case 'i': installCracklibDict(); break; case 'v': BuildProperties.VALIDATE_DOCS = true; break; case 'r': resetPreferences(); System.exit(0); break; case 's': wnd.showSelftest(); break; case 'l': // we need to show the window wnd.setVisible(true); wnd.loadLastFile(); break; default: showHelp(); System.exit(1); } } Runtime.getRuntime().addShutdownHook(new Thread() { @Override public void run() { if (Settings.isExportingPrefs()) { try { exportPreferences(); } catch (final KisKisException e) { MessageBox.showException(e); } } } }); wnd.showTray(); wnd.setVisible(show); wnd.showTipOfTheDay(); }
From source file:com.boulmier.machinelearning.jobexecutor.JobExecutor.java
public static void main(String[] args) throws ParseException, IOException, InterruptedException { Options options = defineOptions();// ww w. j av a 2s. com sysMon = new JavaSysMon(); InetAddress vmscheduler_ip, mongodb_ip = null; Integer vmscheduler_port = null, mongodb_port = null; CommandLineParser parser = new BasicParser(); try { CommandLine cmd = parser.parse(options, args); if (cmd.hasOption(JobExecutorConfig.OPTIONS.CMD.LONGPORTFIELD)) { vmscheduler_port = Integer.valueOf(cmd.getOptionValue(JobExecutorConfig.OPTIONS.CMD.LONGPORTFIELD)); } mongodb_port = (int) (cmd.hasOption(JobExecutorConfig.OPTIONS.CMD.LONGMONGOPORTFIELD) ? cmd.hasOption(JobExecutorConfig.OPTIONS.CMD.LONGMONGOPORTFIELD) : JobExecutorConfig.OPTIONS.LOGGING.MONGO_DEFAULT_PORT); mongodb_ip = InetAddress.getByName(cmd.getOptionValue(JobExecutorConfig.OPTIONS.CMD.LONGMONGOIPFIELD)); vmscheduler_ip = InetAddress.getByName(cmd.getOptionValue(JobExecutorConfig.OPTIONS.CMD.LONGIPFIELD)); decryptKey = cmd.getOptionValue("decrypt-key"); debugState = cmd.hasOption(JobExecutorConfig.OPTIONS.CMD.LONGDEBUGFIELD); logger = LoggerFactory.getLogger(); logger.info("Attempt to connect on master @" + vmscheduler_ip + ":" + vmscheduler_port); new RequestConsumer().start(); } catch (MissingOptionException moe) { logger.error(moe.getMissingOptions() + " are missing"); HelpFormatter help = new HelpFormatter(); help.printHelp(JobExecutor.class.getSimpleName(), options); } catch (UnknownHostException ex) { logger.error(ex.getMessage()); } finally { Runtime.getRuntime().addShutdownHook(new Thread() { @Override public void run() { logger.info("JobExeutor is shutting down"); } }); } }
From source file:it.polito.tellmefirst.web.rest.TMFServer.java
/** * TMF starting point. From rest directory, launch this command: * mvn exec:java -Dexec.mainClass="it.polito.temefirst.web.rest.TMFServer" -Dexec.args="<path_to_TMF_installation>/conf/server.properties" * or use the run.sh file in bin directory *//*from www .j a v a2 s . c om*/ public static void main(String[] args) throws TMFConfigurationException, TMFIndexesWarmUpException, URISyntaxException, InterruptedException, IOException { LOG.debug("[main] - BEGIN"); URI serverURI = new URI("http://localhost:2222/rest/"); String configFileName = args[0]; new TMFVariables(configFileName); // XXX I put the code of IndexUtil.init() here, because, for now, I need a reference of SimpleSearchers for the Enhancer // build italian searcher Directory contextIndexDirIT = LuceneManager.pickDirectory(new File(TMFVariables.CORPUS_INDEX_IT)); LOG.info("Corpus index used for italian: " + contextIndexDirIT); LuceneManager contextLuceneManagerIT = new LuceneManager(contextIndexDirIT); contextLuceneManagerIT .setLuceneDefaultAnalyzer(new ItalianAnalyzer(Version.LUCENE_36, TMFVariables.STOPWORDS_IT)); ITALIAN_CORPUS_INDEX_SEARCHER = new SimpleSearcher(contextLuceneManagerIT); // build english searcher Directory contextIndexDirEN = LuceneManager.pickDirectory(new File(TMFVariables.CORPUS_INDEX_EN)); LOG.info("Corpus index used for english: " + contextIndexDirEN); LuceneManager contextLuceneManagerEN = new LuceneManager(contextIndexDirEN); contextLuceneManagerEN .setLuceneDefaultAnalyzer(new EnglishAnalyzer(Version.LUCENE_36, TMFVariables.STOPWORDS_EN)); ENGLISH_CORPUS_INDEX_SEARCHER = new SimpleSearcher(contextLuceneManagerEN); // build kb italian searcher String kbDirIT = TMFVariables.KB_IT; String residualKbDirIT = TMFVariables.RESIDUAL_KB_IT; ITALIAN_KB_INDEX_SEARCHER = new KBIndexSearcher(kbDirIT, residualKbDirIT); // build kb english searcher String kbDirEN = TMFVariables.KB_EN; String residualKbDirEN = TMFVariables.RESIDUAL_KB_EN; ENGLISH_KB_INDEX_SEARCHER = new KBIndexSearcher(kbDirEN, residualKbDirEN); enhancer = new Enhancer(ITALIAN_CORPUS_INDEX_SEARCHER, ENGLISH_CORPUS_INDEX_SEARCHER, ITALIAN_KB_INDEX_SEARCHER, ENGLISH_KB_INDEX_SEARCHER); italianClassifier = new Classifier("it", ITALIAN_CORPUS_INDEX_SEARCHER); englishClassifier = new Classifier("en", ENGLISH_CORPUS_INDEX_SEARCHER); //The following is adapted from DBpedia Spotlight (https://github.com/dbpedia-spotlight/dbpedia-spotlight) final Map<String, String> initParams = new HashMap<String, String>(); initParams.put("com.sun.jersey.config.property.resourceConfigClass", "com.sun.jersey.api.core." + "PackagesResourceConfig"); initParams.put("com.sun.jersey.config.property.packages", "it.polito.tellmefirst.web.rest.services"); initParams.put("com.sun.jersey.config.property.WadlGeneratorConfig", "it.polito.tellmefirst.web.rest.wadl." + "ExternalUriWadlGeneratorConfig"); SelectorThread threadSelector = GrizzlyWebContainerFactory.create(serverURI, initParams); threadSelector.start(); System.err.println("Server started in " + System.getProperty("user.dir") + " listening on " + serverURI); Thread warmUp = new Thread() { public void run() { } }; warmUp.start(); while (running) { Thread.sleep(100); } threadSelector.stopEndpoint(); System.exit(0); LOG.debug("[main] - END"); }
From source file:com.github.brandtg.switchboard.MysqlReplicator.java
/** Main. */ public static void main(String[] args) throws Exception { Options options = new Options(); options.addOption("u", "user", true, "MySQL user"); options.addOption("p", "password", true, "MySQL password"); options.addOption("h", "host", true, "MySQL host"); options.addOption("P", "port", true, "MySQL port"); options.addOption("s", "sinkPort", true, "Local sink port"); options.addOption("l", "lastIndex", true, "Last transaction ID"); options.addOption("h", "help", false, "Prints help message"); CommandLine commandLine = new GnuParser().parse(options, args); if (commandLine.getArgs().length < 2 || commandLine.hasOption("help")) { HelpFormatter helpFormatter = new HelpFormatter(); helpFormatter.printHelp("usage: [opts] switchboardHost:port db1 [db2 ...]", options); System.exit(1);/*from w w w . java 2 s.c o m*/ } // Switchboard host String[] hostPort = commandLine.getArgs()[0].split(":"); InetSocketAddress source = new InetSocketAddress(hostPort[0], Integer.valueOf(hostPort[1])); InetSocketAddress sink = new InetSocketAddress( Integer.valueOf(commandLine.getOptionValue("sinkPort", "9090"))); // Databases to replicate String[] databases = Arrays.copyOfRange(commandLine.getArgs(), 1, commandLine.getArgs().length); // JDBC params for local copy String user = commandLine.getOptionValue("user", "root"); String password = commandLine.getOptionValue("password", ""); String jdbcString = String.format("jdbc:mysql://%s:%d", commandLine.getOptionValue("host", "localhost"), Integer.valueOf(commandLine.getOptionValue("port", "3306"))); Long lastIndex = Long.valueOf(commandLine.getOptionValue("lastIndex", "-1")); // Create replicators final List<MysqlReplicator> replicators = new ArrayList<>(); for (String database : databases) { MysqlReplicator replicator = new MysqlReplicator(database, source, sink, jdbcString, user, password, lastIndex); replicators.add(replicator); } // Shutdown hook Runtime.getRuntime().addShutdownHook(new Thread() { @Override public void run() { for (MysqlReplicator replicator : replicators) { try { replicator.shutdown(); } catch (Exception e) { LOG.error("Could not shut down {}", replicator, e); } } } }); for (MysqlReplicator replicator : replicators) { replicator.start(); LOG.info("Started {}", replicator); } }
From source file:com.norconex.jefmon.server.JEFMonServer.java
public static void main(String[] args) throws Exception { Properties props = getSetupProperties(); String localesString = props.getString("locales"); String[] localeStrings = localesString.split(","); Locale[] locales = new Locale[localeStrings.length]; for (int i = 0; i < localeStrings.length; i++) { locales[i] = LocaleUtils.toLocale(localeStrings[i].trim()); }//from ww w .j a va 2 s . com final JEFMonServer server = new JEFMonServer(props.getInt("port", 80), props.getBoolean("https", false), locales); Runtime.getRuntime().addShutdownHook(new Thread() { @Override public void run() { try { server.stop(); } catch (Exception e) { e.printStackTrace(); } } }); server.run(); }
From source file:com.meidusa.venus.benchmark.FileLineRandomData.java
public static void main(String[] args) throws Exception { final FileLineRandomData mapping = new FileLineRandomData(); mapping.setFile(new File("./role.txt")); mapping.init();// ww w . java 2 s . co m List<Thread> list = new ArrayList<Thread>(); long start = TimeUtil.currentTimeMillis(); for (int j = 0; j < 1; j++) { Thread thread = new Thread() { public void run() { for (int i = 0; i < 1000; i++) { System.out.println(((String[]) mapping.nextData())[1]); } } }; list.add(thread); thread.start(); } for (int i = 0; i < list.size(); i++) { list.get(i).join(); } System.out.println("time=" + (TimeUtil.currentTimeMillis() - start)); }
From source file:io.anserini.index.IndexTweetsUpdatePlace.java
@SuppressWarnings("static-access") public static void main(String[] args) throws Exception { Options options = new Options(); options.addOption(new Option(HELP_OPTION, "show help")); options.addOption(new Option(OPTIMIZE_OPTION, "merge indexes into a single segment")); options.addOption(new Option(STORE_TERM_VECTORS_OPTION, "store term vectors")); options.addOption(OptionBuilder.withArgName("collection").hasArg() .withDescription("source collection directory").create(COLLECTION_OPTION)); options.addOption(// w w w .ja v a2s.c o m OptionBuilder.withArgName("dir").hasArg().withDescription("index location").create(INDEX_OPTION)); options.addOption(OptionBuilder.withArgName("file").hasArg().withDescription("file with deleted tweetids") .create(DELETES_OPTION)); options.addOption(OptionBuilder.withArgName("id").hasArg().withDescription("max id").create(MAX_ID_OPTION)); CommandLine cmdline = null; CommandLineParser parser = new GnuParser(); try { cmdline = parser.parse(options, args); } catch (ParseException exp) { System.err.println("Error parsing command line: " + exp.getMessage()); System.exit(-1); } if (cmdline.hasOption(HELP_OPTION) || !cmdline.hasOption(COLLECTION_OPTION) || !cmdline.hasOption(INDEX_OPTION)) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp(IndexTweetsUpdatePlace.class.getName(), options); System.exit(-1); } String collectionPath = cmdline.getOptionValue(COLLECTION_OPTION); String indexPath = cmdline.getOptionValue(INDEX_OPTION); System.out.println(collectionPath + " " + indexPath); LOG.info("collection: " + collectionPath); LOG.info("index: " + indexPath); long startTime = System.currentTimeMillis(); File file = new File(collectionPath); if (!file.exists()) { System.err.println("Error: " + file + " does not exist!"); System.exit(-1); } final FieldType textOptions = new FieldType(); textOptions.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); textOptions.setStored(true); textOptions.setTokenized(true); if (cmdline.hasOption(STORE_TERM_VECTORS_OPTION)) { textOptions.setStoreTermVectors(true); } final StatusStream stream = new JsonStatusCorpusReader(file); final Directory dir = new SimpleFSDirectory(Paths.get(cmdline.getOptionValue(INDEX_OPTION))); final IndexWriterConfig config = new IndexWriterConfig(ANALYZER); config.setOpenMode(IndexWriterConfig.OpenMode.APPEND); final IndexWriter writer = new IndexWriter(dir, config); System.out.print("Original # of docs " + writer.numDocs()); int updateCount = 0; Runtime.getRuntime().addShutdownHook(new Thread() { public void run() { try { stream.close(); } catch (IOException e1) { // TODO Auto-generated catch block e1.printStackTrace(); } try { writer.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } try { dir.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } System.out.println("Shutting down"); } }); int cnt = 0; Status status; try { while ((status = stream.next()) != null) { if (status.getPlace() != null) { // Query q = NumericRangeQuery.newLongRange(TweetStreamReader.StatusField.ID.name, status.getId(), // status.getId(), true, true); // System.out.print("Deleting docCount="+writer.numDocs()); // writer.deleteDocuments(q); // writer.commit(); // System.out.print(" Deleted docCount="+writer.numDocs()); Document doc = new Document(); doc.add(new LongField(StatusField.ID.name, status.getId(), Field.Store.YES)); doc.add(new LongField(StatusField.EPOCH.name, status.getEpoch(), Field.Store.YES)); doc.add(new TextField(StatusField.SCREEN_NAME.name, status.getScreenname(), Store.YES)); doc.add(new Field(StatusField.TEXT.name, status.getText(), textOptions)); doc.add(new IntField(StatusField.FRIENDS_COUNT.name, status.getFollowersCount(), Store.YES)); doc.add(new IntField(StatusField.FOLLOWERS_COUNT.name, status.getFriendsCount(), Store.YES)); doc.add(new IntField(StatusField.STATUSES_COUNT.name, status.getStatusesCount(), Store.YES)); doc.add(new DoubleField(StatusField.LONGITUDE.name, status.getLongitude(), Store.YES)); doc.add(new DoubleField(StatusField.LATITUDE.name, status.getlatitude(), Store.YES)); doc.add(new StringField(StatusField.PLACE.name, status.getPlace(), Store.YES)); long inReplyToStatusId = status.getInReplyToStatusId(); if (inReplyToStatusId > 0) { doc.add(new LongField(StatusField.IN_REPLY_TO_STATUS_ID.name, inReplyToStatusId, Field.Store.YES)); doc.add(new LongField(StatusField.IN_REPLY_TO_USER_ID.name, status.getInReplyToUserId(), Field.Store.YES)); } String lang = status.getLang(); if (!lang.equals("unknown")) { doc.add(new TextField(StatusField.LANG.name, status.getLang(), Store.YES)); } long retweetStatusId = status.getRetweetedStatusId(); if (retweetStatusId > 0) { doc.add(new LongField(StatusField.RETWEETED_STATUS_ID.name, retweetStatusId, Field.Store.YES)); doc.add(new LongField(StatusField.RETWEETED_USER_ID.name, status.getRetweetedUserId(), Field.Store.YES)); doc.add(new IntField(StatusField.RETWEET_COUNT.name, status.getRetweetCount(), Store.YES)); if (status.getRetweetCount() < 0 || status.getRetweetedStatusId() < 0) { LOG.warn("Error parsing retweet fields of " + status.getId()); } } long id = status.getId(); BytesRefBuilder brb = new BytesRefBuilder(); NumericUtils.longToPrefixCodedBytes(id, 0, brb); Term term = new Term(StatusField.ID.name, brb.get()); writer.updateDocument(term, doc); // writer.addDocument(doc); updateCount += 1; if (updateCount % 10000 == 0) { LOG.info(updateCount + " statuses updated"); writer.commit(); System.out.println("Updated docCount=" + writer.numDocs()); } } } LOG.info("Total elapsed time: " + (System.currentTimeMillis() - startTime) + "ms"); } catch (Exception e) { e.printStackTrace(); } finally { writer.close(); dir.close(); stream.close(); } }
From source file:io.anserini.index.UserPostFrequencyDistribution.java
@SuppressWarnings("static-access") public static void main(String[] args) throws Exception { Options options = new Options(); options.addOption(new Option(HELP_OPTION, "show help")); options.addOption(new Option(STORE_TERM_VECTORS_OPTION, "store term vectors")); options.addOption(OptionBuilder.withArgName("collection").hasArg() .withDescription("source collection directory").create(COLLECTION_OPTION)); options.addOption(OptionBuilder.withArgName("property").hasArg() .withDescription("source collection directory").create("property")); options.addOption(OptionBuilder.withArgName("collection_pattern").hasArg() .withDescription("source collection directory").create("collection_pattern")); CommandLine cmdline = null;/*from ww w .j av a2 s . c o m*/ CommandLineParser parser = new GnuParser(); try { cmdline = parser.parse(options, args); } catch (ParseException exp) { System.err.println("Error parsing command line: " + exp.getMessage()); System.exit(-1); } if (cmdline.hasOption(HELP_OPTION) || !cmdline.hasOption(COLLECTION_OPTION)) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp(UserPostFrequencyDistribution.class.getName(), options); System.exit(-1); } String collectionPath = cmdline.getOptionValue(COLLECTION_OPTION); final FieldType textOptions = new FieldType(); textOptions.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); textOptions.setStored(true); textOptions.setTokenized(true); textOptions.setStoreTermVectors(true); LOG.info("collection: " + collectionPath); LOG.info("collection_pattern " + cmdline.getOptionValue("collection_pattern")); LOG.info("property " + cmdline.getOptionValue("property")); LongOpenHashSet deletes = null; long startTime = System.currentTimeMillis(); File file = new File(collectionPath); if (!file.exists()) { System.err.println("Error: " + file + " does not exist!"); System.exit(-1); } final JsonStatusCorpusReader stream = new JsonStatusCorpusReader(file, cmdline.getOptionValue("collection_pattern")); Runtime.getRuntime().addShutdownHook(new Thread() { public void run() { try { stream.close(); } catch (IOException e1) { // TODO Auto-generated catch block e1.printStackTrace(); } ; System.out.println("# of users indexed this round: " + userIndexedCount); System.out.println("Shutting down"); } }); Status status; boolean readerNotInitialized = true; try { Properties prop = new Properties(); while ((status = stream.next()) != null) { // try{ // status = DataObjectFactory.createStatus(s); // if (status==null||status.getText() == null) { // continue; // }}catch(Exception e){ // // } // boolean pittsburghRelated = false; try { if (Math.abs(status.getLongitude() - pittsburghLongitude) < 0.05d && Math.abs(status.getlatitude() - pittsburghLatitude) < 0.05d) pittsburghRelated = true; } catch (Exception e) { } try { if (status.getPlace().contains("Pittsburgh, PA")) pittsburghRelated = true; } catch (Exception e) { } try { if (status.getUserLocation().contains("Pittsburgh, PA")) pittsburghRelated = true; } catch (Exception e) { } try { if (status.getText().contains("Pittsburgh")) pittsburghRelated = true; } catch (Exception e) { } if (pittsburghRelated) { int previousPostCount = 0; if (prop.containsKey(String.valueOf(status.getUserid()))) { previousPostCount = Integer .valueOf(prop.getProperty(String.valueOf(status.getUserid())).split(" ")[1]); } prop.setProperty(String.valueOf(status.getUserid()), String.valueOf(status.getStatusesCount()) + " " + (1 + previousPostCount)); if (prop.size() > 0 && prop.size() % 1000 == 0) { Runtime runtime = Runtime.getRuntime(); runtime.gc(); System.out.println("Property size " + prop.size() + "Memory used: " + ((runtime.totalMemory() - runtime.freeMemory()) / (1024L * 1024L)) + " MB\n"); } OutputStream output = new FileOutputStream(cmdline.getOptionValue("property"), false); prop.store(output, null); output.close(); } } // prop.store(output, null); LOG.info(String.format("Total of %s statuses added", userIndexedCount)); LOG.info("Total elapsed time: " + (System.currentTimeMillis() - startTime) + "ms"); } catch (Exception e) { e.printStackTrace(); } finally { stream.close(); } }