List of usage examples for java.util.concurrent TimeUnit SECONDS
TimeUnit SECONDS
To view the source code for java.util.concurrent TimeUnit SECONDS.
Click Source Link
From source file:avantssar.aslanpp.testing.Tester.java
public static void main(String[] args) { Debug.initLog(LogLevel.INFO);//from www .j a v a 2 s .c om TesterCommandLineOptions options = new TesterCommandLineOptions(); try { options.getParser().parseArgument(args); options.ckeckAtEnd(); } catch (CmdLineException ex) { reportException("Inconsistent options.", ex, System.err); options.showShortHelp(System.err); return; } if (options.isShowHelp()) { options.showLongHelp(System.out); return; } ASLanPPConnectorImpl translator = new ASLanPPConnectorImpl(); if (options.isShowVersion()) { System.out.println(translator.getFullTitleLine()); return; } ISpecificationBundleProvider sbp; File realInDir; if (options.isLibrary()) { if (options.getHornClausesLevel() != HornClausesLevel.ALL) { System.out.println("When checking the internal library we output all Horn clauses."); options.setHornClausesLevel(HornClausesLevel.ALL); } if (!options.isStripOutput()) { System.out.println( "When checking the internal library, the ouput is stripped of comments and line information."); options.setStripOutput(true); } File modelsDir = new File(FilenameUtils.concat(options.getOut().getAbsolutePath(), "_models")); try { FileUtils.forceMkdir(modelsDir); } catch (IOException e1) { System.out.println("Failed to create models folder: " + e1.getMessage()); Debug.logger.error("Failed to create models folder.", e1); } realInDir = modelsDir; sbp = new LibrarySpecificationsProvider(modelsDir.getAbsolutePath()); } else { realInDir = options.getIn(); sbp = new DiskSpecificationsProvider(options.getIn().getAbsolutePath()); } System.setProperty(EntityManager.ASLAN_ENVVAR, sbp.getASLanPath()); // try { // EntityManager.loadASLanPath(); // } // catch (IOException e) { // System.out.println("Exception while reloading ASLANPATH: " + // e.getMessage()); // Debug.logger.error("Exception while loading ASLANPATH.", e); // } try { bm = BackendsManager.instance(); if (bm != null) { for (IBackendRunner br : bm.getBackendRunners()) { System.out.println(br.getFullDescription()); if (br.getTimeout() > finalTimeout) { finalTimeout = br.getTimeout(); } } } } catch (IOException e) { System.out.println("Failed to load backends: " + e); } int threadsCount = 50; if (options.getThreads() > 0) { threadsCount = options.getThreads(); } System.out.println("Will launch " + threadsCount + " threads in parallel (+ will show that a thread starts, - that a thread ends)."); TranslationReport rep = new TranslationReport( bm != null ? bm.getBackendRunners() : new ArrayList<IBackendRunner>(), options.getOut()); long startTime = System.currentTimeMillis(); int specsCount = 0; pool = Executors.newFixedThreadPool(threadsCount); for (ITestTask task : sbp) { doTest(rep, task, realInDir, options.getOut(), translator, options, System.err); specsCount++; } pool.shutdown(); String reportFile = FilenameUtils.concat(options.getOut().getAbsolutePath(), "index.html"); try { while (!pool.awaitTermination(finalTimeout, TimeUnit.SECONDS)) { } } catch (InterruptedException e) { Debug.logger.error("Interrupted while waiting for pool termination.", e); System.out.println("Interrupted while waiting for pool termination: " + e.getMessage()); System.out.println("The report may be incomplete."); } long endTime = System.currentTimeMillis(); long duration = (endTime - startTime) / 1000; System.out.println(); System.out.println(specsCount + " specifications checked in " + duration + " seconds."); rep.report(reportFile); System.out.println("You can find an overview report at '" + reportFile + "'."); }
From source file:com.btoddb.fastpersitentqueue.speedtest.SpeedTest.java
public static void main(String[] args) throws Exception { if (0 == args.length) { System.out.println();//w ww . ja v a 2 s .c om System.out.println("ERROR: must specify the config file path/name"); System.out.println(); System.exit(1); } SpeedTestConfig config = SpeedTestConfig.create(args[0]); System.out.println(config.toString()); File theDir = new File(config.getDirectory(), "speed-" + UUID.randomUUID().toString()); FileUtils.forceMkdir(theDir); Fpq queue = config.getFpq(); queue.setJournalDirectory(new File(theDir, "journals")); queue.setPagingDirectory(new File(theDir, "pages")); try { queue.init(); // // start workers // AtomicLong counter = new AtomicLong(); AtomicLong pushSum = new AtomicLong(); AtomicLong popSum = new AtomicLong(); long startTime = System.currentTimeMillis(); Set<SpeedPushWorker> pushWorkers = new HashSet<SpeedPushWorker>(); for (int i = 0; i < config.getNumberOfPushers(); i++) { pushWorkers.add(new SpeedPushWorker(queue, config, counter, pushSum)); } Set<SpeedPopWorker> popWorkers = new HashSet<SpeedPopWorker>(); for (int i = 0; i < config.getNumberOfPoppers(); i++) { popWorkers.add(new SpeedPopWorker(queue, config, popSum)); } ExecutorService pusherExecSrvc = Executors.newFixedThreadPool( config.getNumberOfPushers() + config.getNumberOfPoppers(), new ThreadFactory() { @Override public Thread newThread(Runnable runnable) { Thread t = new Thread(runnable); t.setName("SpeedTest-Pusher"); return t; } }); ExecutorService popperExecSrvc = Executors.newFixedThreadPool( config.getNumberOfPushers() + config.getNumberOfPoppers(), new ThreadFactory() { @Override public Thread newThread(Runnable runnable) { Thread t = new Thread(runnable); t.setName("SpeedTest-Popper"); return t; } }); long startPushing = System.currentTimeMillis(); for (SpeedPushWorker sw : pushWorkers) { pusherExecSrvc.submit(sw); } long startPopping = System.currentTimeMillis(); for (SpeedPopWorker sw : popWorkers) { popperExecSrvc.submit(sw); } // // wait to finish // long endTime = startTime + config.getDurationOfTest() * 1000; long endPushing = 0; long displayTimer = 0; while (0 == endPushing || !queue.isEmpty()) { // display status every second if (1000 < (System.currentTimeMillis() - displayTimer)) { System.out.println(String.format("status (%ds) : journals = %d : memory segments = %d", (endTime - System.currentTimeMillis()) / 1000, queue.getJournalMgr().getJournalIdMap().size(), queue.getMemoryMgr().getSegments().size())); displayTimer = System.currentTimeMillis(); } pusherExecSrvc.shutdown(); if (pusherExecSrvc.awaitTermination(100, TimeUnit.MILLISECONDS)) { endPushing = System.currentTimeMillis(); // tell poppers, all pushers are finished for (SpeedPopWorker sw : popWorkers) { sw.stopWhenQueueEmpty(); } } } long endPopping = System.currentTimeMillis(); popperExecSrvc.shutdown(); popperExecSrvc.awaitTermination(10, TimeUnit.SECONDS); long numberOfPushes = 0; for (SpeedPushWorker sw : pushWorkers) { numberOfPushes += sw.getNumberOfEntries(); } long numberOfPops = 0; for (SpeedPopWorker sw : popWorkers) { numberOfPops += sw.getNumberOfEntries(); } long pushDuration = endPushing - startPushing; long popDuration = endPopping - startPopping; System.out.println("push - pop checksum = " + pushSum.get() + " - " + popSum.get() + " = " + (pushSum.get() - popSum.get())); System.out.println("push duration = " + pushDuration); System.out.println("pop duration = " + popDuration); System.out.println(); System.out.println("pushed = " + numberOfPushes); System.out.println("popped = " + numberOfPops); System.out.println(); System.out.println("push entries/sec = " + numberOfPushes / (pushDuration / 1000f)); System.out.println("pop entries/sec = " + numberOfPops / (popDuration / 1000f)); System.out.println(); System.out.println("journals created = " + queue.getJournalsCreated()); System.out.println("journals removed = " + queue.getJournalsRemoved()); } finally { if (null != queue) { queue.shutdown(); } // FileUtils.deleteDirectory(theDir); } }
From source file:com.mapr.synth.Synth.java
public static void main(String[] args) throws IOException, CmdLineException, InterruptedException, ExecutionException { final Options opts = new Options(); CmdLineParser parser = new CmdLineParser(opts); try {// ww w . j a v a2 s .c om parser.parseArgument(args); } catch (CmdLineException e) { System.err.println("Usage: " + "[ -count <number>G|M|K ] " + "-schema schema-file " + "[-quote DOUBLE_QUOTE|BACK_SLASH|OPTIMISTIC] " + "[-format JSON|TSV|CSV|XML ] " + "[-threads n] " + "[-output output-directory-name] "); throw e; } Preconditions.checkArgument(opts.threads > 0 && opts.threads <= 2000, "Must have at least one thread and no more than 2000"); if (opts.threads > 1) { Preconditions.checkArgument(!"-".equals(opts.output), "If more than on thread is used, you have to use -output to set the output directory"); } File outputDir = new File(opts.output); if (!"-".equals(opts.output)) { if (!outputDir.exists()) { Preconditions.checkState(outputDir.mkdirs(), String.format("Couldn't create output directory %s", opts.output)); } Preconditions.checkArgument(outputDir.exists() && outputDir.isDirectory(), String.format("Couldn't create directory %s", opts.output)); } if (opts.schema == null) { throw new IllegalArgumentException("Must specify schema file using [-schema filename] option"); } final SchemaSampler sampler = new SchemaSampler(opts.schema); final AtomicLong rowCount = new AtomicLong(); final List<ReportingWorker> tasks = Lists.newArrayList(); int limit = (opts.count + opts.threads - 1) / opts.threads; int remaining = opts.count; for (int i = 0; i < opts.threads; i++) { final int count = Math.min(limit, remaining); remaining -= count; tasks.add(new ReportingWorker(opts, sampler, rowCount, count, i)); } final double t0 = System.nanoTime() * 1e-9; ExecutorService pool = Executors.newFixedThreadPool(opts.threads); ScheduledExecutorService blinker = Executors.newScheduledThreadPool(1); final AtomicBoolean finalRun = new AtomicBoolean(false); final PrintStream sideLog = new PrintStream(new FileOutputStream("side-log")); Runnable blink = new Runnable() { public double oldT; private long oldN; @Override public void run() { double t = System.nanoTime() * 1e-9; long n = rowCount.get(); System.err.printf("%s\t%d\t%.1f\t%d\t%.1f\t%.3f\n", finalRun.get() ? "F" : "R", opts.threads, t - t0, n, n / (t - t0), (n - oldN) / (t - oldT)); for (ReportingWorker task : tasks) { ReportingWorker.ThreadReport r = task.report(); sideLog.printf("\t%d\t%.2f\t%.2f\t%.2f\t%.1f\t%.1f\n", r.fileNumber, r.threadTime, r.userTime, r.wallTime, r.rows / r.threadTime, r.rows / r.wallTime); } oldN = n; oldT = t; } }; if (!"-".equals(opts.output)) { blinker.scheduleAtFixedRate(blink, 0, 10, TimeUnit.SECONDS); } List<Future<Integer>> results = pool.invokeAll(tasks); int total = 0; for (Future<Integer> result : results) { total += result.get(); } Preconditions.checkState(total == opts.count, String .format("Expected to generate %d lines of output, but actually generated %d", opts.count, total)); pool.shutdownNow(); blinker.shutdownNow(); finalRun.set(true); sideLog.close(); blink.run(); }
From source file:gobblin.restli.throttling.LocalStressTest.java
public static void main(String[] args) throws Exception { CommandLine cli = StressTestUtils.parseCommandLine(OPTIONS, args); int stressorThreads = Integer.parseInt( cli.getOptionValue(STRESSOR_THREADS.getOpt(), Integer.toString(DEFAULT_STRESSOR_THREADS))); int processorThreads = Integer.parseInt( cli.getOptionValue(PROCESSOR_THREADS.getOpt(), Integer.toString(DEFAULT_PROCESSOR_THREADS))); int artificialLatency = Integer.parseInt( cli.getOptionValue(ARTIFICIAL_LATENCY.getOpt(), Integer.toString(DEFAULT_ARTIFICIAL_LATENCY))); long targetQps = Integer.parseInt(cli.getOptionValue(QPS.getOpt(), Integer.toString(DEFAULT_TARGET_QPS))); Configuration configuration = new Configuration(); StressTestUtils.populateConfigFromCli(configuration, cli); String resourceLimited = LocalStressTest.class.getSimpleName(); Map<String, String> configMap = Maps.newHashMap(); ThrottlingPolicyFactory factory = new ThrottlingPolicyFactory(); SharedLimiterKey res1key = new SharedLimiterKey(resourceLimited); configMap.put(BrokerConfigurationKeyGenerator.generateKey(factory, res1key, null, ThrottlingPolicyFactory.POLICY_KEY), QPSPolicy.FACTORY_ALIAS); configMap.put(BrokerConfigurationKeyGenerator.generateKey(factory, res1key, null, QPSPolicy.QPS), Long.toString(targetQps)); ThrottlingGuiceServletConfig guiceServletConfig = new ThrottlingGuiceServletConfig(); guiceServletConfig.initialize(ConfigFactory.parseMap(configMap)); LimiterServerResource limiterServer = guiceServletConfig.getInjector() .getInstance(LimiterServerResource.class); RateComputingLimiterContainer limiterContainer = new RateComputingLimiterContainer(); Class<? extends Stressor> stressorClass = configuration.getClass(StressTestUtils.STRESSOR_CLASS, StressTestUtils.DEFAULT_STRESSOR_CLASS, Stressor.class); ExecutorService executorService = Executors.newFixedThreadPool(stressorThreads); SharedResourcesBroker broker = guiceServletConfig.getInjector().getInstance( Key.get(SharedResourcesBroker.class, Names.named(LimiterServerResource.BROKER_INJECT_NAME))); ThrottlingPolicy policy = (ThrottlingPolicy) broker.getSharedResource(new ThrottlingPolicyFactory(), new SharedLimiterKey(resourceLimited)); ScheduledExecutorService reportingThread = Executors.newSingleThreadScheduledExecutor(); reportingThread.scheduleAtFixedRate(new Reporter(limiterContainer, policy), 0, 15, TimeUnit.SECONDS); Queue<Future<?>> futures = new LinkedList<>(); MockRequester requester = new MockRequester(limiterServer, artificialLatency, processorThreads); requester.start();//w w w. ja v a2 s.c o m for (int i = 0; i < stressorThreads; i++) { RestliServiceBasedLimiter restliLimiter = RestliServiceBasedLimiter.builder() .resourceLimited(resourceLimited).requestSender(requester).serviceIdentifier("stressor" + i) .build(); Stressor stressor = stressorClass.newInstance(); stressor.configure(configuration); futures.add(executorService .submit(new StressorRunner(limiterContainer.decorateLimiter(restliLimiter), stressor))); } int stressorFailures = 0; for (Future<?> future : futures) { try { future.get(); } catch (ExecutionException ee) { stressorFailures++; } } requester.stop(); executorService.shutdownNow(); if (stressorFailures > 0) { log.error("There were " + stressorFailures + " failed stressor threads."); } System.exit(stressorFailures); }
From source file:io.lavagna.loader.Loader.java
public static void main(String[] args) throws InterruptedException { System.setProperty("datasource.dialect", "MYSQL"); AnnotationConfigApplicationContext ac = new AnnotationConfigApplicationContext(TestServiceConfig.class, PersistenceAndServiceConfig.class); ac.register(CreateCardService.class); ProjectService ps = ac.getBean(ProjectService.class); CreateCardService ccs = ac.getBean(CreateCardService.class); UserRepository ur = ac.getBean(UserRepository.class); List<User> users = new ArrayList<>(); System.out.println("creating users"); for (int i = 0; i < 30; i++) { ur.createUser("loader", "user" + i, null, null, true); users.add(ur.findUserByName("loader", "user" + i)); }/* w ww . j a v a 2 s . c om*/ System.out.println("end creation"); CardLabelRepository clr = ac.getBean(CardLabelRepository.class); List<Project> projects = new ArrayList<>(PROJECT_NUMBERS); List<Integer> milestonesIds = new ArrayList<>(); System.out.println("creating projects"); for (int i = 0; i < PROJECT_NUMBERS; i++) { String name = "Load test project " + i; Project p = ps.create(name, "LDTEST_" + i, name); projects.add(p); // create user labels for (int iLabel = 0; iLabel < 10; iLabel++) { clr.addLabel(p.getId(), true, LabelType.NULL, LabelDomain.USER, "label-" + iLabel, 0); } // update milestone label CardLabel milestoneLabel = clr.findLabelByName(p.getId(), "MILESTONE", LabelDomain.SYSTEM); for (int j = 0; j < MILESTONES_PER_PROJECT; j++) { milestonesIds.add(clr.addLabelListValue(milestoneLabel.getId(), Integer.toString(j)).getId()); } } System.out.println("end creation"); System.out.println("creating boards"); for (Project project : projects) { buildBoards(project, ac, users, milestonesIds); } System.out.println("end creation"); long processed = ccs.getCreatedCardCounter().get(); while (!executor.awaitTermination(10, TimeUnit.SECONDS)) { long current = ccs.getCreatedCardCounter().get(); System.err.println("processed: " + (current - processed) + " cards in 10s"); processed = current; } ac.close(); }
From source file:com.yahoo.pulsar.testclient.PerformanceReader.java
public static void main(String[] args) throws Exception { final Arguments arguments = new Arguments(); JCommander jc = new JCommander(arguments); jc.setProgramName("pulsar-perf-reader"); try {/* w w w. j a v a2 s .co m*/ jc.parse(args); } catch (ParameterException e) { System.out.println(e.getMessage()); jc.usage(); System.exit(-1); } if (arguments.help) { jc.usage(); System.exit(-1); } if (arguments.topic.size() != 1) { System.out.println("Only one topic name is allowed"); jc.usage(); System.exit(-1); } if (arguments.confFile != null) { Properties prop = new Properties(System.getProperties()); prop.load(new FileInputStream(arguments.confFile)); if (arguments.serviceURL == null) { arguments.serviceURL = prop.getProperty("brokerServiceUrl"); } if (arguments.serviceURL == null) { arguments.serviceURL = prop.getProperty("webServiceUrl"); } // fallback to previous-version serviceUrl property to maintain backward-compatibility if (arguments.serviceURL == null) { arguments.serviceURL = prop.getProperty("serviceUrl", "http://localhost:8080/"); } if (arguments.authPluginClassName == null) { arguments.authPluginClassName = prop.getProperty("authPlugin", null); } if (arguments.authParams == null) { arguments.authParams = prop.getProperty("authParams", null); } } // Dump config variables ObjectMapper m = new ObjectMapper(); ObjectWriter w = m.writerWithDefaultPrettyPrinter(); log.info("Starting Pulsar performance reader with config: {}", w.writeValueAsString(arguments)); final DestinationName prefixTopicName = DestinationName.get(arguments.topic.get(0)); final RateLimiter limiter = arguments.rate > 0 ? RateLimiter.create(arguments.rate) : null; ReaderListener listener = (reader, msg) -> { messagesReceived.increment(); bytesReceived.add(msg.getData().length); if (limiter != null) { limiter.acquire(); } }; EventLoopGroup eventLoopGroup; if (SystemUtils.IS_OS_LINUX) { eventLoopGroup = new EpollEventLoopGroup(Runtime.getRuntime().availableProcessors() * 2, new DefaultThreadFactory("pulsar-perf-reader")); } else { eventLoopGroup = new NioEventLoopGroup(Runtime.getRuntime().availableProcessors(), new DefaultThreadFactory("pulsar-perf-reader")); } ClientConfiguration clientConf = new ClientConfiguration(); clientConf.setConnectionsPerBroker(arguments.maxConnections); clientConf.setStatsInterval(arguments.statsIntervalSeconds, TimeUnit.SECONDS); if (isNotBlank(arguments.authPluginClassName)) { clientConf.setAuthentication(arguments.authPluginClassName, arguments.authParams); } PulsarClient pulsarClient = new PulsarClientImpl(arguments.serviceURL, clientConf, eventLoopGroup); List<CompletableFuture<Reader>> futures = Lists.newArrayList(); ReaderConfiguration readerConfig = new ReaderConfiguration(); readerConfig.setReaderListener(listener); readerConfig.setReceiverQueueSize(arguments.receiverQueueSize); MessageId startMessageId; if ("earliest".equals(arguments.startMessageId)) { startMessageId = MessageId.earliest; } else if ("latest".equals(arguments.startMessageId)) { startMessageId = MessageId.latest; } else { String[] parts = arguments.startMessageId.split(":"); startMessageId = new MessageIdImpl(Long.parseLong(parts[0]), Long.parseLong(parts[1]), -1); } for (int i = 0; i < arguments.numDestinations; i++) { final DestinationName destinationName = (arguments.numDestinations == 1) ? prefixTopicName : DestinationName.get(String.format("%s-%d", prefixTopicName, i)); futures.add(pulsarClient.createReaderAsync(destinationName.toString(), startMessageId, readerConfig)); } FutureUtil.waitForAll(futures).get(); log.info("Start reading from {} topics", arguments.numDestinations); long oldTime = System.nanoTime(); while (true) { try { Thread.sleep(10000); } catch (InterruptedException e) { break; } long now = System.nanoTime(); double elapsed = (now - oldTime) / 1e9; double rate = messagesReceived.sumThenReset() / elapsed; double throughput = bytesReceived.sumThenReset() / elapsed * 8 / 1024 / 1024; log.info("Read throughput: {} msg/s -- {} Mbit/s", dec.format(rate), dec.format(throughput)); oldTime = now; } pulsarClient.close(); }
From source file:com.yahoo.pulsar.testclient.PerformanceConsumer.java
public static void main(String[] args) throws Exception { final Arguments arguments = new Arguments(); JCommander jc = new JCommander(arguments); jc.setProgramName("pulsar-perf-consumer"); try {/* ww w.j a v a 2 s. c om*/ jc.parse(args); } catch (ParameterException e) { System.out.println(e.getMessage()); jc.usage(); System.exit(-1); } if (arguments.help) { jc.usage(); System.exit(-1); } if (arguments.topic.size() != 1) { System.out.println("Only one destination name is allowed"); jc.usage(); System.exit(-1); } if (arguments.confFile != null) { Properties prop = new Properties(System.getProperties()); prop.load(new FileInputStream(arguments.confFile)); if (arguments.serviceURL == null) { arguments.serviceURL = prop.getProperty("brokerServiceUrl"); } if (arguments.serviceURL == null) { arguments.serviceURL = prop.getProperty("webServiceUrl"); } // fallback to previous-version serviceUrl property to maintain backward-compatibility if (arguments.serviceURL == null) { arguments.serviceURL = prop.getProperty("serviceUrl", "http://localhost:8080/"); } if (arguments.authPluginClassName == null) { arguments.authPluginClassName = prop.getProperty("authPlugin", null); } if (arguments.authParams == null) { arguments.authParams = prop.getProperty("authParams", null); } } // Dump config variables ObjectMapper m = new ObjectMapper(); ObjectWriter w = m.writerWithDefaultPrettyPrinter(); log.info("Starting Pulsar performance consumer with config: {}", w.writeValueAsString(arguments)); final DestinationName prefixDestinationName = DestinationName.get(arguments.topic.get(0)); final RateLimiter limiter = arguments.rate > 0 ? RateLimiter.create(arguments.rate) : null; MessageListener listener = new MessageListener() { public void received(Consumer consumer, Message msg) { messagesReceived.increment(); bytesReceived.add(msg.getData().length); if (limiter != null) { limiter.acquire(); } consumer.acknowledgeAsync(msg); } }; EventLoopGroup eventLoopGroup; if (SystemUtils.IS_OS_LINUX) { eventLoopGroup = new EpollEventLoopGroup(Runtime.getRuntime().availableProcessors() * 2, new DefaultThreadFactory("pulsar-perf-consumer")); } else { eventLoopGroup = new NioEventLoopGroup(Runtime.getRuntime().availableProcessors(), new DefaultThreadFactory("pulsar-perf-consumer")); } ClientConfiguration clientConf = new ClientConfiguration(); clientConf.setConnectionsPerBroker(arguments.maxConnections); clientConf.setStatsInterval(arguments.statsIntervalSeconds, TimeUnit.SECONDS); if (isNotBlank(arguments.authPluginClassName)) { clientConf.setAuthentication(arguments.authPluginClassName, arguments.authParams); } PulsarClient pulsarClient = new PulsarClientImpl(arguments.serviceURL, clientConf, eventLoopGroup); List<Future<Consumer>> futures = Lists.newArrayList(); ConsumerConfiguration consumerConfig = new ConsumerConfiguration(); consumerConfig.setMessageListener(listener); consumerConfig.setReceiverQueueSize(arguments.receiverQueueSize); for (int i = 0; i < arguments.numDestinations; i++) { final DestinationName destinationName = (arguments.numDestinations == 1) ? prefixDestinationName : DestinationName.get(String.format("%s-%d", prefixDestinationName, i)); log.info("Adding {} consumers on destination {}", arguments.numConsumers, destinationName); for (int j = 0; j < arguments.numConsumers; j++) { String subscriberName; if (arguments.numConsumers > 1) { subscriberName = String.format("%s-%d", arguments.subscriberName, j); } else { subscriberName = arguments.subscriberName; } futures.add( pulsarClient.subscribeAsync(destinationName.toString(), subscriberName, consumerConfig)); } } for (Future<Consumer> future : futures) { future.get(); } log.info("Start receiving from {} consumers on {} destinations", arguments.numConsumers, arguments.numDestinations); long oldTime = System.nanoTime(); while (true) { try { Thread.sleep(10000); } catch (InterruptedException e) { break; } long now = System.nanoTime(); double elapsed = (now - oldTime) / 1e9; double rate = messagesReceived.sumThenReset() / elapsed; double throughput = bytesReceived.sumThenReset() / elapsed * 8 / 1024 / 1024; log.info("Throughput received: {} msg/s -- {} Mbit/s", dec.format(rate), dec.format(throughput)); oldTime = now; } pulsarClient.close(); }
From source file:com.yahoo.pulsar.testclient.PerformanceProducer.java
public static void main(String[] args) throws Exception { final Arguments arguments = new Arguments(); JCommander jc = new JCommander(arguments); jc.setProgramName("pulsar-perf-producer"); try {/* w w w. j a v a 2s . c o m*/ jc.parse(args); } catch (ParameterException e) { System.out.println(e.getMessage()); jc.usage(); System.exit(-1); } if (arguments.help) { jc.usage(); System.exit(-1); } if (arguments.destinations.size() != 1) { System.out.println("Only one topic name is allowed"); jc.usage(); System.exit(-1); } if (arguments.confFile != null) { Properties prop = new Properties(System.getProperties()); prop.load(new FileInputStream(arguments.confFile)); if (arguments.serviceURL == null) { arguments.serviceURL = prop.getProperty("brokerServiceUrl"); } if (arguments.serviceURL == null) { arguments.serviceURL = prop.getProperty("webServiceUrl"); } // fallback to previous-version serviceUrl property to maintain backward-compatibility if (arguments.serviceURL == null) { arguments.serviceURL = prop.getProperty("serviceUrl", "http://localhost:8080/"); } if (arguments.authPluginClassName == null) { arguments.authPluginClassName = prop.getProperty("authPlugin", null); } if (arguments.authParams == null) { arguments.authParams = prop.getProperty("authParams", null); } } arguments.testTime = TimeUnit.SECONDS.toMillis(arguments.testTime); // Dump config variables ObjectMapper m = new ObjectMapper(); ObjectWriter w = m.writerWithDefaultPrettyPrinter(); log.info("Starting Pulsar perf producer with config: {}", w.writeValueAsString(arguments)); // Read payload data from file if needed byte payloadData[]; if (arguments.payloadFilename != null) { payloadData = Files.readAllBytes(Paths.get(arguments.payloadFilename)); } else { payloadData = new byte[arguments.msgSize]; } // Now processing command line arguments String prefixTopicName = arguments.destinations.get(0); List<Future<Producer>> futures = Lists.newArrayList(); EventLoopGroup eventLoopGroup; if (SystemUtils.IS_OS_LINUX) { eventLoopGroup = new EpollEventLoopGroup(Runtime.getRuntime().availableProcessors(), new DefaultThreadFactory("pulsar-perf-producer")); } else { eventLoopGroup = new NioEventLoopGroup(Runtime.getRuntime().availableProcessors(), new DefaultThreadFactory("pulsar-perf-producer")); } ClientConfiguration clientConf = new ClientConfiguration(); clientConf.setConnectionsPerBroker(arguments.maxConnections); clientConf.setStatsInterval(arguments.statsIntervalSeconds, TimeUnit.SECONDS); if (isNotBlank(arguments.authPluginClassName)) { clientConf.setAuthentication(arguments.authPluginClassName, arguments.authParams); } PulsarClient client = new PulsarClientImpl(arguments.serviceURL, clientConf, eventLoopGroup); ProducerConfiguration producerConf = new ProducerConfiguration(); producerConf.setSendTimeout(0, TimeUnit.SECONDS); producerConf.setCompressionType(arguments.compression); // enable round robin message routing if it is a partitioned topic producerConf.setMessageRoutingMode(MessageRoutingMode.RoundRobinPartition); if (arguments.batchTime > 0) { producerConf.setBatchingMaxPublishDelay(arguments.batchTime, TimeUnit.MILLISECONDS); producerConf.setBatchingEnabled(true); producerConf.setMaxPendingMessages(arguments.msgRate); } for (int i = 0; i < arguments.numTopics; i++) { String topic = (arguments.numTopics == 1) ? prefixTopicName : String.format("%s-%d", prefixTopicName, i); log.info("Adding {} publishers on destination {}", arguments.numProducers, topic); for (int j = 0; j < arguments.numProducers; j++) { futures.add(client.createProducerAsync(topic, producerConf)); } } final List<Producer> producers = Lists.newArrayListWithCapacity(futures.size()); for (Future<Producer> future : futures) { producers.add(future.get()); } log.info("Created {} producers", producers.size()); Runtime.getRuntime().addShutdownHook(new Thread() { public void run() { printAggregatedStats(); } }); Collections.shuffle(producers); AtomicBoolean isDone = new AtomicBoolean(); executor.submit(() -> { try { RateLimiter rateLimiter = RateLimiter.create(arguments.msgRate); long startTime = System.currentTimeMillis(); // Send messages on all topics/producers long totalSent = 0; while (true) { for (Producer producer : producers) { if (arguments.testTime > 0) { if (System.currentTimeMillis() - startTime > arguments.testTime) { log.info("------------------- DONE -----------------------"); printAggregatedStats(); isDone.set(true); Thread.sleep(5000); System.exit(0); } } if (arguments.numMessages > 0) { if (totalSent++ >= arguments.numMessages) { log.info("------------------- DONE -----------------------"); printAggregatedStats(); isDone.set(true); Thread.sleep(5000); System.exit(0); } } rateLimiter.acquire(); final long sendTime = System.nanoTime(); producer.sendAsync(payloadData).thenRun(() -> { messagesSent.increment(); bytesSent.add(payloadData.length); long latencyMicros = NANOSECONDS.toMicros(System.nanoTime() - sendTime); recorder.recordValue(latencyMicros); cumulativeRecorder.recordValue(latencyMicros); }).exceptionally(ex -> { log.warn("Write error on message", ex); System.exit(-1); return null; }); } } } catch (Throwable t) { log.error("Got error", t); } }); // Print report stats long oldTime = System.nanoTime(); Histogram reportHistogram = null; String statsFileName = "perf-producer-" + System.currentTimeMillis() + ".hgrm"; log.info("Dumping latency stats to {}", statsFileName); PrintStream histogramLog = new PrintStream(new FileOutputStream(statsFileName), false); HistogramLogWriter histogramLogWriter = new HistogramLogWriter(histogramLog); // Some log header bits histogramLogWriter.outputLogFormatVersion(); histogramLogWriter.outputLegend(); while (true) { try { Thread.sleep(10000); } catch (InterruptedException e) { break; } if (isDone.get()) { break; } long now = System.nanoTime(); double elapsed = (now - oldTime) / 1e9; double rate = messagesSent.sumThenReset() / elapsed; double throughput = bytesSent.sumThenReset() / elapsed / 1024 / 1024 * 8; reportHistogram = recorder.getIntervalHistogram(reportHistogram); log.info( "Throughput produced: {} msg/s --- {} Mbit/s --- Latency: mean: {} ms - med: {} - 95pct: {} - 99pct: {} - 99.9pct: {} - 99.99pct: {} - Max: {}", throughputFormat.format(rate), throughputFormat.format(throughput), dec.format(reportHistogram.getMean() / 1000.0), dec.format(reportHistogram.getValueAtPercentile(50) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(95) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99.9) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99.99) / 1000.0), dec.format(reportHistogram.getMaxValue() / 1000.0)); histogramLogWriter.outputIntervalHistogram(reportHistogram); reportHistogram.reset(); oldTime = now; } client.close(); }
From source file:AmazonKinesisFirehoseToRedshiftSample.java
public static void main(String[] args) throws Exception { init();//from w w w . ja v a2 s.c om try { // Create S3 bucket for DeliveryStream to deliver data createS3Bucket(); // Create the DeliveryStream createDeliveryStream(); // Print the list of delivery streams printDeliveryStreams(); // Put records into DeliveryStream LOG.info("Putting records in DeliveryStream : " + deliveryStreamName + " via Put Record method."); putRecordIntoDeliveryStream(); // Batch Put records into DeliveryStream LOG.info("Putting records in DeliveryStream : " + deliveryStreamName + " via Put Record Batch method. Now you can check your S3 bucket " + s3BucketName + " for the data delivered by DeliveryStream."); putRecordBatchIntoDeliveryStream(); // Wait for some interval for the firehose to write data to redshift destination int waitTimeSecs = s3DestinationIntervalInSeconds == null ? DEFAULT_WAIT_INTERVAL_FOR_DATA_DELIVERY_SECS : s3DestinationIntervalInSeconds; waitForDataDelivery(waitTimeSecs); // Update the DeliveryStream and Put records into updated DeliveryStream, only if the flag is set if (enableUpdateDestination) { // Update the DeliveryStream updateDeliveryStream(); // Wait for some interval to propagate the updated configuration options before ingesting data LOG.info("Waiting for few seconds to propagate the updated configuration options."); TimeUnit.SECONDS.sleep(60); // Put records into updated DeliveryStream. LOG.info("Putting records in updated DeliveryStream : " + deliveryStreamName + " via Put Record method."); putRecordIntoDeliveryStream(); // Batch Put records into updated DeliveryStream. LOG.info("Putting records in updated DeliveryStream : " + deliveryStreamName + " via Put Record Batch method."); putRecordBatchIntoDeliveryStream(); // Wait for some interval for the DeliveryStream to write data to redshift destination waitForDataDelivery(waitTimeSecs); } } catch (AmazonServiceException ase) { LOG.error("Caught Amazon Service Exception"); LOG.error("Status Code " + ase.getErrorCode()); LOG.error("Message: " + ase.getErrorMessage(), ase); } catch (AmazonClientException ace) { LOG.error("Caught Amazon Client Exception"); LOG.error("Exception Message " + ace.getMessage(), ace); } }
From source file:AmazonKinesisFirehoseToS3Sample.java
public static void main(String[] args) throws Exception { init();//w w w.j av a 2 s . c o m try { // Create S3 bucket for deliveryStream to deliver data createS3Bucket(); // Create the DeliveryStream createDeliveryStream(); // Print the list of delivery streams printDeliveryStreams(); // Put records into deliveryStream LOG.info("Putting records in deliveryStream : " + deliveryStreamName + " via Put Record method."); putRecordIntoDeliveryStream(); // Batch Put records into deliveryStream LOG.info("Putting records in deliveryStream : " + deliveryStreamName + " via Put Record Batch method. Now you can check your S3 bucket " + s3BucketName + " for the data delivered by deliveryStream."); putRecordBatchIntoDeliveryStream(); // Wait for some interval for the firehose to write data to the S3 bucket int waitTimeSecs = s3DestinationIntervalInSeconds == null ? DEFAULT_WAIT_INTERVAL_FOR_DATA_DELIVERY_SECS : s3DestinationIntervalInSeconds; waitForDataDelivery(waitTimeSecs); // Update the deliveryStream and Put records into updated deliveryStream, only if the flag is set if (enableUpdateDestination) { // Update the deliveryStream updateDeliveryStream(); // Wait for some interval to propagate the updated configuration options before ingesting data LOG.info("Waiting for few seconds to propagate the updated configuration options."); TimeUnit.SECONDS.sleep(60); // Put records into updated deliveryStream. Records will be delivered to new S3 prefix location LOG.info("Putting records in updated deliveryStream : " + deliveryStreamName + " via Put Record method."); putRecordIntoDeliveryStream(); // Batch Put records into updated deliveryStream. Records will be delivered to new S3 prefix location LOG.info("Putting records in updated deliveryStream : " + deliveryStreamName + " via Put Record Batch method."); putRecordBatchIntoDeliveryStream(); // Wait for some interval for the deliveryStream to write data to new prefix location in S3 bucket waitTimeSecs = updateIntervalInSeconds == null ? waitTimeSecs : updateIntervalInSeconds; waitForDataDelivery(waitTimeSecs); } } catch (AmazonServiceException ase) { LOG.error("Caught Amazon Service Exception"); LOG.error("Status Code " + ase.getErrorCode()); LOG.error("Message: " + ase.getErrorMessage(), ase); } catch (AmazonClientException ace) { LOG.error("Caught Amazon Client Exception"); LOG.error("Exception Message " + ace.getMessage(), ace); } }