List of usage examples for java.lang Thread join
public final void join() throws InterruptedException
From source file:alluxio.cli.JournalCrashTest.java
/** * Runs the crash test./*from www . j a va2s . c om*/ * * @param args no arguments */ public static void main(String[] args) { // Parse the input args. if (!parseInputArgs(args)) { System.exit(EXIT_FAILED); } System.out.println("Stop the current Alluxio cluster..."); stopCluster(); // Set NO_STORE and NO_PERSIST so that this test can work without AlluxioWorker. sCreateFileOptions = CreateFileOptions.defaults().setWriteType(WriteType.NONE); // Set the max retry to avoid long pending for client disconnect. if (System.getProperty(PropertyKey.USER_RPC_RETRY_MAX_NUM_RETRY.toString()) == null) { System.setProperty(PropertyKey.USER_RPC_RETRY_MAX_NUM_RETRY.toString(), "10"); } System.out.println("Start Journal Crash Test..."); long startTimeMs = System.currentTimeMillis(); boolean ret = true; startMaster(); int rounds = 0; while (System.currentTimeMillis() - startTimeMs < sTotalTimeMs) { rounds++; long aliveTimeMs = (long) (Math.random() * sMaxAliveTimeMs) + 100; LOG.info("Round {}: Planning Master Alive Time {}ms.", rounds, aliveTimeMs); System.out.println("Round " + rounds + " : Launch Clients..."); sFileSystem = FileSystem.Factory.get(); try { sFileSystem.delete(new AlluxioURI(sTestDir)); } catch (Exception e) { // Test Directory not exist } // Launch all the client threads. setupClientThreads(); for (Thread thread : sClientThreadList) { thread.start(); } CommonUtils.sleepMs(LOG, aliveTimeMs); System.out.println("Round " + rounds + " : Crash Master..."); killMaster(); for (ClientThread clientThread : sClientThreadList) { clientThread.setIsStopped(true); } for (Thread thread : sClientThreadList) { try { thread.join(); } catch (InterruptedException e) { LOG.error("Error when waiting thread", e); } } System.out.println("Round " + rounds + " : Check Status..."); startMaster(); boolean checkSuccess = false; try { checkSuccess = checkStatus(); } catch (Exception e) { LOG.error("Failed to check status", e); } CliUtils.printPassInfo(checkSuccess); ret &= checkSuccess; } stopCluster(); System.exit(ret ? EXIT_SUCCESS : EXIT_FAILED); }
From source file:com.dxc.temp.SimpleProducerConsumer.java
public static void main(String[] args) throws InterruptedException { int argIndex = 0; final String accessKey = args[argIndex++]; final String secretKey = args[argIndex++]; final AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey); final String endpoint = args[argIndex++]; final String queueName = args[argIndex++]; final int producerCount = Integer.parseInt(args[argIndex++]); final int consumerCount = Integer.parseInt(args[argIndex++]); final int batchSize = Integer.parseInt(args[argIndex++]); final int messageSizeByte = Integer.parseInt(args[argIndex++]); final int runTimeMinutes = Integer.parseInt(args[argIndex++]); // configure the SQS client with enough connections for all producer and // consumer threads AmazonSQS sqsClient = new AmazonSQSClient(credentials, new ClientConfiguration().withMaxConnections(producerCount + consumerCount)); sqsClient.setEndpoint(endpoint);/*from w ww.j a va2s . c o m*/ String queueUrl = sqsClient.getQueueUrl(new GetQueueUrlRequest(queueName)).getQueueUrl(); // the flag to stop producer, consumer, and monitor threads AtomicBoolean stop = new AtomicBoolean(false); // start the producers final AtomicInteger producedCount = new AtomicInteger(); Thread[] producers = new Thread[producerCount]; for (int i = 0; i < producerCount; i++) { if (batchSize == 1) producers[i] = new Producer(sqsClient, queueUrl, messageSizeByte, producedCount, stop); else producers[i] = new BatchProducer(sqsClient, queueUrl, batchSize, messageSizeByte, producedCount, stop); producers[i].start(); } // start the consumers final AtomicInteger consumedCount = new AtomicInteger(); Thread[] consumers = new Thread[consumerCount]; for (int i = 0; i < consumerCount; i++) { if (batchSize == 1) consumers[i] = new Consumer(sqsClient, queueUrl, consumedCount, stop); else consumers[i] = new BatchConsumer(sqsClient, queueUrl, batchSize, consumedCount, stop); consumers[i].start(); } // start the monitor (thread) Thread monitor = new Monitor(producedCount, consumedCount, stop); monitor.start(); // wait for the specified amount of time then stop Thread.sleep(TimeUnit.MINUTES.toMillis(Math.min(runTimeMinutes, MAX_RUNTIME_MINUTES))); stop.set(true); // join all threads for (int i = 0; i < producerCount; i++) producers[i].join(); for (int i = 0; i < consumerCount; i++) consumers[i].join(); monitor.interrupt(); monitor.join(); }
From source file:com.oltpbenchmark.multitenancy.MuTeBench.java
/** * @param args/* w ww . j a v a 2 s .co m*/ * @throws Exception */ public static void main(String[] args) throws Exception { String duration = null; String scenarioFile = null; // ------------------------------------------------------------------- // INITIALIZE LOGGING // ------------------------------------------------------------------- String log4jPath = System.getProperty("log4j.configuration"); if (log4jPath != null) { org.apache.log4j.PropertyConfigurator.configure(log4jPath); } else { throw new RuntimeException("Missing log4j.properties file"); } // ------------------------------------------------------------------- // PARSE COMMAND LINE PARAMETERS // ------------------------------------------------------------------- CommandLineParser parser = new PosixParser(); XMLConfiguration pluginConfig = null; try { pluginConfig = new XMLConfiguration("config/plugin.xml"); } catch (ConfigurationException e1) { LOG.info("Plugin configuration file config/plugin.xml is missing"); e1.printStackTrace(); } pluginConfig.setExpressionEngine(new XPathExpressionEngine()); Options options = new Options(); options.addOption("s", "scenario", true, "[required] Workload scenario file"); options.addOption("a", "analysis-buckets", true, "sampling buckets for result aggregation"); options.addOption("r", "runtime", true, "maximum runtime (no events will be started after finishing runtime)"); options.addOption("v", "verbose", false, "Display Messages"); options.addOption("g", "gui", false, "Show controlling GUI"); options.addOption("h", "help", false, "Print this help"); options.addOption("o", "output", true, "Output file (default System.out)"); options.addOption("b", "baseline", true, "Output files of previous baseline run"); options.addOption(null, "histograms", false, "Print txn histograms"); options.addOption("d", "dialects-export", true, "Export benchmark SQL to a dialects file"); // parse the command line arguments CommandLine argsLine = parser.parse(options, args); if (argsLine.hasOption("h")) { printUsage(options); return; } else if (!argsLine.hasOption("scenario")) { INIT_LOG.fatal("Missing scenario description file"); System.exit(-1); } else scenarioFile = argsLine.getOptionValue("scenario"); if (argsLine.hasOption("r")) duration = argsLine.getOptionValue("r"); if (argsLine.hasOption("runtime")) duration = argsLine.getOptionValue("runtime"); // ------------------------------------------------------------------- // CREATE TENANT SCHEDULE // ------------------------------------------------------------------- INIT_LOG.info("Create schedule"); Schedule schedule = new Schedule(duration, scenarioFile); HashMap<Integer, ScheduleEvents> tenantEvents = schedule.getTenantEvents(); ArrayList<Integer> tenantList = schedule.getTenantList(); List<BenchmarkModule> benchList = new ArrayList<BenchmarkModule>(); for (int tenInd = 0; tenInd < tenantList.size(); tenInd++) { int tenantID = tenantList.get(tenInd); for (int tenEvent = 0; tenEvent < tenantEvents.get(tenantID).size(); tenEvent++) { BenchmarkSettings benchmarkSettings = (BenchmarkSettings) tenantEvents.get(tenantID) .getBenchmarkSettings(tenEvent); // update benchmark Settings benchmarkSettings.setTenantID(tenantID); // ------------------------------------------------------------------- // GET PLUGIN LIST // ------------------------------------------------------------------- String plugins = benchmarkSettings.getBenchmark(); String[] pluginList = plugins.split(","); String configFile = benchmarkSettings.getConfigFile(); XMLConfiguration xmlConfig = new XMLConfiguration(configFile); xmlConfig.setExpressionEngine(new XPathExpressionEngine()); int lastTxnId = 0; for (String plugin : pluginList) { // ---------------------------------------------------------------- // WORKLOAD CONFIGURATION // ---------------------------------------------------------------- String pluginTest = ""; pluginTest = "[@bench='" + plugin + "']"; WorkloadConfiguration wrkld = new WorkloadConfiguration(); wrkld.setTenantId(tenantID); wrkld.setBenchmarkName(setTenantIDinString(plugin, tenantID)); wrkld.setXmlConfig(xmlConfig); wrkld.setDBType(DatabaseType.get(setTenantIDinString(xmlConfig.getString("dbtype"), tenantID))); wrkld.setDBDriver(setTenantIDinString(xmlConfig.getString("driver"), tenantID)); wrkld.setDBConnection(setTenantIDinString(xmlConfig.getString("DBUrl"), tenantID)); wrkld.setDBName(setTenantIDinString(xmlConfig.getString("DBName"), tenantID)); wrkld.setDBUsername(setTenantIDinString(xmlConfig.getString("username"), tenantID)); wrkld.setDBPassword(setTenantIDinString(xmlConfig.getString("password"), tenantID)); String terminalString = setTenantIDinString(xmlConfig.getString("terminals[not(@bench)]", "0"), tenantID); int terminals = Integer.parseInt(xmlConfig.getString("terminals" + pluginTest, terminalString)); wrkld.setTerminals(terminals); int taSize = Integer.parseInt(xmlConfig.getString("taSize", "1")); if (taSize < 0) INIT_LOG.fatal("taSize must not be negative!"); wrkld.setTaSize(taSize); wrkld.setProprietaryTaSyntax(xmlConfig.getBoolean("proprietaryTaSyntax", false)); wrkld.setIsolationMode(setTenantIDinString( xmlConfig.getString("isolation", "TRANSACTION_SERIALIZABLE"), tenantID)); wrkld.setScaleFactor(Double .parseDouble(setTenantIDinString(xmlConfig.getString("scalefactor", "1.0"), tenantID))); wrkld.setRecordAbortMessages(xmlConfig.getBoolean("recordabortmessages", false)); int size = xmlConfig.configurationsAt("/works/work").size(); for (int i = 1; i < size + 1; i++) { SubnodeConfiguration work = xmlConfig.configurationAt("works/work[" + i + "]"); List<String> weight_strings; // use a workaround if there multiple workloads or // single // attributed workload if (pluginList.length > 1 || work.containsKey("weights[@bench]")) { weight_strings = get_weights(plugin, work); } else { weight_strings = work.getList("weights[not(@bench)]"); } int rate = 1; boolean rateLimited = true; boolean disabled = false; // can be "disabled", "unlimited" or a number String rate_string; rate_string = setTenantIDinString(work.getString("rate[not(@bench)]", ""), tenantID); rate_string = setTenantIDinString(work.getString("rate" + pluginTest, rate_string), tenantID); if (rate_string.equals(RATE_DISABLED)) { disabled = true; } else if (rate_string.equals(RATE_UNLIMITED)) { rateLimited = false; } else if (rate_string.isEmpty()) { LOG.fatal(String.format( "Tenant " + tenantID + ": Please specify the rate for phase %d and workload %s", i, plugin)); System.exit(-1); } else { try { rate = Integer.parseInt(rate_string); if (rate < 1) { LOG.fatal("Tenant " + tenantID + ": Rate limit must be at least 1. Use unlimited or disabled values instead."); System.exit(-1); } } catch (NumberFormatException e) { LOG.fatal(String.format( "Tenant " + tenantID + ": Rate string must be '%s', '%s' or a number", RATE_DISABLED, RATE_UNLIMITED)); System.exit(-1); } } Phase.Arrival arrival = Phase.Arrival.REGULAR; String arrive = setTenantIDinString(work.getString("@arrival", "regular"), tenantID); if (arrive.toUpperCase().equals("POISSON")) arrival = Phase.Arrival.POISSON; int activeTerminals; activeTerminals = Integer.parseInt(setTenantIDinString( work.getString("active_terminals[not(@bench)]", String.valueOf(terminals)), tenantID)); activeTerminals = Integer.parseInt(setTenantIDinString( work.getString("active_terminals" + pluginTest, String.valueOf(activeTerminals)), tenantID)); if (activeTerminals > terminals) { LOG.fatal("Tenant " + tenantID + ": Configuration error in work " + i + ": number of active terminals" + "" + "is bigger than the total number of terminals"); System.exit(-1); } wrkld.addWork(Integer.parseInt(setTenantIDinString(work.getString("/time"), tenantID)), rate, weight_strings, rateLimited, disabled, activeTerminals, arrival); } // FOR int numTxnTypes = xmlConfig .configurationsAt("transactiontypes" + pluginTest + "/transactiontype").size(); if (numTxnTypes == 0 && pluginList.length == 1) { // if it is a single workload run, <transactiontypes /> // w/o attribute is used pluginTest = "[not(@bench)]"; numTxnTypes = xmlConfig .configurationsAt("transactiontypes" + pluginTest + "/transactiontype").size(); } wrkld.setNumTxnTypes(numTxnTypes); // CHECKING INPUT PHASES int j = 0; for (Phase p : wrkld.getAllPhases()) { j++; if (p.getWeightCount() != wrkld.getNumTxnTypes()) { LOG.fatal(String.format("Tenant " + tenantID + ": Configuration files is inconsistent, phase %d contains %d weights but you defined %d transaction types", j, p.getWeightCount(), wrkld.getNumTxnTypes())); System.exit(-1); } } // FOR // Generate the dialect map wrkld.init(); assert (wrkld.getNumTxnTypes() >= 0); assert (xmlConfig != null); // ---------------------------------------------------------------- // BENCHMARK MODULE // ---------------------------------------------------------------- String classname = pluginConfig.getString("/plugin[@name='" + plugin + "']"); if (classname == null) { throw new ParseException("Plugin " + plugin + " is undefined in config/plugin.xml"); } BenchmarkModule bench = ClassUtil.newInstance(classname, new Object[] { wrkld }, new Class<?>[] { WorkloadConfiguration.class }); assert (benchList.get(0) != null); Map<String, Object> initDebug = new ListOrderedMap<String, Object>(); initDebug.put("Benchmark", String.format("%s {%s}", plugin.toUpperCase(), classname)); initDebug.put("Configuration", configFile); initDebug.put("Type", wrkld.getDBType()); initDebug.put("Driver", wrkld.getDBDriver()); initDebug.put("URL", wrkld.getDBConnection()); initDebug.put("Isolation", setTenantIDinString( xmlConfig.getString("isolation", "TRANSACTION_SERIALIZABLE [DEFAULT]"), tenantID)); initDebug.put("Scale Factor", wrkld.getScaleFactor()); INIT_LOG.info(SINGLE_LINE + "\n\n" + StringUtil.formatMaps(initDebug)); INIT_LOG.info(SINGLE_LINE); // Load TransactionTypes List<TransactionType> ttypes = new ArrayList<TransactionType>(); // Always add an INVALID type for Carlo ttypes.add(TransactionType.INVALID); int txnIdOffset = lastTxnId; for (int i = 1; i < wrkld.getNumTxnTypes() + 1; i++) { String key = "transactiontypes" + pluginTest + "/transactiontype[" + i + "]"; String txnName = setTenantIDinString(xmlConfig.getString(key + "/name"), tenantID); int txnId = i + 1; if (xmlConfig.containsKey(key + "/id")) { txnId = Integer .parseInt(setTenantIDinString(xmlConfig.getString(key + "/id"), tenantID)); } ttypes.add(bench.initTransactionType(txnName, txnId + txnIdOffset)); lastTxnId = i; } // FOR TransactionTypes tt = new TransactionTypes(ttypes); wrkld.setTransTypes(tt); if (benchmarkSettings.getBenchmarkSlaFile() != null) wrkld.setSlaFromFile(benchmarkSettings.getBenchmarkSlaFile()); LOG.debug("Tenant " + tenantID + ": Using the following transaction types: " + tt); bench.setTenantOffset(tenantEvents.get(tenantID).getTime(tenEvent)); bench.setTenantID(tenantID); bench.setBenchmarkSettings(benchmarkSettings); benchList.add(bench); } } } // create result collector ResultCollector rCollector = new ResultCollector(tenantList); // execute benchmarks in parallel ArrayList<Thread> benchThreads = new ArrayList<Thread>(); for (BenchmarkModule benchmark : benchList) { BenchmarkExecutor benchThread = new BenchmarkExecutor(benchmark, argsLine); Thread t = new Thread(benchThread); t.start(); benchThreads.add(t); benchmark.getWorkloadConfiguration().setrCollector(rCollector); } // waiting for completion of all benchmarks for (Thread t : benchThreads) { t.join(); } // print statistics int analysisBuckets = -1; if (argsLine.hasOption("analysis-buckets")) analysisBuckets = Integer.parseInt(argsLine.getOptionValue("analysis-buckets")); String output = null; if (argsLine.hasOption("o")) output = argsLine.getOptionValue("o"); String baseline = null; if (argsLine.hasOption("b")) baseline = argsLine.getOptionValue("b"); rCollector.printStatistics(output, analysisBuckets, argsLine.hasOption("histograms"), baseline); // create GUI if (argsLine.hasOption("g") && (!rCollector.getAllResults().isEmpty())) { try { Gui gui = new Gui(Integer.parseInt(argsLine.getOptionValue("analysis-buckets", "10")), rCollector, output); } catch (Exception e) { e.printStackTrace(); } } }
From source file:com.github.sdnwiselab.sdnwise.mote.standalone.Loader.java
/** * @param args the command line arguments *//* w w w. j av a 2 s. c o m*/ public static void main(final String[] args) { Options options = new Options(); options.addOption(Option.builder("n").argName("net").hasArg().required().desc("Network ID of the node") .numberOfArgs(1).build()); options.addOption(Option.builder("a").argName("address").hasArg().required() .desc("Address of the node <0-65535>").numberOfArgs(1).build()); options.addOption(Option.builder("p").argName("port").hasArg().required().desc("Listening UDP port") .numberOfArgs(1).build()); options.addOption(Option.builder("t").argName("filename").hasArg().required() .desc("Use given file for neighbors discovery").numberOfArgs(1).build()); options.addOption(Option.builder("c").argName("ip:port").hasArg() .desc("IP address and TCP port of the controller. (SINK ONLY)").numberOfArgs(1).build()); options.addOption(Option.builder("sp").argName("port").hasArg() .desc("Port number of the switch. (SINK ONLY)").numberOfArgs(1).build()); options.addOption(Option.builder("sm").argName("mac").hasArg() .desc("MAC address of the switch. Example: <00:00:00:00:00:00>." + " (SINK ONLY)").numberOfArgs(1) .build()); options.addOption(Option.builder("sd").argName("dpid").hasArg().desc("DPID of the switch (SINK ONLY)") .numberOfArgs(1).build()); options.addOption(Option.builder("l").argName("level").hasArg() .desc("Use given log level. Values: SEVERE, WARNING, INFO, " + "CONFIG, FINE, FINER, FINEST.") .numberOfArgs(1).optionalArg(true).build()); // create the parser CommandLineParser parser = new DefaultParser(); try { CommandLine line = parser.parse(options, args); Thread th; byte cmdNet = (byte) Integer.parseInt(line.getOptionValue("n")); NodeAddress cmdAddress = new NodeAddress(Integer.parseInt(line.getOptionValue("a"))); int cmdPort = Integer.parseInt(line.getOptionValue("p")); String cmdTopo = line.getOptionValue("t"); String cmdLevel; if (!line.hasOption("l")) { cmdLevel = "SEVERE"; } else { cmdLevel = line.getOptionValue("l"); } if (line.hasOption("c")) { if (!line.hasOption("sd")) { throw new ParseException("-sd option missing"); } if (!line.hasOption("sp")) { throw new ParseException("-sp option missing"); } if (!line.hasOption("sm")) { throw new ParseException("-sm option missing"); } String cmdSDpid = line.getOptionValue("sd"); String cmdSMac = line.getOptionValue("sm"); long cmdSPort = Long.parseLong(line.getOptionValue("sp")); String[] ipport = line.getOptionValue("c").split(":"); th = new Thread(new Sink(cmdNet, cmdAddress, cmdPort, new InetSocketAddress(ipport[0], Integer.parseInt(ipport[1])), cmdTopo, cmdLevel, cmdSDpid, cmdSMac, cmdSPort)); } else { th = new Thread(new Mote(cmdNet, cmdAddress, cmdPort, cmdTopo, cmdLevel)); } th.start(); th.join(); } catch (InterruptedException | ParseException ex) { System.out.println("Parsing failed. Reason: " + ex.getMessage()); HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("sdn-wise-data -n id -a address -p port" + " -t filename [-l level] [-sd dpid -sm mac -sp port]", options); } }
From source file:adapter.tdb.sync.clients.GetVocabularyOfMagicDrawAdapterAndPushToStore.java
public static void main(String[] args) { Thread thread = new Thread() { public void start() { URI vocabURI = URI.create("http://localhost:8080/oslc4jmagicdraw/services/sysml-rdfvocabulary"); // create an empty RDF model Model model = ModelFactory.createDefaultModel(); // use FileManager to read OSLC Resource Shape in RDF // String inputFileName = "file:C:/Users/Axel/git/EcoreMetamodel2OSLCSpecification/EcoreMetamodel2OSLCSpecification/Resource Shapes/Block.rdf"; // String inputFileName = "file:C:/Users/Axel/git/ecore2oslc/EcoreMetamodel2OSLCSpecification/RDF Vocabulary/sysmlRDFVocabulary of OMG.rdf"; // InputStream in = FileManager.get().open(inputFileName); // if (in == null) { // throw new IllegalArgumentException("File: " + inputFileName // + " not found"); // } // create TDB dataset String directory = TriplestoreUtil.getTriplestoreLocation(); Dataset dataset = TDBFactory.createDataset(directory); Model tdbModel = dataset.getDefaultModel(); try { HttpGet httpget = new HttpGet(vocabURI); httpget.addHeader("accept", "application/rdf+xml"); CloseableHttpClient httpClient = HttpClients.createDefault(); HttpResponse response = httpClient.execute(httpget); HttpEntity entity = response.getEntity(); if (entity != null) { InputStream inputStream = entity.getContent(); model.read(inputStream, null); tdbModel.add(model); }//w w w .j a va 2 s. co m } catch (IllegalArgumentException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (ClientProtocolException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } tdbModel.close(); dataset.close(); } }; thread.start(); try { thread.join(); System.out.println("Vocabulary of OSLC MagicDraw SysML adapter added to triplestore"); } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } }
From source file:edu.umn.cs.spatialHadoop.operations.RangeQuery.java
public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException { final OperationsParams params = new OperationsParams(new GenericOptionsParser(args)); final Path[] paths = params.getPaths(); if (paths.length <= 1 && !params.checkInput()) { printUsage();// w w w. j a va 2 s. com System.exit(1); } if (paths.length >= 2 && !params.checkInputOutput()) { printUsage(); System.exit(1); } if (params.get("rect") == null) { System.err.println("You must provide a query range"); printUsage(); System.exit(1); } final Path inPath = params.getInputPath(); final Path outPath = params.getOutputPath(); final Rectangle[] queryRanges = params.getShapes("rect", new Rectangle()); // All running jobs final Vector<Long> resultsCounts = new Vector<Long>(); Vector<Job> jobs = new Vector<Job>(); Vector<Thread> threads = new Vector<Thread>(); long t1 = System.currentTimeMillis(); for (int i = 0; i < queryRanges.length; i++) { final OperationsParams queryParams = new OperationsParams(params); OperationsParams.setShape(queryParams, "rect", queryRanges[i]); if (OperationsParams.isLocal(new JobConf(queryParams), inPath)) { // Run in local mode final Rectangle queryRange = queryRanges[i]; final Shape shape = queryParams.getShape("shape"); final Path output = outPath == null ? null : (queryRanges.length == 1 ? outPath : new Path(outPath, String.format("%05d", i))); Thread thread = new Thread() { @Override public void run() { FSDataOutputStream outFile = null; final byte[] newLine = System.getProperty("line.separator", "\n").getBytes(); try { ResultCollector<Shape> collector = null; if (output != null) { FileSystem outFS = output.getFileSystem(queryParams); final FSDataOutputStream foutFile = outFile = outFS.create(output); collector = new ResultCollector<Shape>() { final Text tempText = new Text2(); @Override public synchronized void collect(Shape r) { try { tempText.clear(); r.toText(tempText); foutFile.write(tempText.getBytes(), 0, tempText.getLength()); foutFile.write(newLine); } catch (IOException e) { e.printStackTrace(); } } }; } else { outFile = null; } long resultCount = rangeQueryLocal(inPath, queryRange, shape, queryParams, collector); resultsCounts.add(resultCount); } catch (IOException e) { e.printStackTrace(); } catch (InterruptedException e) { e.printStackTrace(); } finally { try { if (outFile != null) outFile.close(); } catch (IOException e) { e.printStackTrace(); } } } }; thread.start(); threads.add(thread); } else { // Run in MapReduce mode queryParams.setBoolean("background", true); Job job = rangeQueryMapReduce(inPath, outPath, queryParams); jobs.add(job); } } while (!jobs.isEmpty()) { Job firstJob = jobs.firstElement(); firstJob.waitForCompletion(false); if (!firstJob.isSuccessful()) { System.err.println("Error running job " + firstJob); System.err.println("Killing all remaining jobs"); for (int j = 1; j < jobs.size(); j++) jobs.get(j).killJob(); System.exit(1); } Counters counters = firstJob.getCounters(); Counter outputRecordCounter = counters.findCounter(Task.Counter.MAP_OUTPUT_RECORDS); resultsCounts.add(outputRecordCounter.getValue()); jobs.remove(0); } while (!threads.isEmpty()) { try { Thread thread = threads.firstElement(); thread.join(); threads.remove(0); } catch (InterruptedException e) { e.printStackTrace(); } } long t2 = System.currentTimeMillis(); System.out.println("Time for " + queryRanges.length + " jobs is " + (t2 - t1) + " millis"); System.out.println("Results counts: " + resultsCounts); }
From source file:com.yahoo.ycsb.db.AsyncHBaseClient.java
public static void main(String[] args) { if (args.length != 4) { System.out.println("usage: ahc zkquorum zkpath threadcount operation_count"); System.exit(0);// w w w . j ava2s. com } final int keyspace = 10000; //120000000; final String zkQuorum = args[0]; final String zkPath = args[1]; final int threadcount = Integer.parseInt(args[2]); final int opcount = Integer.parseInt(args[3]) / threadcount; Vector<Thread> allthreads = new Vector<Thread>(); for (int i = 0; i < threadcount; i++) { Thread t = new Thread() { public void run() { try { Random random = new Random(); AsyncHBaseClient cli = new AsyncHBaseClient(); Properties props = new Properties(); props.setProperty("columnfamily", "f1"); props.setProperty("zkquorum", zkQuorum); props.setProperty("zkpath", zkPath); props.setProperty("debug", "true"); cli.setProperties(props); cli.init(); HashMap<String, ByteIterator> result = Maps.newHashMap(); long accum = 0; for (int i = 0; i < opcount; i++) { int keynum = random.nextInt(keyspace); String key = "user" + keynum; long st = System.currentTimeMillis(); int rescode; HashMap<String, ByteIterator> hm = Maps.newHashMap(); hm.put("field1", new ByteArrayByteIterator("value1".getBytes("UTF-8"))); hm.put("field2", new ByteArrayByteIterator("value2".getBytes("UTF-8"))); hm.put("field3", new ByteArrayByteIterator("value3".getBytes("UTF-8"))); hm.put("efield", new ByteArrayByteIterator(HBaseClient.EMPTY_ARRAY)); rescode = cli.insert("bench", key, hm); HashSet<String> s = Sets.newHashSet(); s.add("field1"); s.add("field2"); rescode = cli.read("bench", key, s, result); rescode = cli.delete("bench", key); rescode = cli.read("bench", key, s, result); HashSet<String> scanFields = Sets.newHashSet(); scanFields.add("field1"); scanFields.add("field3"); Vector<HashMap<String, ByteIterator>> scanResults = new Vector<HashMap<String, ByteIterator>>(); rescode = cli.scan("bench", "user2", 20, null, scanResults); long en = System.currentTimeMillis(); accum += (en - st); if (rescode != Ok) { System.out.println("Error " + rescode + " for " + key); } if (i % 10 == 0) { System.out.println( i + " operations, average latency: " + (((double) accum) / ((double) i))); } } System.out.println(new ToStringBuilder(cli._client.stats(), ToStringStyle.MULTI_LINE_STYLE) .toString()); } catch (Exception e) { e.printStackTrace(); } } }; allthreads.add(t); } long st = System.currentTimeMillis(); for (Thread t : allthreads) { t.start(); } for (Thread t : allthreads) { try { t.join(); } catch (InterruptedException e) { } } long en = System.currentTimeMillis(); System.out.println("Throughput: " + ((1000.0) * (((double) (opcount * threadcount)) / ((double) (en - st)))) + " ops/sec"); }
From source file:fresto.datastore.titan.TitanEventWriter.java
public static void main(String[] args) throws Exception { if (args.length < 4) { LOGGER.error(//from www .jav a 2s . co m "Argumests needed : <streamer url> <storage backend> <storage hostname> <max commit count>"); System.exit(1); } else { STREAMER_URL = args[0]; STORAGE_BACKEND = args[1]; STORAGE_HOSTNAME = args[2]; try { MAX_COMMIT_COUNT = Integer.parseInt(args[3]); } catch (NumberFormatException e) { LOGGER.error("Commit count should be an integer"); } } final ZMQ.Context context = ZMQ.context(1); final FrestoEventQueue frestoEventQueue = new FrestoEventQueue(); final Thread queueMonitorThread = new Thread() { //Logger _LOGGER = Logger.getLogger("writerThread"); @Override public void run() { while (work) { try { LOGGER.info("frestoEventQueue size = " + frestoEventQueue.size()); Thread.sleep(1000); } catch (InterruptedException ie) { } } } }; final Thread writerThread = new Thread() { //Logger _LOGGER = Logger.getLogger("writerThread"); @Override public void run() { //StopWatch _watch = new JavaLogStopWatch(_LOGGER); FrestoStopWatch _watch = new FrestoStopWatch(); FrestoStopWatch _durationWatch = new FrestoStopWatch(); TitanEventWriter eventWriter = new TitanEventWriter(); // Open database eventWriter.openTitanGraph(); ZMQ.Socket puller = context.socket(ZMQ.PULL); //puller.connect("tcp://" + frontHost + ":" + frontPort); puller.connect(STREAMER_URL); //Consume socket data frestoEventQueue.setPullerSocket(puller); frestoEventQueue.start(); int nextCommitCount = 0; int count = 0; long elapsedTime = 0; long duration = 0; _durationWatch.start(); while (work) { // To wait until at least one event in queue if (frestoEventQueue.isEmpty()) { try { Thread.sleep(SLEEP_TIME); continue; } catch (InterruptedException ie) { } } nextCommitCount = frestoEventQueue.size(); for (int i = 0; i < nextCommitCount; i++) { _watch.start(); count++; FrestoEvent frestoEvent = frestoEventQueue.poll(); try { eventWriter.writeEventData(frestoEvent.topic, frestoEvent.eventBytes); } catch (Exception e) { e.printStackTrace(); } finally { // } elapsedTime += _watch.stop(); duration += _durationWatch.stop(); if (count == MAX_COMMIT_COUNT) { eventWriter.commitGraph(); LOGGER.info(count + " events processed for " + elapsedTime + " ms. (total time " + duration + " ms.) Remaining events " + frestoEventQueue.size()); count = 0; elapsedTime = 0; duration = 0; // Stop FOR clause } } eventWriter.commitGraph(); LOGGER.info(count + " events processed for " + elapsedTime + " ms. (total time " + duration + " ms.) Remaining events " + frestoEventQueue.size()); count = 0; elapsedTime = 0; duration = 0; } LOGGER.info("Shutting down..."); if (g.isOpen()) { g.commit(); g.shutdown(); } puller.close(); context.term(); LOGGER.info("Good bye."); } }; Runtime.getRuntime().addShutdownHook(new Thread() { @Override public void run() { System.out.println("Interrupt received, killing server"); // To break while clause frestoEventQueue.stopWork(); work = false; try { writerThread.join(); frestoEventQueue.join(); //queueMonitorThread.join(); } catch (InterruptedException e) { } } }); //queueMonitorThread.start(); writerThread.start(); }
From source file:edu.umn.cs.sthadoop.trajectory.TrajectoryOverlap.java
public static void main(String[] args) throws Exception { // args = new String[8]; // args[0] = "/export/scratch/mntgData/geolifeGPS/geolife_Trajectories_1.3/HDFS/index_geolife"; // args[1] = "/export/scratch/mntgData/geolifeGPS/geolife_Trajectories_1.3/HDFS/knn-dis-result"; // args[2] = "shape:edu.umn.cs.sthadoop.trajectory.GeolifeTrajectory"; // args[3] = "interval:2008-05-01,2008-05-30"; // args[4] = "time:month"; // args[5] = "traj:39.9119983,116.606835;39.9119783,116.6065483;39.9119599,116.6062649;39.9119416,116.6059899;39.9119233,116.6057282;39.9118999,116.6054783;39.9118849,116.6052366;39.9118666,116.6050099;39.91185,116.604775;39.9118299,116.604525;39.9118049,116.6042649;39.91177,116.6040166;39.9117516,116.6037583;39.9117349,116.6035066;39.9117199,116.6032666;39.9117083,116.6030232;39.9117,116.6027566;39.91128,116.5969383;39.9112583,116.5966766;39.9112383,116.5964232;39.9112149,116.5961699;39.9111933,116.5959249;39.9111716,116.5956883"; // args[6] = "-overwrite"; // args[7] = "-local";//"-no-local"; final OperationsParams params = new OperationsParams(new GenericOptionsParser(args)); final Path[] paths = params.getPaths(); if (paths.length <= 1 && !params.checkInput()) { printUsage();//w w w . java 2 s. co m System.exit(1); } if (paths.length >= 2 && !params.checkInputOutput()) { printUsage(); System.exit(1); } if (params.get("traj") == null) { System.err.println("Trajectory query is missing"); printUsage(); System.exit(1); } // Invoke method to compute the trajectory MBR. String rectangle = getTrajectoryRectangle(params.get("traj")); params.set("rect", rectangle); if (params.get("rect") == null) { System.err.println("You must provide a Trajectory Query"); printUsage(); System.exit(1); } if (params.get("interval") == null) { System.err.println("Temporal range missing"); printUsage(); System.exit(1); } TextSerializable inObj = params.getShape("shape"); if (!(inObj instanceof STPoint)) { LOG.error("Shape is not instance of STPoint"); printUsage(); System.exit(1); } // Get spatio-temporal slices. List<Path> STPaths = getIndexedSlices(params); final Path outPath = params.getOutputPath(); final Rectangle[] queryRanges = params.getShapes("rect", new Rectangle()); // All running jobs final Vector<Long> resultsCounts = new Vector<Long>(); Vector<Job> jobs = new Vector<Job>(); Vector<Thread> threads = new Vector<Thread>(); long t1 = System.currentTimeMillis(); for (Path stPath : STPaths) { final Path inPath = stPath; for (int i = 0; i < queryRanges.length; i++) { final OperationsParams queryParams = new OperationsParams(params); OperationsParams.setShape(queryParams, "rect", queryRanges[i]); if (OperationsParams.isLocal(new JobConf(queryParams), inPath)) { // Run in local mode final Rectangle queryRange = queryRanges[i]; final Shape shape = queryParams.getShape("shape"); final Path output = outPath == null ? null : (queryRanges.length == 1 ? outPath : new Path(outPath, String.format("%05d", i))); Thread thread = new Thread() { @Override public void run() { FSDataOutputStream outFile = null; final byte[] newLine = System.getProperty("line.separator", "\n").getBytes(); try { ResultCollector<Shape> collector = null; if (output != null) { FileSystem outFS = output.getFileSystem(queryParams); final FSDataOutputStream foutFile = outFile = outFS.create(output); collector = new ResultCollector<Shape>() { final Text tempText = new Text2(); @Override public synchronized void collect(Shape r) { try { tempText.clear(); r.toText(tempText); foutFile.write(tempText.getBytes(), 0, tempText.getLength()); foutFile.write(newLine); } catch (IOException e) { e.printStackTrace(); } } }; } else { outFile = null; } long resultCount = rangeQueryLocal(inPath, queryRange, shape, queryParams, collector); resultsCounts.add(resultCount); } catch (IOException e) { e.printStackTrace(); } catch (InterruptedException e) { e.printStackTrace(); } finally { try { if (outFile != null) outFile.close(); } catch (IOException e) { e.printStackTrace(); } } } }; thread.start(); threads.add(thread); } else { // Run in MapReduce mode Path outTempPath = outPath == null ? null : new Path(outPath, String.format("%05d", i) + "-" + inPath.getName()); queryParams.setBoolean("background", true); Job job = rangeQueryMapReduce(inPath, outTempPath, queryParams); jobs.add(job); } } } while (!jobs.isEmpty()) { Job firstJob = jobs.firstElement(); firstJob.waitForCompletion(false); if (!firstJob.isSuccessful()) { System.err.println("Error running job " + firstJob); System.err.println("Killing all remaining jobs"); for (int j = 1; j < jobs.size(); j++) jobs.get(j).killJob(); System.exit(1); } Counters counters = firstJob.getCounters(); Counter outputRecordCounter = counters.findCounter(Task.Counter.MAP_OUTPUT_RECORDS); resultsCounts.add(outputRecordCounter.getValue()); jobs.remove(0); } while (!threads.isEmpty()) { try { Thread thread = threads.firstElement(); thread.join(); threads.remove(0); } catch (InterruptedException e) { e.printStackTrace(); } } long t2 = System.currentTimeMillis(); System.out.println("QueryPlan:"); for (Path stPath : STPaths) { System.out.println(stPath.getName()); } System.out.println("Time for " + queryRanges.length + " jobs is " + (t2 - t1) + " millis"); System.out.println("Results counts: " + resultsCounts); }
From source file:edu.umn.cs.sthadoop.operations.STRangeQuery.java
public static void main(String[] args) throws Exception { // args = new String[7]; // args[0] = "/home/louai/nyc-taxi/yellowIndex"; // args[1] = "/home/louai/nyc-taxi/resultSTRQ"; // args[2] = "shape:edu.umn.cs.sthadoop.core.STPoint"; // args[3] = "rect:-74.98451232910156,35.04014587402344,-73.97936248779295,41.49399566650391"; // args[4] = "interval:2015-01-01,2015-01-02"; // args[5] = "-overwrite"; // args[6] = "-no-local"; // Query for test with output // args = new String[6]; // args[0] = "/home/louai/nyc-taxi/yellowIndex"; // args[1] = "shape:edu.umn.cs.sthadoop.core.STPoint"; // args[2] = "rect:-74.98451232910156,35.04014587402344,-73.97936248779295,41.49399566650391"; // args[3] = "interval:2015-01-01,2015-01-03"; // args[4] = "-overwrite"; // args[5 ] = "-no-local"; final OperationsParams params = new OperationsParams(new GenericOptionsParser(args)); final Path[] paths = params.getPaths(); if (paths.length <= 1 && !params.checkInput()) { printUsage();//www.jav a 2 s .c o m System.exit(1); } if (paths.length >= 2 && !params.checkInputOutput()) { printUsage(); System.exit(1); } if (params.get("rect") == null) { String x1 = "-" + Double.toString(Double.MAX_VALUE); String y1 = "-" + Double.toString(Double.MAX_VALUE); String x2 = Double.toString(Double.MAX_VALUE); String y2 = Double.toString(Double.MAX_VALUE); System.out.println(x1 + "," + y1 + "," + x2 + "," + y2); params.set("rect", x1 + "," + y1 + "," + x2 + "," + y2); // System.err.println("You must provide a query range"); // printUsage(); // System.exit(1); } if (params.get("interval") == null) { System.err.println("Temporal range missing"); printUsage(); System.exit(1); } TextSerializable inObj = params.getShape("shape"); if (!(inObj instanceof STPoint) && !(inObj instanceof STRectangle)) { LOG.error("Shape is not instance of STPoint or STRectangle"); printUsage(); System.exit(1); } // Get spatio-temporal slices. List<Path> STPaths = getIndexedSlices(params); final Path outPath = params.getOutputPath(); final Rectangle[] queryRanges = params.getShapes("rect", new Rectangle()); // All running jobs final Vector<Long> resultsCounts = new Vector<Long>(); Vector<Job> jobs = new Vector<Job>(); Vector<Thread> threads = new Vector<Thread>(); long t1 = System.currentTimeMillis(); for (Path stPath : STPaths) { final Path inPath = stPath; for (int i = 0; i < queryRanges.length; i++) { final OperationsParams queryParams = new OperationsParams(params); OperationsParams.setShape(queryParams, "rect", queryRanges[i]); if (OperationsParams.isLocal(new JobConf(queryParams), inPath)) { // Run in local mode final Rectangle queryRange = queryRanges[i]; final Shape shape = queryParams.getShape("shape"); final Path output = outPath == null ? null : (queryRanges.length == 1 ? outPath : new Path(outPath, String.format("%05d", i))); Thread thread = new Thread() { @Override public void run() { FSDataOutputStream outFile = null; final byte[] newLine = System.getProperty("line.separator", "\n").getBytes(); try { ResultCollector<Shape> collector = null; if (output != null) { FileSystem outFS = output.getFileSystem(queryParams); final FSDataOutputStream foutFile = outFile = outFS.create(output); collector = new ResultCollector<Shape>() { final Text tempText = new Text2(); @Override public synchronized void collect(Shape r) { try { tempText.clear(); r.toText(tempText); foutFile.write(tempText.getBytes(), 0, tempText.getLength()); foutFile.write(newLine); } catch (IOException e) { e.printStackTrace(); } } }; } else { outFile = null; } long resultCount = rangeQueryLocal(inPath, queryRange, shape, queryParams, collector); resultsCounts.add(resultCount); } catch (IOException e) { e.printStackTrace(); } catch (InterruptedException e) { e.printStackTrace(); } finally { try { if (outFile != null) outFile.close(); } catch (IOException e) { e.printStackTrace(); } } } }; thread.start(); threads.add(thread); } else { // Run in MapReduce mode Path outTempPath = outPath == null ? null : new Path(outPath, String.format("%05d", i) + "-" + inPath.getName()); queryParams.setBoolean("background", true); Job job = rangeQueryMapReduce(inPath, outTempPath, queryParams); jobs.add(job); } } } while (!jobs.isEmpty()) { Job firstJob = jobs.firstElement(); firstJob.waitForCompletion(false); if (!firstJob.isSuccessful()) { System.err.println("Error running job " + firstJob); System.err.println("Killing all remaining jobs"); for (int j = 1; j < jobs.size(); j++) jobs.get(j).killJob(); System.exit(1); } Counters counters = firstJob.getCounters(); Counter outputRecordCounter = counters.findCounter(Task.Counter.MAP_OUTPUT_RECORDS); resultsCounts.add(outputRecordCounter.getValue()); jobs.remove(0); } while (!threads.isEmpty()) { try { Thread thread = threads.firstElement(); thread.join(); threads.remove(0); } catch (InterruptedException e) { e.printStackTrace(); } } long t2 = System.currentTimeMillis(); System.out.println("QueryPlan:"); for (Path stPath : STPaths) { System.out.println(stPath.getName()); } System.out.println("Time for " + queryRanges.length + " jobs is " + (t2 - t1) + " millis"); System.out.println("Results counts: " + resultsCounts); }