List of usage examples for java.lang String valueOf
public static String valueOf(double d)
From source file:de.tudarmstadt.ukp.experiments.dip.wp1.documents.Step6HITPreparator.java
public static void main(String[] args) throws Exception { // input dir - list of xml query containers // step5-linguistic-annotation/ System.err.println("Starting step 6 HIT Preparation"); File inputDir = new File(args[0]); // output dir File outputDir = new File(args[1]); if (outputDir.exists()) { outputDir.delete();//from w ww . j a v a2 s. c o m } outputDir.mkdir(); List<String> queries = new ArrayList<>(); // iterate over query containers int countClueWeb = 0; int countSentence = 0; for (File f : FileUtils.listFiles(inputDir, new String[] { "xml" }, false)) { QueryResultContainer queryResultContainer = QueryResultContainer .fromXML(FileUtils.readFileToString(f, "utf-8")); if (queries.contains(f.getName()) || queries.size() == 0) { // groups contain only non-empty documents Map<Integer, List<QueryResultContainer.SingleRankedResult>> groups = new HashMap<>(); // split to groups according to number of sentences for (QueryResultContainer.SingleRankedResult rankedResult : queryResultContainer.rankedResults) { if (rankedResult.originalXmi != null) { byte[] bytes = new BASE64Decoder() .decodeBuffer(new ByteArrayInputStream(rankedResult.originalXmi.getBytes())); JCas jCas = JCasFactory.createJCas(); XmiCasDeserializer.deserialize(new ByteArrayInputStream(bytes), jCas.getCas()); Collection<Sentence> sentences = JCasUtil.select(jCas, Sentence.class); int groupId = sentences.size() / 40; if (rankedResult.originalXmi == null) { System.err.println("Empty document: " + rankedResult.clueWebID); } else { if (!groups.containsKey(groupId)) { groups.put(groupId, new ArrayList<>()); } } //handle it groups.get(groupId).add(rankedResult); countClueWeb++; } } for (Map.Entry<Integer, List<QueryResultContainer.SingleRankedResult>> entry : groups.entrySet()) { Integer groupId = entry.getKey(); List<QueryResultContainer.SingleRankedResult> rankedResults = entry.getValue(); // make sure the results are sorted // DEBUG // for (QueryResultContainer.SingleRankedResult r : rankedResults) { // System.out.print(r.rank + "\t"); // } Collections.sort(rankedResults, (o1, o2) -> o1.rank.compareTo(o2.rank)); // iterate over results for one query and group for (int i = 0; i < rankedResults.size() && i < TOP_RESULTS_PER_GROUP; i++) { QueryResultContainer.SingleRankedResult rankedResult = rankedResults.get(i); QueryResultContainer.SingleRankedResult r = rankedResults.get(i); int rank = r.rank; MustacheFactory mf = new DefaultMustacheFactory(); Mustache mustache = mf.compile("template/template.html"); String queryId = queryResultContainer.qID; String query = queryResultContainer.query; // make the first letter uppercase query = query.substring(0, 1).toUpperCase() + query.substring(1); List<String> relevantInformationExamples = queryResultContainer.relevantInformationExamples; List<String> irrelevantInformationExamples = queryResultContainer.irrelevantInformationExamples; byte[] bytes = new BASE64Decoder() .decodeBuffer(new ByteArrayInputStream(rankedResult.originalXmi.getBytes())); JCas jCas = JCasFactory.createJCas(); XmiCasDeserializer.deserialize(new ByteArrayInputStream(bytes), jCas.getCas()); List<generators.Sentence> sentences = new ArrayList<>(); List<Integer> paragraphs = new ArrayList<>(); paragraphs.add(0); for (WebParagraph webParagraph : JCasUtil.select(jCas, WebParagraph.class)) { for (Sentence s : JCasUtil.selectCovered(Sentence.class, webParagraph)) { String sentenceBegin = String.valueOf(s.getBegin()); generators.Sentence sentence = new generators.Sentence(s.getCoveredText(), sentenceBegin); sentences.add(sentence); countSentence++; } int SentenceID = paragraphs.get(paragraphs.size() - 1); if (sentences.size() > 120) while (SentenceID < sentences.size()) { if (!paragraphs.contains(SentenceID)) paragraphs.add(SentenceID); SentenceID = SentenceID + 120; } paragraphs.add(sentences.size()); } System.err.println("Output dir: " + outputDir); int startID = 0; int endID; for (int j = 0; j < paragraphs.size(); j++) { endID = paragraphs.get(j); int sentLength = endID - startID; if (sentLength > 120 || j == paragraphs.size() - 1) { if (sentLength > 120) { endID = paragraphs.get(j - 1); j--; } sentLength = endID - startID; if (sentLength <= 40) groupId = 40; else if (sentLength <= 80 && sentLength > 40) groupId = 80; else if (sentLength > 80) groupId = 120; File folder = new File(outputDir + "/" + groupId); if (!folder.exists()) { System.err.println("creating directory: " + outputDir + "/" + groupId); boolean result = false; try { folder.mkdir(); result = true; } catch (SecurityException se) { //handle it } if (result) { System.out.println("DIR created"); } } String newHtmlFile = folder.getAbsolutePath() + "/" + f.getName() + "_" + rankedResult.clueWebID + "_" + sentLength + ".html"; System.err.println("Printing a file: " + newHtmlFile); File newHTML = new File(newHtmlFile); int t = 0; while (newHTML.exists()) { newHTML = new File(folder.getAbsolutePath() + "/" + f.getName() + "_" + rankedResult.clueWebID + "_" + sentLength + "." + t + ".html"); t++; } mustache.execute(new PrintWriter(new FileWriter(newHTML)), new generators(query, relevantInformationExamples, irrelevantInformationExamples, sentences.subList(startID, endID), queryId, rank)) .flush(); startID = endID; } } } } } } System.out.println("Printed " + countClueWeb + " documents with " + countSentence + " sentences"); }
From source file:io.s4.util.LoadGenerator.java
public static void main(String args[]) { Options options = new Options(); boolean warmUp = false; options.addOption(/* w w w . ja v a 2s. c om*/ OptionBuilder.withArgName("rate").hasArg().withDescription("Rate (events per second)").create("r")); options.addOption(OptionBuilder.withArgName("display_rate").hasArg() .withDescription("Display Rate at specified second boundary").create("d")); options.addOption(OptionBuilder.withArgName("start_boundary").hasArg() .withDescription("Start boundary in seconds").create("b")); options.addOption(OptionBuilder.withArgName("run_for").hasArg() .withDescription("Run for a specified number of seconds").create("x")); options.addOption(OptionBuilder.withArgName("cluster_manager").hasArg().withDescription("Cluster manager") .create("z")); options.addOption(OptionBuilder.withArgName("sender_application_name").hasArg() .withDescription("Sender application name").create("a")); options.addOption(OptionBuilder.withArgName("listener_application_name").hasArg() .withDescription("Listener application name").create("g")); options.addOption( OptionBuilder.withArgName("sleep_overhead").hasArg().withDescription("Sleep overhead").create("o")); options.addOption(new Option("w", "Warm-up")); CommandLineParser parser = new GnuParser(); CommandLine line = null; try { // parse the command line arguments line = parser.parse(options, args); } catch (ParseException exp) { // oops, something went wrong System.err.println("Parsing failed. Reason: " + exp.getMessage()); System.exit(1); } int expectedRate = 250; if (line.hasOption("r")) { try { expectedRate = Integer.parseInt(line.getOptionValue("r")); } catch (Exception e) { System.err.println("Bad expected rate specified " + line.getOptionValue("r")); System.exit(1); } } int displayRateIntervalSeconds = 20; if (line.hasOption("d")) { try { displayRateIntervalSeconds = Integer.parseInt(line.getOptionValue("d")); } catch (Exception e) { System.err.println("Bad display rate value specified " + line.getOptionValue("d")); System.exit(1); } } int startBoundary = 2; if (line.hasOption("b")) { try { startBoundary = Integer.parseInt(line.getOptionValue("b")); } catch (Exception e) { System.err.println("Bad start boundary value specified " + line.getOptionValue("b")); System.exit(1); } } int updateFrequency = 0; if (line.hasOption("f")) { try { updateFrequency = Integer.parseInt(line.getOptionValue("f")); } catch (Exception e) { System.err.println("Bad query udpdate frequency specified " + line.getOptionValue("f")); System.exit(1); } System.out.printf("Update frequency is %d\n", updateFrequency); } int runForTime = 0; if (line.hasOption("x")) { try { runForTime = Integer.parseInt(line.getOptionValue("x")); } catch (Exception e) { System.err.println("Bad run for time specified " + line.getOptionValue("x")); System.exit(1); } System.out.printf("Run for time is %d\n", runForTime); } String clusterManagerAddress = null; if (line.hasOption("z")) { clusterManagerAddress = line.getOptionValue("z"); } String senderApplicationName = null; if (line.hasOption("a")) { senderApplicationName = line.getOptionValue("a"); } String listenerApplicationName = null; if (line.hasOption("a")) { listenerApplicationName = line.getOptionValue("g"); } if (listenerApplicationName == null) { listenerApplicationName = senderApplicationName; } long sleepOverheadMicros = -1; if (line.hasOption("o")) { try { sleepOverheadMicros = Long.parseLong(line.getOptionValue("o")); } catch (NumberFormatException e) { System.err.println("Bad sleep overhead specified " + line.getOptionValue("o")); System.exit(1); } System.out.printf("Specified sleep overhead is %d\n", sleepOverheadMicros); } if (line.hasOption("w")) { warmUp = true; } List loArgs = line.getArgList(); if (loArgs.size() < 1) { System.err.println("No input file specified"); System.exit(1); } String inputFilename = (String) loArgs.get(0); EventEmitter emitter = null; SerializerDeserializer serDeser = new KryoSerDeser(); CommLayerEmitter clEmitter = new CommLayerEmitter(); clEmitter.setAppName(senderApplicationName); clEmitter.setListenerAppName(listenerApplicationName); clEmitter.setClusterManagerAddress(clusterManagerAddress); clEmitter.setSenderId(String.valueOf(System.currentTimeMillis() / 1000)); clEmitter.setSerDeser(serDeser); clEmitter.init(); emitter = clEmitter; long endTime = 0; if (runForTime > 0) { endTime = System.currentTimeMillis() + (runForTime * 1000); } LoadGenerator loadGenerator = new LoadGenerator(); loadGenerator.setInputFilename(inputFilename); loadGenerator.setEventEmitter(clEmitter); loadGenerator.setDisplayRateInterval(displayRateIntervalSeconds); loadGenerator.setExpectedRate(expectedRate); loadGenerator.run(); System.exit(0); }
From source file:com.heliosdecompiler.bootstrapper.Bootstrapper.java
public static void main(String[] args) { Options options = new Options(); options.addOption(Option.builder("Xfu").longOpt("Xforceupdate").desc("Force the patching process").build()); options.addOption(Option.builder("Xh").longOpt("Xhelp").desc("Help!").build()); CommandLineParser parser = new DefaultParser(); try {// w w w . ja v a 2 s . c om CommandLine commandLine = parser.parse(options, args, true); if (commandLine.hasOption("Xhelp")) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("java -jar bootstrapper.jar", options); } else if (commandLine.hasOption("Xforceupdate")) { forceUpdate(); } else { // boolean shouldStart = true; // ServerSocket socket = null; // try { // socket = new ServerSocket(21354); // } catch (IOException e) { // shouldStart = false; // } finally { // if (socket != null) { // socket.close(); // } // } // if (shouldStart) { String[] forward = commandLine.getArgs(); HeliosData heliosData = loadHelios(); System.out.println("Running Helios version " + heliosData.buildNumber); System.getProperties().put("com.heliosdecompiler.buildNumber", String.valueOf(heliosData.buildNumber)); System.getProperties().put("com.heliosdecompiler.version", String.valueOf(heliosData.version)); System.getProperties().put("com.heliosdecompiler.args", args); new Thread(new UpdaterTask(heliosData.buildNumber)).start(); ClassLoader classLoader = new URLClassLoader(new URL[] { IMPL_FILE.toURI().toURL() }, null); Class<?> bootloader = Class.forName(heliosData.mainClass, false, classLoader); bootloader.getMethod("main", String[].class).invoke(null, new Object[] { forward }); // } else { // String message = Arrays.asList(forward).stream().collect(Collectors.joining(" ")); // Socket clientSocket = new Socket("127.0.0.1", 21354); // clientSocket.getOutputStream().write(message.getBytes(StandardCharsets.UTF_8)); // clientSocket.getOutputStream().close(); // clientSocket.close(); // } } } catch (Throwable t) { displayError(t); System.exit(1); } }
From source file:jeffaschenk.tomcat.zuul.util.KeyFileUtils.java
/** * Utility Main to provide capabilities to generate a protected KeyFile for use with Zuul * * @param args - Incoming program Arguments * @throws Exception/*from w w w. j ava2 s. c om*/ */ public static void main(String[] args) throws Exception { // Initialize our Ciphers KeyFileUtils keyFileUtils = new KeyFileUtils(); keyFileUtils.InitCiphers(); // Assume we our sole purpose in life is to prompt for a Password, which is the same, // ensure we have an argument to be out filename. if ((args == null) || (args.length != 1)) { System.out.println("Invalid number of Arguments specified!"); System.out.println( "Usage: " + keyFileUtils.getClass().getName() + " <Fully Qualified Output Key Filename>"); return; } // Validate Argument as a file. File outputKeyFile = new File(args[0]); if (outputKeyFile.exists()) { // Prompt to overwrite... char[] answer = readPassword("Specified Key File:[" + outputKeyFile + "], exists. Overwrite? (y/n)? "); if ((answer == null) || (answer.length == 0) || (!String.valueOf(answer).toUpperCase().startsWith("Y"))) { System.out.println( "Existing Key File:[" + outputKeyFile + "], exists. Will not be Overwritten, Done."); return; } else { outputKeyFile.delete(); } } // Prompt Operator for the Zuul Encryption Password to be encrypted in a file and saved for later access. // Prompt to overwrite... char[] password = obtainValidatedPassword("Enter Zuul Encryption Password: "); if (password == null) { System.out.println("Unable to obtain Password, Done."); return; } // Create our Input and Output Streams for the encryption of the Password. InputStream is = IOUtils.toInputStream(String.valueOf(password), "UTF-8"); FileOutputStream fos = new FileOutputStream(outputKeyFile); // Encrypt keyFileUtils.CBCEncrypt(is, fos); // Now lets us Validate the file. keyFileUtils.ResetCiphers(); keyFileUtils = new KeyFileUtils(); keyFileUtils.InitCiphers(); // Obtain the Decrypted Contents byte[] obtainedDecryptedData = keyFileUtils.CBCDecrypt(new FileInputStream(outputKeyFile)); // Compare to validate... boolean valid = String.valueOf(password).equals(new String(obtainedDecryptedData)); Arrays.fill(password, ' '); Arrays.fill(obtainedDecryptedData, (byte) 0x00); if (valid) { System.out.println("Key File:[" + outputKeyFile + "], validated and available for use, Done."); return; } else { System.out.println("Key File:[" + outputKeyFile + "], Did not Validate, very bad!"); return; } }
From source file:com.weibo.wesync.client.NHttpClient2.java
public static void main(String[] args) throws Exception { RSAPublicKey publicKey = RSAEncrypt.loadPublicKey("D:\\weibo\\meyou_gw\\conf\\public.pem"); // // ww w .j av a 2 s . c o m byte[] cipher = RSAEncrypt.encrypt(publicKey, password.getBytes()); password = RSAEncrypt.toHexString(cipher); // HTTP parameters for the client HttpParams params = new SyncBasicHttpParams(); params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT, 30000) .setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, 30000) .setIntParameter(CoreConnectionPNames.SOCKET_BUFFER_SIZE, 8 * 1024) .setBooleanParameter(CoreConnectionPNames.TCP_NODELAY, true); // Create HTTP protocol processing chain HttpProcessor httpproc = new ImmutableHttpProcessor(new HttpRequestInterceptor[] { // Use standard client-side protocol interceptors new RequestContent(), new RequestTargetHost(), new RequestConnControl(), new RequestUserAgent(), new RequestExpectContinue() }); // Create client-side HTTP protocol handler HttpAsyncRequestExecutor protocolHandler = new HttpAsyncRequestExecutor(); // Create client-side I/O event dispatch final IOEventDispatch ioEventDispatch = new DefaultHttpClientIODispatch(protocolHandler, params); // Create client-side I/O reactor IOReactorConfig config = new IOReactorConfig(); config.setIoThreadCount(1); final ConnectingIOReactor ioReactor = new DefaultConnectingIOReactor(config); // Create HTTP connection pool BasicNIOConnPool pool = new BasicNIOConnPool(ioReactor, params); // Limit total number of connections to just two pool.setDefaultMaxPerRoute(2); pool.setMaxTotal(1); // Run the I/O reactor in a separate thread Thread t = new Thread(new Runnable() { public void run() { try { // Ready to go! ioReactor.execute(ioEventDispatch); } catch (InterruptedIOException ex) { System.err.println("Interrupted"); } catch (IOException e) { System.err.println("I/O error: " + e.getMessage()); } System.out.println("Shutdown"); } }); // Start the client thread t.start(); // Create HTTP requester // HttpAsyncRequester requester = new HttpAsyncRequester( // httpproc, new DefaultConnectionReuseStrategy(), params); // Execute HTTP GETs to the following hosts and HttpHost[] targets = new HttpHost[] { // new HttpHost("123.125.106.28", 8093, "http"), // new HttpHost("123.125.106.28", 8093, "http"), // new HttpHost("123.125.106.28", 8093, "http"), // new HttpHost("123.125.106.28", 8093, "http"), // new HttpHost("123.125.106.28", 8093, "http"), // new HttpHost("123.125.106.28", 8093, "http"), // new HttpHost("123.125.106.28", 8093, "http"), // new HttpHost("123.125.106.28", 8093, "http"), // new HttpHost("123.125.106.28", 8093, "http"), // new HttpHost("123.125.106.28", 8093, "http"), // new HttpHost("123.125.106.28", 8093, "http"), new HttpHost("123.125.106.28", 8082, "http") }; final CountDownLatch latch = new CountDownLatch(targets.length); int callbackId = 0; for (int i = 0; i < 1; i++) { for (final HttpHost target : targets) { BasicHttpEntityEnclosingRequest request = new BasicHttpEntityEnclosingRequest("POST", "/wesync"); // String usrpwd = Base64.encodeBase64String((username + ":" + password).getBytes()); // request.setHeader("authorization", "Basic " + usrpwd); request.setHeader("uid", "2565640713"); Meyou.MeyouPacket packet = null; if (callbackId == 0) { packet = Meyou.MeyouPacket.newBuilder().setCallbackId(String.valueOf(callbackId++)) .setSort(MeyouSort.notice).build(); } else { packet = Meyou.MeyouPacket.newBuilder().setCallbackId(String.valueOf(callbackId++)) .setSort(MeyouSort.wesync).build(); } ByteArrayEntity entity = new ByteArrayEntity(packet.toByteArray()); request.setEntity(entity); // BasicHttpRequest request = new BasicHttpRequest("GET", "/test.html"); System.out.println("send ..."); HttpAsyncRequester requester = new HttpAsyncRequester(httpproc, new DefaultConnectionReuseStrategy(), params); requester.execute(new BasicAsyncRequestProducer(target, request), new BasicAsyncResponseConsumer(), pool, new BasicHttpContext(), // Handle HTTP response from a callback new FutureCallback<HttpResponse>() { public void completed(final HttpResponse response) { StatusLine status = response.getStatusLine(); int code = status.getStatusCode(); if (code == 200) { try { latch.countDown(); DataInputStream in; in = new DataInputStream(response.getEntity().getContent()); int packetLength = in.readInt(); int start = 0; while (packetLength > 0) { ByteArrayOutputStream outstream = new ByteArrayOutputStream( packetLength); byte[] buffer = new byte[1024]; int len = 0; while (start < packetLength && (len = in.read(buffer, start, packetLength)) > 0) { outstream.write(buffer, 0, len); start += len; } Meyou.MeyouPacket packet0 = Meyou.MeyouPacket .parseFrom(outstream.toByteArray()); System.out.println(target + "->" + packet0); if ((len = in.read(buffer, start, 4)) > 0) { packetLength = Util.readPacketLength(buffer); } else { break; } } } catch (IllegalStateException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } else { System.out.println("error code=" + code + "|" + status.getReasonPhrase()); } } public void failed(final Exception ex) { latch.countDown(); System.out.println(target + "->" + ex); } public void cancelled() { latch.countDown(); System.out.println(target + " cancelled"); } }); Thread.sleep((long) (Math.random() * 10000)); } } // latch.await(); // System.out.println("Shutting down I/O reactor"); // ioReactor.shutdown(); // System.out.println("Done"); }
From source file:com.mvdb.etl.actions.ExtractDBChanges.java
public static void main(String[] args) throws JSONException { ActionUtils.setUpInitFileProperty(); // boolean success = ActionUtils.markActionChainBroken("Just Testing"); // System.exit(success ? 0 : 1); ActionUtils.assertActionChainNotBroken(); ActionUtils.assertEnvironmentSetupOk(); ActionUtils.assertFileExists("~/.mvdb", "~/.mvdb missing. Existing."); ActionUtils.assertFileExists("~/.mvdb/status.InitCustomerData.complete", "300init-customer-data.sh not executed yet. Exiting"); //This check is not required as data can be modified any number of times //ActionUtils.assertFileDoesNotExist("~/.mvdb/status.ModifyCustomerData.complete", "ModifyCustomerData already done. Start with 100init.sh if required. Exiting"); ActionUtils.createMarkerFile("~/.mvdb/status.ExtractDBChanges.start", true); //String schemaDescription = "{ 'root' : [{'table' : 'orders', 'keyColumn' : 'order_id', 'updateTimeColumn' : 'update_time'}]}"; String customerName = null;/*w ww. jav a2s.c om*/ final CommandLineParser cmdLinePosixParser = new PosixParser(); final Options posixOptions = constructPosixOptions(); CommandLine commandLine; try { commandLine = cmdLinePosixParser.parse(posixOptions, args); if (commandLine.hasOption("customer")) { customerName = commandLine.getOptionValue("customer"); } } catch (ParseException parseException) // checked exception { System.err.println( "Encountered exception while parsing using PosixParser:\n" + parseException.getMessage()); } if (customerName == null) { System.err.println("Could not find customerName. Aborting..."); System.exit(1); } ApplicationContext context = Top.getContext(); final OrderDAO orderDAO = (OrderDAO) context.getBean("orderDAO"); final ConfigurationDAO configurationDAO = (ConfigurationDAO) context.getBean("configurationDAO"); final GenericDAO genericDAO = (GenericDAO) context.getBean("genericDAO"); File snapshotDirectory = getSnapshotDirectory(configurationDAO, customerName); try { FileUtils.writeStringToFile(new File("/tmp/etl.extractdbchanges.directory.txt"), snapshotDirectory.getName(), false); } catch (IOException e) { e.printStackTrace(); System.exit(1); return; } long currentTime = new Date().getTime(); Configuration lastRefreshTimeConf = configurationDAO.find(customerName, "last-refresh-time"); Configuration schemaDescriptionConf = configurationDAO.find(customerName, "schema-description"); long lastRefreshTime = Long.parseLong(lastRefreshTimeConf.getValue()); OrderJsonFileConsumer orderJsonFileConsumer = new OrderJsonFileConsumer(snapshotDirectory); Map<String, ColumnMetadata> metadataMap = orderDAO.findMetadata(); //write file schema-orders.dat in snapshotDirectory genericDAO.fetchMetadata("orders", snapshotDirectory); //writes files: header-orders.dat, data-orders.dat in snapshotDirectory JSONObject json = new JSONObject(schemaDescriptionConf.getValue()); JSONArray rootArray = json.getJSONArray("root"); int length = rootArray.length(); for (int i = 0; i < length; i++) { JSONObject jsonObject = rootArray.getJSONObject(i); String table = jsonObject.getString("table"); String keyColumnName = jsonObject.getString("keyColumn"); String updateTimeColumnName = jsonObject.getString("updateTimeColumn"); System.out.println("table:" + table + ", keyColumn: " + keyColumnName + ", updateTimeColumn: " + updateTimeColumnName); genericDAO.fetchAll2(snapshotDirectory, new Timestamp(lastRefreshTime), table, keyColumnName, updateTimeColumnName); } //Unlikely failure //But Need to factor this into a separate task so that extraction does not have to be repeated. //Extraction is an expensive task. try { String sourceDirectoryAbsolutePath = snapshotDirectory.getAbsolutePath(); File sourceRelativeDirectoryPath = getRelativeSnapShotDirectory(configurationDAO, sourceDirectoryAbsolutePath); String hdfsRoot = ActionUtils.getConfigurationValue(ConfigurationKeys.GLOBAL_CUSTOMER, ConfigurationKeys.GLOBAL_HDFS_ROOT); String targetDirectoryFullPath = hdfsRoot + "/data" + sourceRelativeDirectoryPath; ActionUtils.copyLocalDirectoryToHdfsDirectory(sourceDirectoryAbsolutePath, targetDirectoryFullPath); String dirName = snapshotDirectory.getName(); ActionUtils.setConfigurationValue(customerName, ConfigurationKeys.LAST_COPY_TO_HDFS_DIRNAME, dirName); } catch (Throwable e) { e.printStackTrace(); logger.error("Objects Extracted from database. But copy of snapshot directory<" + snapshotDirectory.getAbsolutePath() + "> to hdfs <" + "" + ">failed. Fix the problem and redo extract.", e); System.exit(1); } //Unlikely failure //But Need to factor this into a separate task so that extraction does not have to be repeated. //Extraction is an expensive task. String targetZip = null; try { File targetZipDirectory = new File(snapshotDirectory.getParent(), "archives"); if (!targetZipDirectory.exists()) { boolean success = targetZipDirectory.mkdirs(); if (success == false) { logger.error("Objects copied to hdfs. But able to create archive directory <" + targetZipDirectory.getAbsolutePath() + ">. Fix the problem and redo extract."); System.exit(1); } } targetZip = new File(targetZipDirectory, snapshotDirectory.getName() + ".zip").getAbsolutePath(); ActionUtils.zipFullDirectory(snapshotDirectory.getAbsolutePath(), targetZip); } catch (Throwable e) { e.printStackTrace(); logger.error("Objects copied to hdfs. But zipping of snapshot directory<" + snapshotDirectory.getAbsolutePath() + "> to <" + targetZip + ">failed. Fix the problem and redo extract.", e); System.exit(1); } //orderDAO.findAll(new Timestamp(lastRefreshTime), orderJsonFileConsumer); Configuration updateRefreshTimeConf = new Configuration(customerName, "last-refresh-time", String.valueOf(currentTime)); configurationDAO.update(updateRefreshTimeConf, String.valueOf(lastRefreshTimeConf.getValue())); ActionUtils.createMarkerFile("~/.mvdb/status.ExtractDBChanges.complete", true); }
From source file:io.anserini.index.UserPostFrequencyDistribution.java
@SuppressWarnings("static-access") public static void main(String[] args) throws Exception { Options options = new Options(); options.addOption(new Option(HELP_OPTION, "show help")); options.addOption(new Option(STORE_TERM_VECTORS_OPTION, "store term vectors")); options.addOption(OptionBuilder.withArgName("collection").hasArg() .withDescription("source collection directory").create(COLLECTION_OPTION)); options.addOption(OptionBuilder.withArgName("property").hasArg() .withDescription("source collection directory").create("property")); options.addOption(OptionBuilder.withArgName("collection_pattern").hasArg() .withDescription("source collection directory").create("collection_pattern")); CommandLine cmdline = null;/* ww w . j a v a2 s. c om*/ CommandLineParser parser = new GnuParser(); try { cmdline = parser.parse(options, args); } catch (ParseException exp) { System.err.println("Error parsing command line: " + exp.getMessage()); System.exit(-1); } if (cmdline.hasOption(HELP_OPTION) || !cmdline.hasOption(COLLECTION_OPTION)) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp(UserPostFrequencyDistribution.class.getName(), options); System.exit(-1); } String collectionPath = cmdline.getOptionValue(COLLECTION_OPTION); final FieldType textOptions = new FieldType(); textOptions.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); textOptions.setStored(true); textOptions.setTokenized(true); textOptions.setStoreTermVectors(true); LOG.info("collection: " + collectionPath); LOG.info("collection_pattern " + cmdline.getOptionValue("collection_pattern")); LOG.info("property " + cmdline.getOptionValue("property")); LongOpenHashSet deletes = null; long startTime = System.currentTimeMillis(); File file = new File(collectionPath); if (!file.exists()) { System.err.println("Error: " + file + " does not exist!"); System.exit(-1); } final JsonStatusCorpusReader stream = new JsonStatusCorpusReader(file, cmdline.getOptionValue("collection_pattern")); Runtime.getRuntime().addShutdownHook(new Thread() { public void run() { try { stream.close(); } catch (IOException e1) { // TODO Auto-generated catch block e1.printStackTrace(); } ; System.out.println("# of users indexed this round: " + userIndexedCount); System.out.println("Shutting down"); } }); Status status; boolean readerNotInitialized = true; try { Properties prop = new Properties(); while ((status = stream.next()) != null) { // try{ // status = DataObjectFactory.createStatus(s); // if (status==null||status.getText() == null) { // continue; // }}catch(Exception e){ // // } // boolean pittsburghRelated = false; try { if (Math.abs(status.getLongitude() - pittsburghLongitude) < 0.05d && Math.abs(status.getlatitude() - pittsburghLatitude) < 0.05d) pittsburghRelated = true; } catch (Exception e) { } try { if (status.getPlace().contains("Pittsburgh, PA")) pittsburghRelated = true; } catch (Exception e) { } try { if (status.getUserLocation().contains("Pittsburgh, PA")) pittsburghRelated = true; } catch (Exception e) { } try { if (status.getText().contains("Pittsburgh")) pittsburghRelated = true; } catch (Exception e) { } if (pittsburghRelated) { int previousPostCount = 0; if (prop.containsKey(String.valueOf(status.getUserid()))) { previousPostCount = Integer .valueOf(prop.getProperty(String.valueOf(status.getUserid())).split(" ")[1]); } prop.setProperty(String.valueOf(status.getUserid()), String.valueOf(status.getStatusesCount()) + " " + (1 + previousPostCount)); if (prop.size() > 0 && prop.size() % 1000 == 0) { Runtime runtime = Runtime.getRuntime(); runtime.gc(); System.out.println("Property size " + prop.size() + "Memory used: " + ((runtime.totalMemory() - runtime.freeMemory()) / (1024L * 1024L)) + " MB\n"); } OutputStream output = new FileOutputStream(cmdline.getOptionValue("property"), false); prop.store(output, null); output.close(); } } // prop.store(output, null); LOG.info(String.format("Total of %s statuses added", userIndexedCount)); LOG.info("Total elapsed time: " + (System.currentTimeMillis() - startTime) + "ms"); } catch (Exception e) { e.printStackTrace(); } finally { stream.close(); } }
From source file:org.apache.asterix.experiment.client.LSMExperimentSetRunner.java
public static void main(String[] args) throws Exception { java.util.logging.Logger.getLogger("org.apache.http.headers").setLevel(Level.FINEST); // LogManager.getRootLogger().setLevel(org.apache.log4j.Level.OFF); LSMExperimentSetRunnerConfig config = new LSMExperimentSetRunnerConfig( String.valueOf(System.currentTimeMillis()), 3); CmdLineParser clp = new CmdLineParser(config); try {//from w ww . j a v a 2s. c om clp.parseArgument(args); } catch (CmdLineException e) { System.err.println(e.getMessage()); clp.printUsage(System.err); System.exit(1); } final String pkg = "org.apache.asterix.experiment.builder.suite"; Reflections reflections = //new Reflections("org.apache.asterix.experiment.builder.suite"); new Reflections(new ConfigurationBuilder().setUrls(ClasspathHelper.forPackage(pkg)) .filterInputsBy(new FilterBuilder().includePackage(pkg)) .setScanners(new TypeElementsScanner().publicOnly(), new MethodParameterScanner())); Map<String, AbstractLSMBaseExperimentBuilder> nameMap = new TreeMap<>(); for (Constructor c : reflections.getConstructorsMatchParams(LSMExperimentSetRunnerConfig.class)) { AbstractLSMBaseExperimentBuilder b = (AbstractLSMBaseExperimentBuilder) c.newInstance(config); nameMap.put(b.getName(), b); } Pattern p = config.getRegex() == null ? null : Pattern.compile(config.getRegex()); SequentialActionList exps = new SequentialActionList(); for (Map.Entry<String, AbstractLSMBaseExperimentBuilder> e : nameMap.entrySet()) { if (p == null || p.matcher(e.getKey()).matches()) { exps.add(e.getValue().build()); if (LOGGER.isLoggable(Level.INFO)) { LOGGER.info("Added " + e.getKey() + " to run list..."); } } } exps.perform(); }
From source file:com.inkubator.common.util.NewMain.java
/** * @param args the command line arguments *///from w w w . jav a 2 s.com public static void main(String[] args) throws IOException { File file1 = new File( "C:\\Users\\deni.fahri\\AppData\\Roaming\\Skype\\My Skype Received Files\\JSON_Ek\\Surabaya\\Page1.txt"); File file2 = new File( "C:\\Users\\deni.fahri\\AppData\\Roaming\\Skype\\My Skype Received Files\\JSON_Ek\\Surabaya\\Page2.txt"); // File file3 = new File("C:\\Users\\deni.fahri\\AppData\\Roaming\\Skype\\My Skype Received Files\\json\\json\\menado\\page3.txt"); File file3 = new File( "C:\\Users\\deni.fahri\\AppData\\Roaming\\Skype\\My Skype Received Files\\JSON_Ek\\Surabaya\\Page3.txt"); File file4 = new File( "C:\\Users\\deni.fahri\\AppData\\Roaming\\Skype\\My Skype Received Files\\JSON_Ek\\Surabaya\\Page4.txt"); File file5 = new File( "C:\\Users\\deni.fahri\\AppData\\Roaming\\Skype\\My Skype Received Files\\JSON_Ek\\Surabaya\\Page5.txt"); File file6 = new File( "C:\\Users\\deni.fahri\\AppData\\Roaming\\Skype\\My Skype Received Files\\JSON_Ek\\Surabaya\\Page6.txt"); // File file7 = new File("C:\\Users\\deni.fahri\\AppData\\Roaming\\Skype\\My Skype Received Files\\Bandung\\Bandung\\Page 7.txt"); // File file8 = new File("C:\\Users\\deni.fahri\\AppData\\Roaming\\Skype\\My Skype Received Files\\Bandung\\Bandung\\Page 8.txt"); // File file9 = new File("C:\\Users\\deni.fahri\\AppData\\Roaming\\Skype\\My Skype Received Files\\Bandung\\Bandung\\Page 9.txt"); // File file10 = new File("C:\\Users\\deni.fahri\\AppData\\Roaming\\Skype\\My Skype Received Files\\Bandung\\Bandung\\Page 10.txt"); // File file11 = new File("C:\\Users\\deni.fahri\\AppData\\Roaming\\Skype\\My Skype Received Files\\Bandung\\Bandung\\Page 11.txt"); // File file12 = new File("C:\\Users\\deni.fahri\\AppData\\Roaming\\Skype\\My Skype Received Files\\Bandung\\Bandung\\Page 12.txt"); // File file13 = new File("C:\\Users\\deni.fahri\\AppData\\Roaming\\Skype\\My Skype Received Files\\Bandung\\Bandung\\Page 13.txt"); // File file14 = new File("C:\\Users\\deni.fahri\\AppData\\Roaming\\Skype\\My Skype Received Files\\Bandung\\Bandung\\Page 14.txt"); // File file15 = new File("C:\\Users\\deni.fahri\\AppData\\Roaming\\Skype\\My Skype Received Files\\Bandung\\Bandung\\Page 15.txt"); // File file16 = new File("C:\\Users\\deni.fahri\\Downloads\\page16.txt"); // File file2 = new File("C:\\Users\\deni.fahri\\Documents\\hasil.txt"); String agoda = FilesUtil.getAsStringFromFile(file1); String agoda1 = FilesUtil.getAsStringFromFile(file2); String agoda2 = FilesUtil.getAsStringFromFile(file3); String agoda3 = FilesUtil.getAsStringFromFile(file4); String agoda4 = FilesUtil.getAsStringFromFile(file5); String agoda5 = FilesUtil.getAsStringFromFile(file6); // String agoda6 = FilesUtil.getAsStringFromFile(file7); // String agoda7 = FilesUtil.getAsStringFromFile(file8); // String agoda8 = FilesUtil.getAsStringFromFile(file9); // String agoda9 = FilesUtil.getAsStringFromFile(file10); // String agoda10 = FilesUtil.getAsStringFromFile(file11); // String agoda11 = FilesUtil.getAsStringFromFile(file12); // String agoda12 = FilesUtil.getAsStringFromFile(file13); // String agoda13 = FilesUtil.getAsStringFromFile(file14); // String agoda14 = FilesUtil.getAsStringFromFile(file15); // String agoda15 = FilesUtil.getAsStringFromFile(file16); //// System.out.println(" Test Nya adalah :" + agoda); //// String a=StringUtils.substringAfter("\"HotelTranslatedName\":", agoda); //// System.out.println(" hasil; "+a); //// // TODO code application logic here //// System.out.println("Nilai " + JsonConverter.getValueByKeyStatic(agoda, "HotelTranslatedName")); TypeToken<List<HotelModel>> token = new TypeToken<List<HotelModel>>() { }; Gson gson = new GsonBuilder().create(); // List<HotelModel> data = new ArrayList<>(); // HotelModel hotelModel = new HotelModel(); // hotelModel.setAddress("sdfsdffsfsdfsdfdsfdsf"); // hotelModel.setAccommodationName("Aku"); // HotelModel hotelModel1 = new HotelModel(); // hotelModel1.setAddress("sdfsdffsfsdfsdfdsfdsf"); // hotelModel1.setAccommodationName("Avvvku"); // HotelModel hotelModel2 = new HotelModel(); // hotelModel2.setAddress("sdfsdffsfsdfsdfdsfdsf"); // hotelModel2.setAccommodationName("Akvvvu"); // data.add(hotelModel); // data.add(hotelModel1); // data.add(hotelModel2); // String json = gson.toJson(data); List<HotelModel> total = new ArrayList<>(); List<HotelModel> data1 = new ArrayList<>(); List<HotelModel> data2 = new ArrayList<>(); List<HotelModel> data3 = new ArrayList<>(); List<HotelModel> data4 = new ArrayList<>(); List<HotelModel> data5 = new ArrayList<>(); List<HotelModel> data6 = new ArrayList<>(); List<HotelModel> data7 = new ArrayList<>(); List<HotelModel> data8 = new ArrayList<>(); List<HotelModel> data9 = new ArrayList<>(); List<HotelModel> data10 = new ArrayList<>(); List<HotelModel> data11 = new ArrayList<>(); List<HotelModel> data12 = new ArrayList<>(); List<HotelModel> data13 = new ArrayList<>(); List<HotelModel> data14 = new ArrayList<>(); List<HotelModel> data15 = new ArrayList<>(); List<HotelModel> data16 = new ArrayList<>(); data1 = gson.fromJson(agoda, token.getType()); data2 = gson.fromJson(agoda1, token.getType()); data3 = gson.fromJson(agoda2, token.getType()); data4 = gson.fromJson(agoda3, token.getType()); data5 = gson.fromJson(agoda4, token.getType()); data6 = gson.fromJson(agoda5, token.getType()); // data7 = gson.fromJson(agoda6, token.getType()); // data8 = gson.fromJson(agoda7, token.getType()); // data9 = gson.fromJson(agoda8, token.getType()); // data10 = gson.fromJson(agoda9, token.getType()); // data11 = gson.fromJson(agoda10, token.getType()); // data12 = gson.fromJson(agoda11, token.getType()); // data13 = gson.fromJson(agoda12, token.getType()); // data14 = gson.fromJson(agoda13, token.getType()); // data15 = gson.fromJson(agoda14, token.getType()); // data16 = gson.fromJson(agoda15, token.getType()); total.addAll(data1); total.addAll(data2); total.addAll(data3); total.addAll(data4); total.addAll(data5); total.addAll(data6); // total.addAll(data7); // total.addAll(data8); // total.addAll(data9); // total.addAll(data10); // total.addAll(data11); // total.addAll(data12); // total.addAll(data13); // total.addAll(data14); // total.addAll(data15); // total.addAll(data16); System.out.println(" Ukurannn nya " + total.size()); // System.out.println(" Ukurannya " + data2.size()); for (HotelModel mode : total) { System.out.println(mode); } // HotelModel hotelModel = gson.fromJson(agoda, HotelModel.class); // String Data = hotelModel.getHotelTranslatedName() + ";" + hotelModel.getStarRating() + ";" + hotelModel.getAddress() + ";" + hotelModel.getIsFreeWifi(); // FilesUtil.writeToFileFromString(file2, Data); // System.out.println(hotelModel); // HSSFWorkbook workbook = new HSSFWorkbook(); HSSFSheet sheet = workbook.createSheet("Agoda Data Hotel Surabaya"); //// TreeMap<String, Object[]> datatoExel = new TreeMap<>(); int i = 1; // datatoExel.put("1", new Object[]{"Hotel Agoda Jakarta"}); datatoExel.put("1", new Object[] { "Nama Hotel", "Arena", "Alamat", "Rating", "Apakah Gratis Wifi", "Harga Mulai Dari", "Longitude", "Latitude" }); for (HotelModel mode : total) { datatoExel.put(String.valueOf(i + 1), new Object[] { mode.getHotelTranslatedName(), mode.getAreaName(), mode.getAddress(), mode.getStarRating(), mode.getIsFreeWifi(), mode.getTextPrice() + " " + mode.getCurrencyCode(), mode.getCoordinate().getLongitude(), mode.getCoordinate().getLatitude() }); i++; } // //// int i=1; //// for (HotelModel mode : data2) { //// datatoExel.put(String.valueOf(i), new Object[]{1d, "John", 1500000d}); ////// } //// //// datatoExel.put("4", new Object[]{3d, "Dean", 700000d}); //// Set<String> keyset = datatoExel.keySet(); int rownum = 0; for (String key : keyset) { Row row = sheet.createRow(rownum++); Object[] objArr = datatoExel.get(key); int cellnum = 0; for (Object obj : objArr) { Cell cell = row.createCell(cellnum++); if (obj instanceof Date) { cell.setCellValue((Date) obj); } else if (obj instanceof Boolean) { cell.setCellValue((Boolean) obj); } else if (obj instanceof String) { cell.setCellValue((String) obj); } else if (obj instanceof Double) { cell.setCellValue((Double) obj); } } } try { FileOutputStream out = new FileOutputStream(new File("C:\\Users\\deni.fahri\\Documents\\Surabaya.xls")); workbook.write(out); out.close(); System.out.println("Excel written successfully.."); } catch (FileNotFoundException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } }
From source file:com.cloudera.cdk.morphline.solr.EnvironmentTest.java
private static Version getMinorLuceneVersion(String version) { return Version.valueOf(version.replaceFirst("^(\\d)\\.(\\d).*", "LUCENE_$1$2")); }