List of usage examples for java.util List get
E get(int index);
From source file:test.TestJavaService.java
/** * Main. The cmdline arguments either do or don't contain the flag -remote: that's the only supported flag. If it is specified, * we test code on the AWS instance. If it is not, we test code running locally. * The first non-flag argument is the "verb", which (currently) should be one of the verbs set at the top of source file * org.qcert.javasrc.Main. Remaining non-flag arguments are file names. There must be at least one. The number of such * arguments and what they should contain depends on the verb. * @throws Exception//from w w w.j a v a2 s . c om */ public static void main(String[] args) throws Exception { /* Parse command line */ List<String> files = new ArrayList<>(); String loc = "localhost", verb = null; for (String arg : args) { if (arg.equals("-remote")) loc = REMOTE_LOC; else if (arg.startsWith("-")) illegal(); else if (verb == null) verb = arg; else files.add(arg); } /* Simple consistency checks and verb-specific parsing */ if (files.size() == 0) illegal(); String file = files.remove(0); String schema = null; switch (verb) { case "parseSQL": case "serialRule2CAMP": case "sqlSchema2JSON": if (files.size() != 0) illegal(); break; case "techRule2CAMP": if (files.size() != 1) illegal(); schema = files.get(0); break; case "csv2JSON": if (files.size() < 1) illegal(); break; default: illegal(); } /* Assemble information from arguments */ String url = String.format("http://%s:9879?verb=%s", loc, verb); byte[] contents = Files.readAllBytes(Paths.get(file)); String toSend; if ("serialRule2CAMP".equals(verb)) toSend = Base64.getEncoder().encodeToString(contents); else toSend = new String(contents); if ("techRule2CAMP".equals(verb)) toSend = makeSpecialJson(toSend, schema); else if ("csv2JSON".equals(verb)) toSend = makeSpecialJson(toSend, files); HttpClient client = HttpClients.createDefault(); HttpPost post = new HttpPost(url); StringEntity entity = new StringEntity(toSend); entity.setContentType("text/plain"); post.setEntity(entity); HttpResponse resp = client.execute(post); int code = resp.getStatusLine().getStatusCode(); if (code == HttpStatus.SC_OK) { HttpEntity answer = resp.getEntity(); InputStream s = answer.getContent(); BufferedReader rdr = new BufferedReader(new InputStreamReader(s)); String line = rdr.readLine(); while (line != null) { System.out.println(line); line = rdr.readLine(); } rdr.close(); s.close(); } else System.out.println(resp.getStatusLine()); }
From source file:at.illecker.hama.hybrid.examples.matrixmultiplication2.MatrixMultiplicationHybridBSP.java
public static void main(String[] args) throws Exception { // Defaults//from ww w . ja v a 2s. co m int numBspTask = 1; int numGpuBspTask = 1; int numRowsA = 4;// 1024; int numColsA = 4;// 1024; int numRowsB = 4;// 1024; int numColsB = 4;// 1024; int tileWidth = 32; // 2 * 32 = 1024 threads matches the blocksize int GPUPercentage = 100; boolean isDebugging = true; Configuration conf = new HamaConfiguration(); if (args.length > 0) { if (args.length == 9) { numBspTask = Integer.parseInt(args[0]); numGpuBspTask = Integer.parseInt(args[1]); numRowsA = Integer.parseInt(args[2]); numColsA = Integer.parseInt(args[3]); numRowsB = Integer.parseInt(args[4]); numColsB = Integer.parseInt(args[5]); tileWidth = Integer.parseInt(args[6]); GPUPercentage = Integer.parseInt(args[7]); isDebugging = Boolean.parseBoolean(args[8]); } else { System.out.println("Wrong argument size!"); System.out.println(" Argument1=numBspTask"); System.out.println(" Argument2=numGpuBspTask"); System.out.println(" Argument3=numRowsA | Number of rows of the first input matrix"); System.out.println(" Argument4=numColsA | Number of columns of the first input matrix"); System.out.println(" Argument5=numRowsB | Number of rows of the second input matrix"); System.out.println(" Argument6=numColsB | Number of columns of the second input matrix"); System.out.println(" Argument7=tileWidth | TileWidth denotes the size of a submatrix"); System.out.println(" Argument8=GPUPercentage (percentage of input)"); System.out.println(" Argument9=debug | Enable debugging (true|false)"); return; } } // Set config variables conf.setBoolean("hama.pipes.logging", false); // Set CPU tasks conf.setInt("bsp.peers.num", numBspTask); // Set GPU tasks conf.setInt("bsp.peers.gpu.num", numGpuBspTask); // Set GPU workload // conf.setInt(CONF_GPU_PERCENTAGE, GPUPercentage); LOG.info("NumBspTask: " + conf.getInt("bsp.peers.num", 0)); LOG.info("NumGpuBspTask: " + conf.getInt("bsp.peers.gpu.num", 0)); LOG.info("bsp.tasks.maximum: " + conf.get("bsp.tasks.maximum")); // LOG.info("GPUPercentage: " + conf.get(CONF_GPU_PERCENTAGE)); LOG.info("numRowsA: " + numRowsA); LOG.info("numColsA: " + numColsA); LOG.info("numRowsB: " + numRowsB); LOG.info("numColsB: " + numColsB); LOG.info("isDebugging: " + isDebugging); LOG.info("inputPath: " + CONF_INPUT_DIR); LOG.info("outputPath: " + CONF_OUTPUT_DIR); if (numColsA != numRowsB) { throw new Exception("Cols of MatrixA != rows of MatrixB! (" + numColsA + "!=" + numRowsB + ")"); } // Create random DistributedRowMatrix // use constant seeds to get reproducible results // Matrix A DistributedRowMatrix.createRandomDistributedRowMatrix(conf, numRowsA, numColsA, new Random(42L), MATRIX_A_SPLITS_PATH, false, numBspTask, numGpuBspTask, GPUPercentage); // Matrix B is stored in transposed order List<Path> transposedMatrixBPaths = DistributedRowMatrix.createRandomDistributedRowMatrix(conf, numRowsB, numColsB, new Random(1337L), MATRIX_B_TRANSPOSED_PATH, true); // Execute MatrixMultiplication BSP Job long startTime = System.currentTimeMillis(); BSPJob job = MatrixMultiplicationHybridBSP.createMatrixMultiplicationHybridBSPConf(conf, MATRIX_A_SPLITS_PATH, transposedMatrixBPaths.get(0), MATRIX_C_PATH, tileWidth, isDebugging); // Multiply Matrix DistributedRowMatrix matrixC = null; if (job.waitForCompletion(true)) { // Rename result file to output path Path matrixCOutPath = new Path(MATRIX_C_PATH + "/part0.seq"); FileSystem fs = MATRIX_C_PATH.getFileSystem(conf); FileStatus[] files = fs.listStatus(MATRIX_C_PATH); for (int i = 0; i < files.length; i++) { if ((files[i].getPath().getName().startsWith("part-")) && (files[i].getLen() > 97)) { fs.rename(files[i].getPath(), matrixCOutPath); break; } } // Read resulting Matrix from HDFS matrixC = new DistributedRowMatrix(matrixCOutPath, MATRIX_C_PATH, numRowsA, numColsB); matrixC.setConf(conf); } LOG.info("MatrixMultiplicationHybrid using Hama finished in " + (System.currentTimeMillis() - startTime) / 1000.0 + " seconds"); // Create matrix A in one file for verification List<Path> matrixAPaths = DistributedRowMatrix.createRandomDistributedRowMatrix(conf, numRowsA, numColsA, new Random(42L), MATRIX_A_PATH, false); DistributedRowMatrix matrixA = new DistributedRowMatrix(matrixAPaths.get(0), CONF_INPUT_DIR, numRowsA, numColsA); matrixA.setConf(conf); // Create matrix B, NOT transposed for verification List<Path> matrixBPaths = DistributedRowMatrix.createRandomDistributedRowMatrix(conf, numRowsB, numColsB, new Random(1337L), MATRIX_B_PATH, false); DistributedRowMatrix matrixB = new DistributedRowMatrix(matrixBPaths.get(0), CONF_INPUT_DIR, numRowsB, numColsB); matrixB.setConf(conf); // Verification DistributedRowMatrix matrixD = matrixA.multiplyJava(matrixB, MATRIX_D_PATH); if (matrixC.verify(matrixD)) { System.out.println("Verify PASSED!"); } else { System.out.println("Verify FAILED!"); } if (isDebugging) { System.out.println("\nMatrix A:"); matrixA.printDistributedRowMatrix(); System.out.println("\nMatrix B:"); matrixB.printDistributedRowMatrix(); System.out.println("\nTransposedMatrix B:"); // Load DistributedRowMatrix transposedMatrixB DistributedRowMatrix transposedMatrixB = new DistributedRowMatrix(transposedMatrixBPaths.get(0), CONF_INPUT_DIR, numColsB, numRowsB); transposedMatrixB.setConf(conf); transposedMatrixB.printDistributedRowMatrix(); System.out.println("\nMatrix C:"); matrixC.printDistributedRowMatrix(); System.out.println("\nMatrix D:"); matrixD.printDistributedRowMatrix(); // Print out log files printOutput(conf); } }
From source file:de.tudarmstadt.ukp.experiments.dip.wp1.documents.Step6HITPreparator.java
public static void main(String[] args) throws Exception { // input dir - list of xml query containers // step5-linguistic-annotation/ System.err.println("Starting step 6 HIT Preparation"); File inputDir = new File(args[0]); // output dir File outputDir = new File(args[1]); if (outputDir.exists()) { outputDir.delete();//from ww w . j a va 2s. c o m } outputDir.mkdir(); List<String> queries = new ArrayList<>(); // iterate over query containers int countClueWeb = 0; int countSentence = 0; for (File f : FileUtils.listFiles(inputDir, new String[] { "xml" }, false)) { QueryResultContainer queryResultContainer = QueryResultContainer .fromXML(FileUtils.readFileToString(f, "utf-8")); if (queries.contains(f.getName()) || queries.size() == 0) { // groups contain only non-empty documents Map<Integer, List<QueryResultContainer.SingleRankedResult>> groups = new HashMap<>(); // split to groups according to number of sentences for (QueryResultContainer.SingleRankedResult rankedResult : queryResultContainer.rankedResults) { if (rankedResult.originalXmi != null) { byte[] bytes = new BASE64Decoder() .decodeBuffer(new ByteArrayInputStream(rankedResult.originalXmi.getBytes())); JCas jCas = JCasFactory.createJCas(); XmiCasDeserializer.deserialize(new ByteArrayInputStream(bytes), jCas.getCas()); Collection<Sentence> sentences = JCasUtil.select(jCas, Sentence.class); int groupId = sentences.size() / 40; if (rankedResult.originalXmi == null) { System.err.println("Empty document: " + rankedResult.clueWebID); } else { if (!groups.containsKey(groupId)) { groups.put(groupId, new ArrayList<>()); } } //handle it groups.get(groupId).add(rankedResult); countClueWeb++; } } for (Map.Entry<Integer, List<QueryResultContainer.SingleRankedResult>> entry : groups.entrySet()) { Integer groupId = entry.getKey(); List<QueryResultContainer.SingleRankedResult> rankedResults = entry.getValue(); // make sure the results are sorted // DEBUG // for (QueryResultContainer.SingleRankedResult r : rankedResults) { // System.out.print(r.rank + "\t"); // } Collections.sort(rankedResults, (o1, o2) -> o1.rank.compareTo(o2.rank)); // iterate over results for one query and group for (int i = 0; i < rankedResults.size() && i < TOP_RESULTS_PER_GROUP; i++) { QueryResultContainer.SingleRankedResult rankedResult = rankedResults.get(i); QueryResultContainer.SingleRankedResult r = rankedResults.get(i); int rank = r.rank; MustacheFactory mf = new DefaultMustacheFactory(); Mustache mustache = mf.compile("template/template.html"); String queryId = queryResultContainer.qID; String query = queryResultContainer.query; // make the first letter uppercase query = query.substring(0, 1).toUpperCase() + query.substring(1); List<String> relevantInformationExamples = queryResultContainer.relevantInformationExamples; List<String> irrelevantInformationExamples = queryResultContainer.irrelevantInformationExamples; byte[] bytes = new BASE64Decoder() .decodeBuffer(new ByteArrayInputStream(rankedResult.originalXmi.getBytes())); JCas jCas = JCasFactory.createJCas(); XmiCasDeserializer.deserialize(new ByteArrayInputStream(bytes), jCas.getCas()); List<generators.Sentence> sentences = new ArrayList<>(); List<Integer> paragraphs = new ArrayList<>(); paragraphs.add(0); for (WebParagraph webParagraph : JCasUtil.select(jCas, WebParagraph.class)) { for (Sentence s : JCasUtil.selectCovered(Sentence.class, webParagraph)) { String sentenceBegin = String.valueOf(s.getBegin()); generators.Sentence sentence = new generators.Sentence(s.getCoveredText(), sentenceBegin); sentences.add(sentence); countSentence++; } int SentenceID = paragraphs.get(paragraphs.size() - 1); if (sentences.size() > 120) while (SentenceID < sentences.size()) { if (!paragraphs.contains(SentenceID)) paragraphs.add(SentenceID); SentenceID = SentenceID + 120; } paragraphs.add(sentences.size()); } System.err.println("Output dir: " + outputDir); int startID = 0; int endID; for (int j = 0; j < paragraphs.size(); j++) { endID = paragraphs.get(j); int sentLength = endID - startID; if (sentLength > 120 || j == paragraphs.size() - 1) { if (sentLength > 120) { endID = paragraphs.get(j - 1); j--; } sentLength = endID - startID; if (sentLength <= 40) groupId = 40; else if (sentLength <= 80 && sentLength > 40) groupId = 80; else if (sentLength > 80) groupId = 120; File folder = new File(outputDir + "/" + groupId); if (!folder.exists()) { System.err.println("creating directory: " + outputDir + "/" + groupId); boolean result = false; try { folder.mkdir(); result = true; } catch (SecurityException se) { //handle it } if (result) { System.out.println("DIR created"); } } String newHtmlFile = folder.getAbsolutePath() + "/" + f.getName() + "_" + rankedResult.clueWebID + "_" + sentLength + ".html"; System.err.println("Printing a file: " + newHtmlFile); File newHTML = new File(newHtmlFile); int t = 0; while (newHTML.exists()) { newHTML = new File(folder.getAbsolutePath() + "/" + f.getName() + "_" + rankedResult.clueWebID + "_" + sentLength + "." + t + ".html"); t++; } mustache.execute(new PrintWriter(new FileWriter(newHTML)), new generators(query, relevantInformationExamples, irrelevantInformationExamples, sentences.subList(startID, endID), queryId, rank)) .flush(); startID = endID; } } } } } } System.out.println("Printed " + countClueWeb + " documents with " + countSentence + " sentences"); }
From source file:HelloSmartsheet.java
public static void main(String[] args) { HttpURLConnection connection = null; StringBuilder response = new StringBuilder(); //We are using Jackson JSON parser to deserialize the JSON. See http://wiki.fasterxml.com/JacksonHome //Feel free to use which ever library you prefer. ObjectMapper mapper = new ObjectMapper(); mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); try {// www . j a v a2 s . c om System.out.println("STARTING HelloSmartsheet..."); //Create a BufferedReader to read user input. BufferedReader in = new BufferedReader(new InputStreamReader(System.in)); System.out.print("Enter Smartsheet API access token:"); String accessToken = in.readLine(); System.out.println("Fetching list of your sheets..."); //Create a connection and fetch the list of sheets connection = (HttpURLConnection) new URL(GET_SHEETS_URL).openConnection(); connection.addRequestProperty("Authorization", "Bearer " + accessToken); BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream())); String line; //Read the response line by line. while ((line = reader.readLine()) != null) { response.append(line); } reader.close(); //Use Jackson to conver the JSON string to a List of Sheets List<Sheet> sheets = mapper.readValue(response.toString(), new TypeReference<List<Sheet>>() { }); if (sheets.size() == 0) { System.out.println("You don't have any sheets. Goodbye!"); return; } System.out.println("Total sheets: " + sheets.size()); int i = 1; for (Sheet sheet : sheets) { System.out.println(i++ + ": " + sheet.name); } System.out.print("Enter the number of the sheet you want to share: "); //Prompt the user to provide the sheet number, the email address, and the access level Integer sheetNumber = Integer.parseInt(in.readLine().trim()); //NOTE: for simplicity, error handling and input validation is neglected. Sheet chosenSheet = sheets.get(sheetNumber - 1); System.out.print("Enter an email address to share " + chosenSheet.getName() + " to: "); String email = in.readLine(); System.out.print("Choose an access level (VIEWER, EDITOR, EDITOR_SHARE, ADMIN) for " + email + ": "); String accessLevel = in.readLine(); //Create a share object Share share = new Share(); share.setEmail(email); share.setAccessLevel(accessLevel); System.out.println("Sharing " + chosenSheet.name + " to " + email + " as " + accessLevel + "."); //Create a connection. Note the SHARE_SHEET_URL uses /sheet as opposed to /sheets (with an 's') connection = (HttpURLConnection) new URL(SHARE_SHEET_URL.replace(SHEET_ID, "" + chosenSheet.getId())) .openConnection(); connection.setDoOutput(true); connection.addRequestProperty("Authorization", "Bearer " + accessToken); connection.addRequestProperty("Content-Type", "application/json"); OutputStreamWriter writer = new OutputStreamWriter(connection.getOutputStream()); //Serialize the Share object writer.write(mapper.writeValueAsString(share)); writer.close(); //Read the response and parse the JSON reader = new BufferedReader(new InputStreamReader(connection.getInputStream())); response = new StringBuilder(); while ((line = reader.readLine()) != null) { response.append(line); } Result result = mapper.readValue(response.toString(), Result.class); System.out.println("Sheet shared successfully, share ID " + result.result.id); System.out.println("Press any key to quit."); in.read(); } catch (IOException e) { BufferedReader reader = new BufferedReader( new InputStreamReader(((HttpURLConnection) connection).getErrorStream())); String line; try { response = new StringBuilder(); while ((line = reader.readLine()) != null) { response.append(line); } reader.close(); Result result = mapper.readValue(response.toString(), Result.class); System.out.println(result.message); } catch (IOException e1) { e1.printStackTrace(); } } catch (Exception e) { System.out.println("Something broke: " + e.getMessage()); e.printStackTrace(); } }
From source file:org.ala.hbase.RepoDataLoader.java
/** * This takes a list of infosource ids... * <p/>//from w w w.ja va 2 s . c om * Usage: -stats or -reindex or -gList and list of infosourceId * * @param args */ public static void main(String[] args) throws Exception { //RepoDataLoader loader = new RepoDataLoader(); ApplicationContext context = SpringUtils.getContext(); RepoDataLoader loader = (RepoDataLoader) context.getBean(RepoDataLoader.class); long start = System.currentTimeMillis(); loader.loadInfoSources(); String filePath = repositoryDir; if (args.length > 0) { if (args[0].equalsIgnoreCase("-stats")) { loader.statsOnly = true; args = (String[]) ArrayUtils.subarray(args, 1, args.length); } if (args[0].equalsIgnoreCase("-reindex")) { loader.reindex = true; loader.indexer = context.getBean(PartialIndex.class); args = (String[]) ArrayUtils.subarray(args, 1, args.length); logger.info("**** -reindex: " + loader.reindex); logger.debug("reindex url: " + loader.reindexUrl); } if (args[0].equalsIgnoreCase("-gList")) { loader.gList = true; args = (String[]) ArrayUtils.subarray(args, 1, args.length); logger.info("**** -gList: " + loader.gList); } if (args[0].equalsIgnoreCase("-biocache")) { Hashtable<String, String> hashTable = new Hashtable<String, String>(); hashTable.put("accept", "application/json"); ObjectMapper mapper = new ObjectMapper(); mapper.getDeserializationConfig().set(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, false); RestfulClient restfulClient = new RestfulClient(0); String fq = "&fq="; if (args.length > 1) { java.util.Date date = new java.util.Date(); if (args[1].equals("-lastWeek")) { date = DateUtils.addWeeks(date, -1); } else if (args[1].equals("-lastMonth")) { date = DateUtils.addMonths(date, -1); } else if (args[1].equals("-lastYear")) { date = DateUtils.addYears(date, -1); } else date = null; if (date != null) { SimpleDateFormat sfd = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); fq += "last_load_date:%5B" + sfd.format(date) + "%20TO%20*%5D"; } } Object[] resp = restfulClient .restGet("http://biocache.ala.org.au/ws/occurrences/search?q=multimedia:Image" + fq + "&facets=data_resource_uid&pageSize=0", hashTable); logger.info("The URL: " + "http://biocache.ala.org.au/ws/occurrences/search?q=multimedia:Image" + fq + "&facets=data_resource_uid&pageSize=0"); if ((Integer) resp[0] == HttpStatus.SC_OK) { String content = resp[1].toString(); logger.debug(resp[1]); if (content != null && content.length() > "[]".length()) { Map map = mapper.readValue(content, Map.class); try { List<java.util.LinkedHashMap<String, String>> list = ((List<java.util.LinkedHashMap<String, String>>) ((java.util.LinkedHashMap) ((java.util.ArrayList) map .get("facetResults")).get(0)).get("fieldResult")); Set<String> arg = new LinkedHashSet<String>(); for (int i = 0; i < list.size(); i++) { java.util.LinkedHashMap<String, String> value = list.get(i); String dataResource = getDataResource(value.get("fq")); Object provider = (loader.getUidInfoSourceMap().get(dataResource)); if (provider != null) { arg.add(provider.toString()); } } logger.info("Set of biocache infosource ids to load: " + arg); args = new String[] {}; args = arg.toArray(args); //handle the situation where biocache-service reports no data resources if (args.length < 1) { logger.error("No biocache data resources found. Unable to load."); System.exit(0); } } catch (Exception e) { logger.error("ERROR: exit process....." + e); e.printStackTrace(); System.exit(0); } } } else { logger.warn("Unable to process url: "); } } } int filesRead = loader.load(filePath, args); //FIX ME - move to config long finish = System.currentTimeMillis(); logger.info(filesRead + " files scanned/loaded in: " + ((finish - start) / 60000) + " minutes " + ((finish - start) / 1000) + " seconds."); System.exit(1); }
From source file:com.cyberway.issue.io.Arc2Warc.java
/** * Command-line interface to Arc2Warc./*from w ww . j a v a 2s . c om*/ * * @param args Command-line arguments. * @throws ParseException Failed parse of the command line. * @throws IOException * @throws java.text.ParseException */ public static void main(String[] args) throws ParseException, IOException, java.text.ParseException { Options options = new Options(); options.addOption(new Option("h", "help", false, "Prints this message and exits.")); options.addOption(new Option("f", "force", false, "Force overwrite of target file.")); PosixParser parser = new PosixParser(); CommandLine cmdline = parser.parse(options, args, false); List cmdlineArgs = cmdline.getArgList(); Option[] cmdlineOptions = cmdline.getOptions(); HelpFormatter formatter = new HelpFormatter(); // If no args, print help. if (cmdlineArgs.size() <= 0) { usage(formatter, options, 0); } // Now look at options passed. boolean force = false; for (int i = 0; i < cmdlineOptions.length; i++) { switch (cmdlineOptions[i].getId()) { case 'h': usage(formatter, options, 0); break; case 'f': force = true; break; default: throw new RuntimeException("Unexpected option: " + +cmdlineOptions[i].getId()); } } // If no args, print help. if (cmdlineArgs.size() != 2) { usage(formatter, options, 0); } (new Arc2Warc()).transform(new File(cmdlineArgs.get(0).toString()), new File(cmdlineArgs.get(1).toString()), force); }
From source file:com.cws.esolutions.core.main.NetworkUtility.java
public static final void main(final String[] args) { final String methodName = NetworkUtility.CNAME + "#main(final String[] args)"; if (DEBUG) {/* w w w . j a va 2s .c o m*/ DEBUGGER.debug(methodName); DEBUGGER.debug("Value: {}", (Object) args); } if (args.length == 0) { new HelpFormatter().printHelp(NetworkUtility.CNAME, options, true); return; } try { CommandLineParser parser = new PosixParser(); CommandLine commandLine = parser.parse(options, args); if (DEBUG) { DEBUGGER.debug("CommandLineParser parser: {}", parser); DEBUGGER.debug("CommandLine commandLine: {}", commandLine); } String coreConfiguration = (StringUtils.isBlank(System.getProperty("coreConfigFile"))) ? NetworkUtility.CORE_SVC_CONFIG : System.getProperty("coreConfigFile"); String securityConfiguration = (StringUtils.isBlank(System.getProperty("secConfigFile"))) ? NetworkUtility.CORE_LOG_CONFIG : System.getProperty("secConfigFile"); String coreLogging = (StringUtils.isBlank(System.getProperty("coreLogConfig"))) ? NetworkUtility.SEC_SVC_CONFIG : System.getProperty("coreLogConfig"); String securityLogging = (StringUtils.isBlank(System.getProperty("secLogConfig"))) ? NetworkUtility.SEC_LOG_CONFIG : System.getProperty("secLogConfig"); if (DEBUG) { DEBUGGER.debug("String coreConfiguration: {}", coreConfiguration); DEBUGGER.debug("String securityConfiguration: {}", securityConfiguration); DEBUGGER.debug("String coreLogging: {}", coreLogging); DEBUGGER.debug("String securityLogging: {}", securityLogging); } SecurityServiceInitializer.initializeService(securityConfiguration, securityLogging, false); CoreServiceInitializer.initializeService(coreConfiguration, coreLogging, false, true); final CoreConfigurationData coreConfigData = NetworkUtility.appBean.getConfigData(); final SecurityConfigurationData secConfigData = NetworkUtility.secBean.getConfigData(); if (DEBUG) { DEBUGGER.debug("CoreConfigurationData coreConfigData: {}", coreConfigData); DEBUGGER.debug("SecurityConfigurationData secConfig: {}", secConfigData); } if (commandLine.hasOption("ssh")) { if (!(commandLine.hasOption("hostname")) || (!(StringUtils.isBlank(commandLine.getOptionValue("hostname"))))) { throw new CoreServiceException("No target host was provided. Unable to process request."); } else if (!(commandLine.hasOption("username")) && (StringUtils.isBlank(commandLine.getOptionValue("username")))) { throw new CoreServiceException("No username was provided. Using default."); } // NetworkUtils.executeSshConnection(commandLine.getOptionValue("hostname"), commandList); } else if (commandLine.hasOption("http")) { if (!(commandLine.hasOption("hostname")) || (!(StringUtils.isBlank(commandLine.getOptionValue("hostname"))))) { throw new CoreServiceException("No target host was provided. Unable to process request."); } else if (!(commandLine.hasOption("method")) && (StringUtils.isBlank(commandLine.getOptionValue("method")))) { throw new CoreServiceException("No HTTP method was provided. Unable to process request."); } NetworkUtils.executeHttpConnection(new URL(commandLine.getOptionValue("hostname")), commandLine.getOptionValue("method")); } else if (commandLine.hasOption("copyFiles")) { if (!(commandLine.hasOption("hostname")) || (StringUtils.isBlank(commandLine.getOptionValue("hostname")))) { throw new CoreServiceException("No target host was provided. Unable to process request."); } else if (!(commandLine.hasOption("sourceFile")) && (StringUtils.isBlank(commandLine.getOptionValue("sourceFile")))) { throw new CoreServiceException("No source file(s) were provided. Unable to process request."); } else if (!(commandLine.hasOption("targetFile")) && (StringUtils.isBlank(commandLine.getOptionValue("targetFile")))) { throw new CoreServiceException("No target file(s) were provided. Unable to process request."); } List<String> filesToTransfer = new ArrayList<String>( Arrays.asList(commandLine.getOptionValues("sourceFile"))); List<String> filesToCreate = new ArrayList<String>( Arrays.asList(commandLine.getOptionValues("targetFile"))); if (DEBUG) { DEBUGGER.debug("List<String> filesToTransfer: {}", filesToTransfer); DEBUGGER.debug("List<String> filesToCreate: {}", filesToCreate); } if (commandLine.hasOption("withSSH")) { for (String fileName : filesToTransfer) { if (DEBUG) { DEBUGGER.debug("String fileName: {}", fileName); } NetworkUtils.executeSftpTransfer(fileName, filesToCreate.get(filesToTransfer.indexOf(fileName)), commandLine.getOptionValue("hostname"), FileUtils.getFile(fileName).exists()); } } else if (commandLine.hasOption("withFTP")) { if ((commandLine.hasOption("withSSL")) && (Boolean.valueOf(commandLine.getOptionValue("withSSL")))) { // ftp/s } // NetworkUtils.executeFtpConnection(sourceFile, targetFile, targetHost, isUpload); } } } catch (ParseException px) { ERROR_RECORDER.error(px.getMessage(), px); System.err.println("An error occurred during processing: " + px.getMessage()); } catch (SecurityException sx) { ERROR_RECORDER.error(sx.getMessage(), sx); System.err.println("An error occurred during processing: " + sx.getMessage()); } catch (SecurityServiceException ssx) { ERROR_RECORDER.error(ssx.getMessage(), ssx); System.err.println("An error occurred during processing: " + ssx.getMessage()); } catch (CoreServiceException csx) { ERROR_RECORDER.error(csx.getMessage(), csx); System.err.println("An error occurred during processing: " + csx.getMessage()); } catch (MalformedURLException mux) { ERROR_RECORDER.error(mux.getMessage(), mux); System.err.println("An error occurred during processing: " + mux.getMessage()); } }
From source file:com.alexoree.jenkins.Main.java
public static void main(String[] args) throws Exception { // create Options object Options options = new Options(); options.addOption("t", false, "throttle the downloads, waits 5 seconds in between each d/l"); // automatically generate the help statement HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("jenkins-sync", options); CommandLineParser parser = new DefaultParser(); CommandLine cmd = parser.parse(options, args); boolean throttle = cmd.hasOption("t"); String plugins = "https://updates.jenkins-ci.org/latest/"; List<String> ps = new ArrayList<String>(); Document doc = Jsoup.connect(plugins).get(); for (Element file : doc.select("td a")) { //System.out.println(file.attr("href")); if (file.attr("href").endsWith(".hpi") || file.attr("href").endsWith(".war")) { ps.add(file.attr("href")); }//w w w . jav a 2 s .c om } File root = new File("."); //https://updates.jenkins-ci.org/latest/AdaptivePlugin.hpi new File("./latest").mkdirs(); //output zip file String zipFile = "jenkinsSync.zip"; // create byte buffer byte[] buffer = new byte[1024]; FileOutputStream fos = new FileOutputStream(zipFile); ZipOutputStream zos = new ZipOutputStream(fos); //download the plugins for (int i = 0; i < ps.size(); i++) { System.out.println("[" + i + "/" + ps.size() + "] downloading " + plugins + ps.get(i)); String outputFile = download(root.getAbsolutePath() + "/latest/" + ps.get(i), plugins + ps.get(i)); FileInputStream fis = new FileInputStream(outputFile); // begin writing a new ZIP entry, positions the stream to the start of the entry data zos.putNextEntry(new ZipEntry(outputFile.replace(root.getAbsolutePath(), "") .replace("updates.jenkins-ci.org/", "").replace("https:/", ""))); int length; while ((length = fis.read(buffer)) > 0) { zos.write(buffer, 0, length); } zos.closeEntry(); fis.close(); if (throttle) Thread.sleep(WAIT); new File(root.getAbsolutePath() + "/latest/" + ps.get(i)).deleteOnExit(); } //download the json metadata plugins = "https://updates.jenkins-ci.org/"; ps = new ArrayList<String>(); doc = Jsoup.connect(plugins).get(); for (Element file : doc.select("td a")) { //System.out.println(file.attr("href")); if (file.attr("href").endsWith(".json")) { ps.add(file.attr("href")); } } for (int i = 0; i < ps.size(); i++) { download(root.getAbsolutePath() + "/" + ps.get(i), plugins + ps.get(i)); FileInputStream fis = new FileInputStream(root.getAbsolutePath() + "/" + ps.get(i)); // begin writing a new ZIP entry, positions the stream to the start of the entry data zos.putNextEntry(new ZipEntry(plugins + ps.get(i))); int length; while ((length = fis.read(buffer)) > 0) { zos.write(buffer, 0, length); } zos.closeEntry(); fis.close(); new File(root.getAbsolutePath() + "/" + ps.get(i)).deleteOnExit(); if (throttle) Thread.sleep(WAIT); } // close the ZipOutputStream zos.close(); }
From source file:com.github.horrorho.inflatabledonkey.Main.java
/** * @param args the command line arguments * @throws IOException//w w w . j a va 2 s . c o m */ public static void main(String[] args) throws IOException { try { if (!PropertyLoader.instance().test(args)) { return; } } catch (IllegalArgumentException ex) { System.out.println("Argument error: " + ex.getMessage()); System.out.println("Try '" + Property.APP_NAME.value() + " --help' for more information."); System.exit(-1); } // SystemDefault HttpClient. // TODO concurrent CloseableHttpClient httpClient = HttpClients.custom().setUserAgent("CloudKit/479 (13A404)") .useSystemProperties().build(); // Auth // TODO rework when we have UncheckedIOException for Authenticator Auth auth = Property.AUTHENTICATION_TOKEN.value().map(Auth::new).orElse(null); if (auth == null) { auth = Authenticator.authenticate(httpClient, Property.AUTHENTICATION_APPLEID.value().get(), Property.AUTHENTICATION_PASSWORD.value().get()); } logger.debug("-- main() - auth: {}", auth); logger.info("-- main() - dsPrsID:mmeAuthToken: {}:{}", auth.dsPrsID(), auth.mmeAuthToken()); if (Property.ARGS_TOKEN.booleanValue().orElse(false)) { System.out.println("DsPrsID:mmeAuthToken " + auth.dsPrsID() + ":" + auth.mmeAuthToken()); return; } logger.info("-- main() - Apple ID: {}", Property.AUTHENTICATION_APPLEID.value()); logger.info("-- main() - password: {}", Property.AUTHENTICATION_PASSWORD.value()); logger.info("-- main() - token: {}", Property.AUTHENTICATION_TOKEN.value()); // Account Account account = Accounts.account(httpClient, auth); // Backup Backup backup = Backup.create(httpClient, account); // BackupAccount BackupAccount backupAccount = backup.backupAccount(httpClient); logger.debug("-- main() - backup account: {}", backupAccount); // Devices List<Device> devices = backup.devices(httpClient, backupAccount.devices()); logger.debug("-- main() - device count: {}", devices.size()); // Snapshots List<SnapshotID> snapshotIDs = devices.stream().map(Device::snapshots).flatMap(Collection::stream) .collect(Collectors.toList()); logger.info("-- main() - total snapshot count: {}", snapshotIDs.size()); Map<String, Snapshot> snapshots = backup.snapshot(httpClient, snapshotIDs).stream().collect( Collectors.toMap(s -> s.record().getRecordIdentifier().getValue().getName(), Function.identity())); boolean repeat = false; do { for (int i = 0; i < devices.size(); i++) { Device device = devices.get(i); List<SnapshotID> deviceSnapshotIDs = device.snapshots(); System.out.println(i + " " + device.info()); for (int j = 0; j < deviceSnapshotIDs.size(); j++) { SnapshotID sid = deviceSnapshotIDs.get(j); System.out.println("\t" + j + snapshots.get(sid.id()).info() + " " + sid.timestamp()); } } if (Property.PRINT_SNAPSHOTS.booleanValue().orElse(false)) { return; } // Selection Scanner input = new Scanner(System.in); int deviceIndex; int snapshotIndex = Property.SELECT_SNAPSHOT_INDEX.intValue().get(); if (devices.size() > 1) { System.out.printf("Select a device [0 - %d]: ", devices.size() - 1); deviceIndex = input.nextInt(); } else deviceIndex = Property.SELECT_DEVICE_INDEX.intValue().get(); if (deviceIndex >= devices.size() || deviceIndex < 0) { System.out.println("No such device: " + deviceIndex); System.exit(-1); } Device device = devices.get(deviceIndex); System.out.println("Selected device: " + deviceIndex + ", " + device.info()); if (device.snapshots().size() > 1) { System.out.printf("Select a snapshot [0 - %d]: ", device.snapshots().size() - 1); snapshotIndex = input.nextInt(); } else snapshotIndex = Property.SELECT_SNAPSHOT_INDEX.intValue().get(); if (snapshotIndex >= devices.get(deviceIndex).snapshots().size() || snapshotIndex < 0) { System.out.println("No such snapshot for selected device: " + snapshotIndex); System.exit(-1); } logger.info("-- main() - arg device index: {}", deviceIndex); logger.info("-- main() - arg snapshot index: {}", snapshotIndex); String selected = devices.get(deviceIndex).snapshots().get(snapshotIndex).id(); Snapshot snapshot = snapshots.get(selected); System.out.println("Selected snapshot: " + snapshotIndex + ", " + snapshot.info()); // Asset list. List<Assets> assetsList = backup.assetsList(httpClient, snapshot); logger.info("-- main() - assets count: {}", assetsList.size()); // Domains filter --domain option String chosenDomain = Property.FILTER_DOMAIN.value().orElse("").toLowerCase(Locale.US); logger.info("-- main() - arg domain substring filter: {}", Property.FILTER_DOMAIN.value()); // Output domains --domains option if (Property.PRINT_DOMAIN_LIST.booleanValue().orElse(false)) { System.out.println("Domains / file count:"); assetsList.stream().filter(a -> a.domain().isPresent()) .map(a -> a.domain().get() + " / " + a.files().size()).sorted() .forEach(System.out::println); System.out.print("Type a domain ('null' to exit): "); chosenDomain = input.next().toLowerCase(Locale.US); if (chosenDomain.equals("null")) return; // TODO check Assets without domain information. } String domainSubstring = chosenDomain; Predicate<Optional<String>> domainFilter = domain -> domain.map(d -> d.toLowerCase(Locale.US)) .map(d -> d.contains(domainSubstring)).orElse(false); List<String> files = Assets.files(assetsList, domainFilter); logger.info("-- main() - domain filtered file count: {}", files.size()); // Output folders. Path outputFolder = Paths.get(Property.OUTPUT_FOLDER.value().orElse("output")); Path assetOutputFolder = outputFolder.resolve("assets"); // TODO assets value injection Path chunkOutputFolder = outputFolder.resolve("chunks"); // TODO chunks value injection logger.info("-- main() - output folder chunks: {}", chunkOutputFolder); logger.info("-- main() - output folder assets: {}", assetOutputFolder); // Download tools. AuthorizeAssets authorizeAssets = AuthorizeAssets.backupd(); DiskChunkStore chunkStore = new DiskChunkStore(chunkOutputFolder); StandardChunkEngine chunkEngine = new StandardChunkEngine(chunkStore); AssetDownloader assetDownloader = new AssetDownloader(chunkEngine); KeyBagManager keyBagManager = backup.newKeyBagManager(); // Mystery Moo. Moo moo = new Moo(authorizeAssets, assetDownloader, keyBagManager); // Filename extension filter. String filenameExtension = Property.FILTER_EXTENSION.value().orElse("").toLowerCase(Locale.US); logger.info("-- main() - arg filename extension filter: {}", Property.FILTER_EXTENSION.value()); Predicate<Asset> assetFilter = asset -> asset.relativePath().map(d -> d.toLowerCase(Locale.US)) .map(d -> d.endsWith(filenameExtension)).orElse(false); // Batch process files in groups of 100. // TODO group files into batches based on file size. List<List<String>> batches = ListUtils.partition(files, 100); for (List<String> batch : batches) { List<Asset> assets = backup.assets(httpClient, batch).stream().filter(assetFilter::test) .collect(Collectors.toList()); logger.info("-- main() - filtered asset count: {}", assets.size()); moo.download(httpClient, assets, assetOutputFolder); } System.out.print("Download other snapshot (Y/N)? "); repeat = input.next().toLowerCase(Locale.US).charAt(0) == 'y'; } while (repeat == true); }
From source file:com.curecomp.primefaces.migrator.PrimefacesMigration.java
public static void main(String[] args) throws Exception { // Let's use some colors :) // AnsiConsole.systemInstall(); CommandLineParser cliParser = new BasicParser(); CommandLine cli = null;/*from w w w . ja va 2 s. c o m*/ try { cli = cliParser.parse(OPTIONS, args); } catch (ParseException e) { printHelp(); } if (!cli.hasOption("s")) { printHelp(); } String sourcePattern; if (cli.hasOption("p")) { sourcePattern = cli.getOptionValue("p"); } else { sourcePattern = DEFAULT_SOURCE_PATTERN; } String defaultAnswer; if (cli.hasOption("default-answer")) { defaultAnswer = cli.getOptionValue("default-answer"); } else { defaultAnswer = DEFAULT_DEFAULT_PROMPT_ANSWER; } boolean defaultAnswerYes = defaultAnswer.equalsIgnoreCase("y"); boolean quiet = cli.hasOption("q"); boolean testWrite = cli.hasOption("t"); Path sourceDirectory = Paths.get(cli.getOptionValue("s")).toAbsolutePath(); // Since we use IO we will have some blocking threads hanging around int threadCount = Runtime.getRuntime().availableProcessors() * 2; ThreadPoolExecutor threadPool = new ThreadPoolExecutor(threadCount, threadCount, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>()); BlockingQueue<WidgetVarLocation> foundUsages = new LinkedBlockingQueue<>(); BlockingQueue<WidgetVarLocation> unusedOrAmbiguous = new LinkedBlockingQueue<>(); BlockingQueue<WidgetVarLocation> skippedUsages = new LinkedBlockingQueue<>(); List<Future<?>> futures = new ArrayList<>(); findWidgetVars(sourceDirectory, sourcePattern, threadPool).forEach(widgetVarLocation -> { // We can't really find usages of widget vars that use EL expressions :( if (widgetVarLocation.widgetVar.contains("#")) { unusedOrAmbiguous.add(widgetVarLocation); return; } try { FileActionVisitor visitor = new FileActionVisitor(sourceDirectory, sourcePattern, sourceFile -> futures.add(threadPool.submit((Callable<?>) () -> { findWidgetVarUsages(sourceFile, widgetVarLocation, foundUsages, skippedUsages, unusedOrAmbiguous); return null; }))); Files.walkFileTree(sourceDirectory, visitor); } catch (IOException ex) { throw new RuntimeException(ex); } }); awaitAll(futures); new TreeSet<>(skippedUsages).forEach(widgetUsage -> { int startIndex = widgetUsage.columnNr; int endIndex = startIndex + widgetUsage.widgetVar.length(); String relativePath = widgetUsage.location.toAbsolutePath().toString() .substring(sourceDirectory.toString().length()); String previous = replace(widgetUsage.line, startIndex, endIndex, Ansi.ansi().bold().fg(Ansi.Color.RED).a(widgetUsage.widgetVar).reset().toString()); System.out.println("Skipped " + relativePath + " at line " + widgetUsage.lineNr + " and col " + widgetUsage.columnNr + " for widgetVar '" + widgetUsage.widgetVar + "'"); System.out.println("\t" + previous); }); Map<WidgetVarLocation, List<WidgetVarLocation>> written = new HashMap<>(); new TreeSet<>(foundUsages).forEach(widgetUsage -> { WidgetVarLocation key = new WidgetVarLocation(null, widgetUsage.location, widgetUsage.lineNr, -1, null); List<WidgetVarLocation> writtenList = written.get(key); int existing = writtenList == null ? 0 : writtenList.size(); int startIndex = widgetUsage.columnNr; int endIndex = startIndex + widgetUsage.widgetVar.length(); String relativePath = widgetUsage.location.toAbsolutePath().toString() .substring(sourceDirectory.toString().length()); String next = replace(widgetUsage.line, startIndex, endIndex, Ansi.ansi().bold().fg(Ansi.Color.RED) .a("PF('" + widgetUsage.widgetVar + "')").reset().toString()); System.out .println(relativePath + " at line " + widgetUsage.lineNr + " and col " + widgetUsage.columnNr); System.out.println("\t" + next); System.out.print("Replace (Y/N)? [" + (defaultAnswerYes ? "Y" : "N") + "]: "); String input; if (quiet) { input = ""; System.out.println(); } else { try { do { input = in.readLine(); } while (input != null && !input.isEmpty() && !"y".equalsIgnoreCase(input) && !"n".equalsIgnoreCase(input)); } catch (IOException ex) { throw new RuntimeException(ex); } } if (input == null) { System.out.println("Aborted!"); } else if (input.isEmpty() && defaultAnswerYes || !input.isEmpty() && !"n".equalsIgnoreCase(input)) { System.out.println("Replaced!"); System.out.print("\t"); if (writtenList == null) { writtenList = new ArrayList<>(); written.put(key, writtenList); } writtenList.add(widgetUsage); List<String> lines; try { lines = Files.readAllLines(widgetUsage.location); } catch (IOException ex) { throw new RuntimeException(ex); } try (OutputStream os = testWrite ? new ByteArrayOutputStream() : Files.newOutputStream(widgetUsage.location); PrintWriter pw = new PrintWriter(new OutputStreamWriter(os, StandardCharsets.UTF_8))) { String line; for (int i = 0; i < lines.size(); i++) { int lineNr = i + 1; line = lines.get(i); if (lineNr == widgetUsage.lineNr) { int begin = widgetUsage.columnNr + (testWrite ? 0 : existing * 6); int end = begin + widgetUsage.widgetVar.length(); String newLine = replace(line, begin, end, "PF('" + widgetUsage.widgetVar + "')", false); if (testWrite) { System.out.println(newLine); } else { pw.println(newLine); } } else { if (!testWrite) { pw.println(line); } } } } catch (IOException ex) { throw new RuntimeException(ex); } } else { System.out.println("Skipped!"); } }); new TreeSet<>(unusedOrAmbiguous).forEach(widgetUsage -> { int startIndex = widgetUsage.columnNr; int endIndex = startIndex + widgetUsage.widgetVar.length(); String relativePath = widgetUsage.location.toAbsolutePath().toString() .substring(sourceDirectory.toString().length()); String previous = replace(widgetUsage.line, startIndex, endIndex, Ansi.ansi().bold().fg(Ansi.Color.RED).a(widgetUsage.widgetVar).reset().toString()); System.out.println("Skipped unused or ambiguous " + relativePath + " at line " + widgetUsage.lineNr + " and col " + widgetUsage.columnNr); System.out.println("\t" + previous); }); threadPool.shutdown(); }