List of usage examples for java.util ArrayList add
public boolean add(E e)
From source file:ValidateLicenseHeaders.java
/** * ValidateLicenseHeaders jboss-src-root * /*w w w . j a v a 2 s.c o m*/ * @param args */ public static void main(String[] args) throws Exception { if (args.length == 0 || args[0].startsWith("-h")) { log.info("Usage: ValidateLicenseHeaders [-addheader] jboss-src-root"); System.exit(1); } int rootArg = 0; if (args.length == 2) { if (args[0].startsWith("-add")) addDefaultHeader = true; else { log.severe("Uknown argument: " + args[0]); log.info("Usage: ValidateLicenseHeaders [-addheader] jboss-src-root"); System.exit(1); } rootArg = 1; } File jbossSrcRoot = new File(args[rootArg]); if (jbossSrcRoot.exists() == false) { log.info("Src root does not exist, check " + jbossSrcRoot.getAbsolutePath()); System.exit(1); } URL u = Thread.currentThread().getContextClassLoader() .getResource("META-INF/services/javax.xml.parsers.DocumentBuilderFactory"); System.err.println(u); // Load the valid copyright statements for the licenses File licenseInfo = new File(jbossSrcRoot, "varia/src/etc/license-info.xml"); if (licenseInfo.exists() == false) { log.severe("Failed to find the varia/src/etc/license-info.xml under the src root"); System.exit(1); } DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); DocumentBuilder db = factory.newDocumentBuilder(); Document doc = db.parse(licenseInfo); NodeList licenses = doc.getElementsByTagName("license"); for (int i = 0; i < licenses.getLength(); i++) { Element license = (Element) licenses.item(i); String key = license.getAttribute("id"); ArrayList headers = new ArrayList(); licenseHeaders.put(key, headers); NodeList copyrights = license.getElementsByTagName("terms-header"); for (int j = 0; j < copyrights.getLength(); j++) { Element copyright = (Element) copyrights.item(j); copyright.normalize(); String id = copyright.getAttribute("id"); // The id will be blank if there is no id attribute if (id.length() == 0) continue; String text = getElementContent(copyright); if (text == null) continue; // Replace all duplicate whitespace and '*' with a single space text = text.replaceAll("[\\s*]+", " "); if (text.length() == 1) continue; text = text.toLowerCase().trim(); // Replace any copyright date0-date1,date2 with copyright ... text = text.replaceAll(COPYRIGHT_REGEX, "..."); LicenseHeader lh = new LicenseHeader(id, text); headers.add(lh); } } log.fine(licenseHeaders.toString()); File[] files = jbossSrcRoot.listFiles(dotJavaFilter); log.info("Root files count: " + files.length); processSourceFiles(files, 0); log.info("Processed " + totalCount); log.info("Updated jboss headers: " + jbossCount); // Files with no headers details log.info("Files with no headers: " + noheaders.size()); FileWriter fw = new FileWriter("NoHeaders.txt"); for (Iterator iter = noheaders.iterator(); iter.hasNext();) { File f = (File) iter.next(); fw.write(f.getAbsolutePath()); fw.write('\n'); } fw.close(); // Files with unknown headers details log.info("Files with invalid headers: " + invalidheaders.size()); fw = new FileWriter("InvalidHeaders.txt"); for (Iterator iter = invalidheaders.iterator(); iter.hasNext();) { File f = (File) iter.next(); fw.write(f.getAbsolutePath()); fw.write('\n'); } fw.close(); // License usage summary log.info("Creating HeadersSummary.txt"); fw = new FileWriter("HeadersSummary.txt"); for (Iterator iter = licenseHeaders.entrySet().iterator(); iter.hasNext();) { Map.Entry entry = (Map.Entry) iter.next(); String key = (String) entry.getKey(); fw.write("+++ License type=" + key); fw.write('\n'); List list = (List) entry.getValue(); Iterator jiter = list.iterator(); while (jiter.hasNext()) { LicenseHeader lh = (LicenseHeader) jiter.next(); fw.write('\t'); fw.write(lh.id); fw.write(", count="); fw.write("" + lh.count); fw.write('\n'); } } fw.close(); }
From source file:de.tudarmstadt.ukp.experiments.dip.wp1.documents.Step8GoldDataAggregator.java
public static void main(String[] args) throws Exception { String inputDir = args[0] + "/"; // output dir File outputDir = new File(args[1]); File turkersConfidence = new File(args[2]); if (outputDir.exists()) { outputDir.delete();/*from ww w . ja va2 s. co m*/ } outputDir.mkdir(); List<String> annotatorsIDs = new ArrayList<>(); // for (File f : FileUtils.listFiles(new File(inputDir), new String[] { "xml" }, false)) { // QueryResultContainer queryResultContainer = QueryResultContainer // .fromXML(FileUtils.readFileToString(f, "utf-8")); // for (QueryResultContainer.SingleRankedResult rankedResults : queryResultContainer.rankedResults) { // for (QueryResultContainer.MTurkRelevanceVote relevanceVote : rankedResults.mTurkRelevanceVotes) { // if (!annotatorsIDs.contains(relevanceVote.turkID)) // annotatorsIDs.add(relevanceVote.turkID); // } // } // } HashMap<String, Integer> countVotesForATurker = new HashMap<>(); // creates temporary file with format for mace // Hashmap annotations: key is the id of a document and a sentence // Value is an array votes[] of turkers decisions: true or false (relevant or not) // the length of this array equals the number of annotators in List<String> annotatorsIDs. // If an annotator worked on the task his decision is written in the array otherwise the value is NULL // key: queryID + clueWebID + sentenceID // value: true and false annotations TreeMap<String, Annotations> annotations = new TreeMap<>(); for (File f : FileUtils.listFiles(new File(inputDir), new String[] { "xml" }, false)) { QueryResultContainer queryResultContainer = QueryResultContainer .fromXML(FileUtils.readFileToString(f, "utf-8")); System.out.println("Reading " + f.getName()); for (QueryResultContainer.SingleRankedResult rankedResults : queryResultContainer.rankedResults) { String documentID = rankedResults.clueWebID; for (QueryResultContainer.MTurkRelevanceVote relevanceVote : rankedResults.mTurkRelevanceVotes) { Integer turkerID; if (!annotatorsIDs.contains(relevanceVote.turkID)) { annotatorsIDs.add(relevanceVote.turkID); turkerID = annotatorsIDs.size() - 1; } else { turkerID = annotatorsIDs.indexOf(relevanceVote.turkID); } Integer count = countVotesForATurker.get(relevanceVote.turkID); if (count == null) { count = 0; } count++; countVotesForATurker.put(relevanceVote.turkID, count); String id; List<Integer> trueVotes; List<Integer> falseVotes; for (QueryResultContainer.SingleSentenceRelevanceVote singleSentenceRelevanceVote : relevanceVote.singleSentenceRelevanceVotes) if (!"".equals(singleSentenceRelevanceVote.sentenceID)) { id = f.getName() + "_" + documentID + "_" + singleSentenceRelevanceVote.sentenceID; Annotations turkerVotes = annotations.get(id); if (turkerVotes == null) { trueVotes = new ArrayList<>(); falseVotes = new ArrayList<>(); turkerVotes = new Annotations(trueVotes, falseVotes); } trueVotes = turkerVotes.trueAnnotations; falseVotes = turkerVotes.falseAnnotations; if ("true".equals(singleSentenceRelevanceVote.relevant)) { // votes[turkerID] = true; trueVotes.add(turkerID); } else if ("false".equals(singleSentenceRelevanceVote.relevant)) { // votes[turkerID] = false; falseVotes.add(turkerID); } else { throw new IllegalStateException("Annotation value of sentence " + singleSentenceRelevanceVote.sentenceID + " in " + rankedResults.clueWebID + " equals " + singleSentenceRelevanceVote.relevant); } try { int allVotesCount = trueVotes.size() + falseVotes.size(); if (allVotesCount > 5) { System.err.println(id + " doesn't have 5 annotators: true: " + trueVotes.size() + " false: " + falseVotes.size()); // nasty hack, we're gonna strip some data; true votes first /* we can't do that, it breaks something down the line int toRemove = allVotesCount - 5; if (trueVotes.size() >= toRemove) { trueVotes = trueVotes .subList(0, trueVotes.size() - toRemove); } else if ( falseVotes.size() >= toRemove) { falseVotes = falseVotes .subList(0, trueVotes.size() - toRemove); } */ System.err.println("Adjusted: " + id + " doesn't have 5 annotators: true: " + trueVotes.size() + " false: " + falseVotes.size()); } } catch (IllegalStateException e) { e.printStackTrace(); } turkerVotes.trueAnnotations = trueVotes; turkerVotes.falseAnnotations = falseVotes; annotations.put(id, turkerVotes); } else { throw new IllegalStateException( "Empty Sentence ID in " + f.getName() + " for turker " + turkerID); } } } } File tmp = printHashMap(annotations, annotatorsIDs.size()); String file = TEMP_DIR + "/" + tmp.getName(); MACE.main(new String[] { "--prefix", file }); //gets the keys of the documents and sentences ArrayList<String> lines = (ArrayList<String>) FileUtils.readLines(new File(file + ".prediction")); int i = 0; TreeMap<String, TreeMap<String, ArrayList<HashMap<String, String>>>> ids = new TreeMap<>(); ArrayList<HashMap<String, String>> sentences; if (lines.size() != annotations.size()) { throw new IllegalStateException( "The size of prediction file is " + lines.size() + "but expected " + annotations.size()); } for (Map.Entry entry : annotations.entrySet()) { //1001.xml_clueweb12-1905wb-13-07360_8783 String key = (String) entry.getKey(); String[] IDs = key.split("_"); if (IDs.length > 2) { String queryID = IDs[0]; String clueWebID = IDs[1]; String sentenceID = IDs[2]; TreeMap<String, ArrayList<HashMap<String, String>>> clueWebIDs = ids.get(queryID); if (clueWebIDs == null) { clueWebIDs = new TreeMap<>(); } sentences = clueWebIDs.get(clueWebID); if (sentences == null) { sentences = new ArrayList<>(); } HashMap<String, String> sentence = new HashMap<>(); sentence.put(sentenceID, lines.get(i)); sentences.add(sentence); clueWebIDs.put(clueWebID, sentences); ids.put(queryID, clueWebIDs); } else { throw new IllegalStateException("Wrong ID " + key); } i++; } for (Map.Entry entry : ids.entrySet()) { TreeMap<Integer, String> value = (TreeMap<Integer, String>) entry.getValue(); String queryID = (String) entry.getKey(); QueryResultContainer queryResultContainer = QueryResultContainer .fromXML(FileUtils.readFileToString(new File(inputDir, queryID), "utf-8")); for (QueryResultContainer.SingleRankedResult rankedResults : queryResultContainer.rankedResults) { for (Map.Entry val : value.entrySet()) { String clueWebID = (String) val.getKey(); if (clueWebID.equals(rankedResults.clueWebID)) { List<QueryResultContainer.SingleSentenceRelevanceVote> goldEstimatedLabels = new ArrayList<>(); List<QueryResultContainer.SingleSentenceRelevanceVote> turkersVotes = new ArrayList<>(); int size = 0; int hitSize = 0; String hitID = ""; for (QueryResultContainer.MTurkRelevanceVote vote : rankedResults.mTurkRelevanceVotes) { if (!hitID.equals(vote.hitID)) { hitID = vote.hitID; hitSize = vote.singleSentenceRelevanceVotes.size(); size = size + hitSize; turkersVotes.addAll(vote.singleSentenceRelevanceVotes); } else { if (vote.singleSentenceRelevanceVotes.size() != hitSize) { hitSize = vote.singleSentenceRelevanceVotes.size(); size = size + hitSize; turkersVotes.addAll(vote.singleSentenceRelevanceVotes); } } } ArrayList<HashMap<String, String>> sentenceList = (ArrayList<HashMap<String, String>>) val .getValue(); if (sentenceList.size() != turkersVotes.size()) { try { throw new IllegalStateException("Expected size of annotations is " + turkersVotes.size() + "but found " + sentenceList.size() + " for document " + rankedResults.clueWebID + " in " + queryID); } catch (IllegalStateException ex) { ex.printStackTrace(); } } for (QueryResultContainer.SingleSentenceRelevanceVote s : turkersVotes) { String valSentence = null; for (HashMap<String, String> anno : sentenceList) { if (anno.keySet().contains(s.sentenceID)) { valSentence = anno.get(s.sentenceID); } } QueryResultContainer.SingleSentenceRelevanceVote singleSentenceVote = new QueryResultContainer.SingleSentenceRelevanceVote(); singleSentenceVote.sentenceID = s.sentenceID; if (("false").equals(valSentence)) { singleSentenceVote.relevant = "false"; } else if (("true").equals(valSentence)) { singleSentenceVote.relevant = "true"; } else { throw new IllegalStateException("Annotation value of sentence " + singleSentenceVote.sentenceID + " equals " + val.getValue()); } goldEstimatedLabels.add(singleSentenceVote); } rankedResults.goldEstimatedLabels = goldEstimatedLabels; } } } File outputFile = new File(outputDir, queryID); FileUtils.writeStringToFile(outputFile, queryResultContainer.toXML(), "utf-8"); System.out.println("Finished " + outputFile); } ArrayList<String> annotators = (ArrayList<String>) FileUtils.readLines(new File(file + ".competence")); FileWriter fileWriter; StringBuilder sb = new StringBuilder(); for (int j = 0; j < annotatorsIDs.size(); j++) { String[] s = annotators.get(0).split("\t"); Float score = Float.parseFloat(s[j]); String turkerID = annotatorsIDs.get(j); System.out.println(turkerID + " " + score + " " + countVotesForATurker.get(turkerID)); sb.append(turkerID).append(" ").append(score).append(" ").append(countVotesForATurker.get(turkerID)) .append("\n"); } fileWriter = new FileWriter(turkersConfidence); fileWriter.append(sb.toString()); fileWriter.close(); }
From source file:de.prozesskraft.ptest.Launch.java
public static void main(String[] args) throws org.apache.commons.cli.ParseException, IOException { // try/* w ww.j av a2 s .c o m*/ // { // if (args.length != 3) // { // System.out.println("Please specify processdefinition file (xml) and an outputfilename"); // } // // } // catch (ArrayIndexOutOfBoundsException e) // { // System.out.println("***ArrayIndexOutOfBoundsException: Please specify processdefinition.xml, openoffice_template.od*, newfile_for_processdefinitions.odt\n" + e.toString()); // } /*---------------------------- get options from ini-file ----------------------------*/ File inifile = new java.io.File( WhereAmI.getInstallDirectoryAbsolutePath(Launch.class) + "/" + "../etc/ptest-launch.ini"); if (inifile.exists()) { try { ini = new Ini(inifile); } catch (InvalidFileFormatException e1) { // TODO Auto-generated catch block e1.printStackTrace(); } catch (IOException e1) { // TODO Auto-generated catch block e1.printStackTrace(); } } else { System.err.println("ini file does not exist: " + inifile.getAbsolutePath()); System.exit(1); } /*---------------------------- create boolean options ----------------------------*/ Option ohelp = new Option("help", "print this message"); Option ov = new Option("v", "prints version and build-date"); /*---------------------------- create argument options ----------------------------*/ Option ospl = OptionBuilder.withArgName("DIR").hasArg() .withDescription("[mandatory] directory with sample input data") // .isRequired() .create("spl"); Option oinstancedir = OptionBuilder.withArgName("DIR").hasArg() .withDescription("[mandatory, default: .] directory where the test will be performed") // .isRequired() .create("instancedir"); Option ocall = OptionBuilder.withArgName("FILE").hasArg() .withDescription("[mandatory, default: random call in spl-directory] file with call-string") // .isRequired() .create("call"); Option oaltapp = OptionBuilder.withArgName("STRING").hasArg() .withDescription( "[optional] alternative app. this String replaces the first line of the .call-file.") // .isRequired() .create("altapp"); Option oaddopt = OptionBuilder.withArgName("STRING").hasArg() .withDescription("[optional] add an option to the call.") // .isRequired() .create("addopt"); Option onolaunch = new Option("nolaunch", "only create instance directory, copy all spl files, but do NOT launch the process"); /*---------------------------- create options object ----------------------------*/ Options options = new Options(); options.addOption(ohelp); options.addOption(ov); options.addOption(ospl); options.addOption(oinstancedir); options.addOption(ocall); options.addOption(oaltapp); options.addOption(oaddopt); options.addOption(onolaunch); /*---------------------------- create the parser ----------------------------*/ CommandLineParser parser = new GnuParser(); try { // parse the command line arguments commandline = parser.parse(options, args); } catch (Exception exp) { // oops, something went wrong System.err.println("Parsing failed. Reason: " + exp.getMessage()); exiter(); } /*---------------------------- usage/help ----------------------------*/ if (commandline.hasOption("help")) { HelpFormatter formatter = new HelpFormatter(); formatter.printHelp("launch", options); System.exit(0); } else if (commandline.hasOption("v")) { System.out.println("web: " + web); System.out.println("author: " + author); System.out.println("version:" + version); System.out.println("date: " + date); System.exit(0); } /*---------------------------- ueberpruefen ob eine schlechte kombination von parametern angegeben wurde ----------------------------*/ boolean error = false; String spl = null; String instancedir = null; String call = null; String altapp = null; ArrayList<String> addopt = new ArrayList<String>(); // spl initialisieren if (commandline.hasOption("spl")) { spl = commandline.getOptionValue("spl"); } else { System.err.println("option -spl is mandatory"); error = true; } // instancedir initialisieren if (commandline.hasOption("instancedir")) { instancedir = commandline.getOptionValue("instancedir"); } else { instancedir = System.getProperty("user.dir"); } // call initialisieren if (commandline.hasOption("call")) { call = commandline.getOptionValue("call"); } // altapp initialisieren if (commandline.hasOption("altapp")) { altapp = commandline.getOptionValue("altapp"); } // addopt initialisieren if (commandline.hasOption("addopt")) { for (String actString : commandline.getOptionValues("addopt")) { addopt.add(actString); } } // wenn fehler, dann exit if (error) { exiter(); } /*---------------------------- die lizenz ueberpruefen und ggf abbrechen ----------------------------*/ // check for valid license ArrayList<String> allPortAtHost = new ArrayList<String>(); allPortAtHost.add(ini.get("license-server", "license-server-1")); allPortAtHost.add(ini.get("license-server", "license-server-2")); allPortAtHost.add(ini.get("license-server", "license-server-3")); MyLicense lic = new MyLicense(allPortAtHost, "1", "user-edition", "0.1"); // lizenz-logging ausgeben for (String actLine : (ArrayList<String>) lic.getLog()) { System.err.println(actLine); } // abbruch, wenn lizenz nicht valide if (!lic.isValid()) { System.exit(1); } /*---------------------------- die eigentliche business logic ----------------------------*/ // das erste spl-objekt geben lassen Spl actSpl = new Splset(spl).getSpl().get(0); // den call, result und altapp ueberschreiben actSpl.setName("default"); if (call != null) { actSpl.setCall(new java.io.File(call)); } if (actSpl.getCall() == null) { System.err.println("error: no call information found"); System.exit(1); } if (altapp != null) { actSpl.setAltapp(altapp); } if (addopt.size() > 0) { actSpl.setAddopt(addopt); } actSpl.setResult(null); // das instancedir erstellen java.io.File actSplInstanceDir = new java.io.File(instancedir); System.err.println("info: creating directory " + actSplInstanceDir.getCanonicalPath()); actSplInstanceDir.mkdirs(); // Inputdaten in das InstanceDir exportieren actSpl.exportInput(actSplInstanceDir); // exit, wenn --nolaunch if (commandline.hasOption("nolaunch")) { System.err.println("info: exiting, because of -nolaunch"); System.exit(0); } // das logfile des Syscalls (zum debuggen des programms "process syscall" gedacht) String AbsLogSyscallWrapper = actSplInstanceDir.getCanonicalPath() + "/.log"; String AbsStdout = actSplInstanceDir.getCanonicalPath() + "/.stdout.txt"; String AbsStderr = actSplInstanceDir.getCanonicalPath() + "/.stderr.txt"; String AbsPid = actSplInstanceDir.getCanonicalPath() + "/.pid"; // beim starten von syscall werden parameter mit whitespaces an diesen auseinandergeschnitten und der nachfolgende aufruf schlaeft fehl // deshalb sollen whitespaces durch eine 'zeichensequenz' ersetzt werden // syscall ersetzt die zeichensequenz wieder zurueck in ein " " ArrayList<String> callFuerSyscall = actSpl.getCallAsArrayList(); ArrayList<String> callFuerSyscallMitTrennzeichen = new ArrayList<String>(); for (String actString : callFuerSyscall) { callFuerSyscallMitTrennzeichen.add(actString.replaceAll("\\s+", "%WHITESPACE%")); } try { // den Aufrufstring fuer die externe App (process syscall --version 0.6.0)) splitten // beim aufruf muss das erste argument im path zu finden sein, sonst gibt die fehlermeldung 'no such file or directory' ArrayList<String> processSyscallWithArgs = new ArrayList<String>( Arrays.asList(ini.get("apps", "pkraft-syscall").split(" "))); // die sonstigen argumente hinzufuegen processSyscallWithArgs.add("-call"); processSyscallWithArgs.add(String.join(" ", callFuerSyscallMitTrennzeichen)); // processSyscallWithArgs.add("\""+call+"\""); processSyscallWithArgs.add("-stdout"); processSyscallWithArgs.add(AbsStdout); processSyscallWithArgs.add("-stderr"); processSyscallWithArgs.add(AbsStderr); processSyscallWithArgs.add("-pid"); processSyscallWithArgs.add(AbsPid); processSyscallWithArgs.add("-mylog"); processSyscallWithArgs.add(AbsLogSyscallWrapper); processSyscallWithArgs.add("-maxrun"); processSyscallWithArgs.add("" + 3000); // erstellen prozessbuilder ProcessBuilder pb = new ProcessBuilder(processSyscallWithArgs); // erweitern des PATHs um den prozesseigenen path // Map<String,String> env = pb.environment(); // String path = env.get("PATH"); // log("debug", "$PATH="+path); // path = this.parent.getAbsPath()+":"+path; // env.put("PATH", path); // log("info", "path: "+path); // setzen der aktuellen directory (in der syscall ausgefuehrt werden soll) java.io.File directory = new java.io.File(instancedir); System.err.println("info: setting execution directory to: " + directory.getCanonicalPath()); pb.directory(directory); // zum debuggen ein paar ausgaben // java.lang.Process p1 = Runtime.getRuntime().exec("date >> ~/tmp.debug.work.txt"); // p1.waitFor(); // java.lang.Process p2 = Runtime.getRuntime().exec("ls -la "+this.getParent().getAbsdir()+" >> ~/tmp.debug.work.txt"); // p2.waitFor(); // java.lang.Process pro = Runtime.getRuntime().exec("nautilus"); // java.lang.Process superpro = Runtime.getRuntime().exec(processSyscallWithArgs.toArray(new String[processSyscallWithArgs.size()])); // p3.waitFor(); System.err.println("info: calling: " + pb.command()); // starten des prozesses java.lang.Process sysproc = pb.start(); // einfangen der stdout- und stderr des subprozesses InputStream is_stdout = sysproc.getInputStream(); InputStream is_stderr = sysproc.getErrorStream(); // Send your InputStream to an InputStreamReader: InputStreamReader isr_stdout = new InputStreamReader(is_stdout); InputStreamReader isr_stderr = new InputStreamReader(is_stderr); // That needs to go to a BufferedReader: BufferedReader br_stdout = new BufferedReader(isr_stdout); BufferedReader br_stderr = new BufferedReader(isr_stderr); // // oeffnen der OutputStreams zu den Ausgabedateien // FileWriter fw_stdout = new FileWriter(sStdout); // FileWriter fw_stderr = new FileWriter(sStderr); // zeilenweise in die files schreiben String line_out = new String(); String line_err = new String(); while (br_stdout.readLine() != null) { } // while (((line_out = br_stdout.readLine()) != null) || ((line_err = br_stderr.readLine()) != null)) // { // if (!(line_out == null)) // { // System.out.println(line_out); // System.out.flush(); // } // if (!(line_err == null)) // { // System.err.println(line_err); // System.err.flush(); // } // } int exitValue = sysproc.waitFor(); // fw_stdout.close(); // fw_stderr.close(); System.err.println("exitvalue: " + exitValue); sysproc.destroy(); System.exit(exitValue); // alternativer aufruf // java.lang.Process sysproc = Runtime.getRuntime().exec(StringUtils.join(args_for_syscall, " ")); // log("info", "call executed. pid="+sysproc.hashCode()); // wait 2 seconds for becoming the pid-file visible // Thread.sleep(2000); // int exitValue = sysproc.waitFor(); // // der prozess soll bis laengstens // if(exitValue != 0) // { // System.err.println("error: call returned a value indicating an error: "+exitValue); // } // else // { // System.err.println("info: call returned value: "+exitValue); // } // System.err.println("info: "+new Date().toString()); // System.err.println("info: bye"); // // sysproc.destroy(); // // System.exit(sysproc.exitValue()); } catch (Exception e2) { System.err.println("error: " + e2.getMessage()); System.exit(1); } }
From source file:edu.nyu.vida.data_polygamy.scalar_function_computation.Aggregation.java
/** * @param args/*from w w w .j av a 2 s . c o m*/ */ @SuppressWarnings({ "deprecation" }) public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException { Options options = new Options(); Option forceOption = new Option("f", "force", false, "force the computation of the aggregate functions " + "even if files already exist"); forceOption.setRequired(false); options.addOption(forceOption); Option gOption = new Option("g", "group", true, "set group of datasets for which the aggregate functions" + " will be computed, followed by their temporal and spatial attribute indices"); gOption.setRequired(true); gOption.setArgName("GROUP"); gOption.setArgs(Option.UNLIMITED_VALUES); options.addOption(gOption); Option machineOption = new Option("m", "machine", true, "machine identifier"); machineOption.setRequired(true); machineOption.setArgName("MACHINE"); machineOption.setArgs(1); options.addOption(machineOption); Option nodesOption = new Option("n", "nodes", true, "number of nodes"); nodesOption.setRequired(true); nodesOption.setArgName("NODES"); nodesOption.setArgs(1); options.addOption(nodesOption); Option s3Option = new Option("s3", "s3", false, "data on Amazon S3"); s3Option.setRequired(false); options.addOption(s3Option); Option awsAccessKeyIdOption = new Option("aws_id", "aws-id", true, "aws access key id; " + "this is required if the execution is on aws"); awsAccessKeyIdOption.setRequired(false); awsAccessKeyIdOption.setArgName("AWS-ACCESS-KEY-ID"); awsAccessKeyIdOption.setArgs(1); options.addOption(awsAccessKeyIdOption); Option awsSecretAccessKeyOption = new Option("aws_key", "aws-id", true, "aws secrect access key; " + "this is required if the execution is on aws"); awsSecretAccessKeyOption.setRequired(false); awsSecretAccessKeyOption.setArgName("AWS-SECRET-ACCESS-KEY"); awsSecretAccessKeyOption.setArgs(1); options.addOption(awsSecretAccessKeyOption); Option bucketOption = new Option("b", "s3-bucket", true, "bucket on s3; " + "this is required if the execution is on aws"); bucketOption.setRequired(false); bucketOption.setArgName("S3-BUCKET"); bucketOption.setArgs(1); options.addOption(bucketOption); Option helpOption = new Option("h", "help", false, "display this message"); helpOption.setRequired(false); options.addOption(helpOption); HelpFormatter formatter = new HelpFormatter(); CommandLineParser parser = new PosixParser(); CommandLine cmd = null; try { cmd = parser.parse(options, args); } catch (ParseException e) { formatter.printHelp("hadoop jar data-polygamy.jar " + "edu.nyu.vida.data_polygamy.scalar_function_computation.Aggregation", options, true); System.exit(0); } if (cmd.hasOption("h")) { formatter.printHelp("hadoop jar data-polygamy.jar " + "edu.nyu.vida.data_polygamy.scalar_function_computation.Aggregation", options, true); System.exit(0); } boolean s3 = cmd.hasOption("s3"); String s3bucket = ""; String awsAccessKeyId = ""; String awsSecretAccessKey = ""; if (s3) { if ((!cmd.hasOption("aws_id")) || (!cmd.hasOption("aws_key")) || (!cmd.hasOption("b"))) { System.out.println( "Arguments 'aws_id', 'aws_key', and 'b'" + " are mandatory if execution is on AWS."); formatter.printHelp( "hadoop jar data-polygamy.jar " + "edu.nyu.vida.data_polygamy.scalar_function_computation.Aggregation", options, true); System.exit(0); } s3bucket = cmd.getOptionValue("b"); awsAccessKeyId = cmd.getOptionValue("aws_id"); awsSecretAccessKey = cmd.getOptionValue("aws_key"); } boolean snappyCompression = false; boolean bzip2Compression = false; String machine = cmd.getOptionValue("m"); int nbNodes = Integer.parseInt(cmd.getOptionValue("n")); Configuration s3conf = new Configuration(); if (s3) { s3conf.set("fs.s3.awsAccessKeyId", awsAccessKeyId); s3conf.set("fs.s3.awsSecretAccessKey", awsSecretAccessKey); s3conf.set("bucket", s3bucket); } String datasetNames = ""; String datasetIds = ""; String preProcessingDatasets = ""; ArrayList<String> shortDataset = new ArrayList<String>(); ArrayList<String> shortDatasetAggregation = new ArrayList<String>(); HashMap<String, String> datasetTempAtt = new HashMap<String, String>(); HashMap<String, String> datasetSpatialAtt = new HashMap<String, String>(); HashMap<String, String> preProcessingDataset = new HashMap<String, String>(); HashMap<String, String> datasetId = new HashMap<String, String>(); boolean removeExistingFiles = cmd.hasOption("f"); String[] datasetArgs = cmd.getOptionValues("g"); for (int i = 0; i < datasetArgs.length; i += 3) { String dataset = datasetArgs[i]; // getting pre-processing String tempPreProcessing = FrameworkUtils.searchPreProcessing(dataset, s3conf, s3); if (tempPreProcessing == null) { System.out.println("No pre-processing available for " + dataset); continue; } preProcessingDataset.put(dataset, tempPreProcessing); shortDataset.add(dataset); datasetTempAtt.put(dataset, ((datasetArgs[i + 1] == "null") ? null : datasetArgs[i + 1])); datasetSpatialAtt.put(dataset, ((datasetArgs[i + 2] == "null") ? null : datasetArgs[i + 2])); datasetId.put(dataset, null); } if (shortDataset.size() == 0) { System.out.println("No datasets to process."); System.exit(0); } // getting dataset id Path path = null; FileSystem fs = null; if (s3) { path = new Path(s3bucket + FrameworkUtils.datasetsIndexDir); fs = FileSystem.get(path.toUri(), s3conf); } else { fs = FileSystem.get(new Configuration()); path = new Path(fs.getHomeDirectory() + "/" + FrameworkUtils.datasetsIndexDir); } BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(path))); String line = br.readLine(); while (line != null) { String[] dt = line.split("\t"); if (datasetId.containsKey(dt[0])) { datasetId.put(dt[0], dt[1]); datasetNames += dt[0] + ","; datasetIds += dt[1] + ","; } line = br.readLine(); } br.close(); if (s3) fs.close(); datasetNames = datasetNames.substring(0, datasetNames.length() - 1); datasetIds = datasetIds.substring(0, datasetIds.length() - 1); Iterator<String> it = shortDataset.iterator(); while (it.hasNext()) { String dataset = it.next(); if (datasetId.get(dataset) == null) { System.out.println("No dataset id for " + dataset); System.exit(0); } } FrameworkUtils.createDir(s3bucket + FrameworkUtils.aggregatesDir, s3conf, s3); // getting smallest resolution HashMap<String, String> tempResMap = new HashMap<String, String>(); HashMap<String, String> spatialResMap = new HashMap<String, String>(); HashMap<String, String> datasetTemporalStrMap = new HashMap<String, String>(); HashMap<String, String> datasetSpatialStrMap = new HashMap<String, String>(); HashSet<String> input = new HashSet<String>(); for (String dataset : shortDataset) { String[] datasetArray = preProcessingDataset.get(dataset).split("-"); String datasetTemporalStr = datasetArray[datasetArray.length - 2]; int datasetTemporal = utils.temporalResolution(datasetTemporalStr); String datasetSpatialStr = datasetArray[datasetArray.length - 1]; int datasetSpatial = utils.spatialResolution(datasetSpatialStr); // finding all possible resolutions String[] temporalResolutions = FrameworkUtils.getAggTempResolutions(datasetTemporal); String[] spatialResolutions = FrameworkUtils.getAggSpatialResolutions(datasetSpatial); String temporalResolution = ""; String spatialResolution = ""; String tempRes = ""; String spatialRes = ""; boolean dataAdded = false; for (int i = 0; i < temporalResolutions.length; i++) { for (int j = 0; j < spatialResolutions.length; j++) { temporalResolution = temporalResolutions[i]; spatialResolution = spatialResolutions[j]; String aggregatesOutputFileName = s3bucket + FrameworkUtils.aggregatesDir + "/" + dataset + "/"; if (removeExistingFiles) { FrameworkUtils.removeFile(aggregatesOutputFileName, s3conf, s3); } if (!FrameworkUtils.fileExists(aggregatesOutputFileName, s3conf, s3)) { dataAdded = true; tempRes += temporalResolution + "-"; spatialRes += spatialResolution + "-"; } } } if (dataAdded) { input.add(s3bucket + FrameworkUtils.preProcessingDir + "/" + preProcessingDataset.get(dataset)); shortDatasetAggregation.add(dataset); tempResMap.put(dataset, tempRes.substring(0, tempRes.length() - 1)); spatialResMap.put(dataset, spatialRes.substring(0, spatialRes.length() - 1)); datasetTemporalStrMap.put(dataset, datasetTemporalStr); datasetSpatialStrMap.put(dataset, datasetSpatialStr); } } if (input.isEmpty()) { System.out.println("All the input datasets have aggregates."); System.out.println("Use -f in the beginning of the command line to force the computation."); System.exit(0); } it = input.iterator(); while (it.hasNext()) { preProcessingDatasets += it.next() + ","; } Job aggJob = null; String aggregatesOutputDir = s3bucket + FrameworkUtils.aggregatesDir + "/tmp/"; String jobName = "aggregates"; FrameworkUtils.removeFile(aggregatesOutputDir, s3conf, s3); Configuration aggConf = new Configuration(); Machine machineConf = new Machine(machine, nbNodes); aggConf.set("dataset-name", datasetNames); aggConf.set("dataset-id", datasetIds); for (int i = 0; i < shortDatasetAggregation.size(); i++) { String dataset = shortDatasetAggregation.get(i); String id = datasetId.get(dataset); aggConf.set("dataset-" + id + "-temporal-resolutions", tempResMap.get(dataset)); aggConf.set("dataset-" + id + "-spatial-resolutions", spatialResMap.get(dataset)); aggConf.set("dataset-" + id + "-temporal-att", datasetTempAtt.get(dataset)); aggConf.set("dataset-" + id + "-spatial-att", datasetSpatialAtt.get(dataset)); aggConf.set("dataset-" + id + "-temporal", datasetTemporalStrMap.get(dataset)); aggConf.set("dataset-" + id + "-spatial", datasetSpatialStrMap.get(dataset)); if (s3) aggConf.set("dataset-" + id, s3bucket + FrameworkUtils.preProcessingDir + "/" + preProcessingDataset.get(dataset)); else aggConf.set("dataset-" + id, FileSystem.get(new Configuration()).getHomeDirectory() + "/" + FrameworkUtils.preProcessingDir + "/" + preProcessingDataset.get(dataset)); } aggConf.set("mapreduce.tasktracker.map.tasks.maximum", String.valueOf(machineConf.getMaximumTasks())); aggConf.set("mapreduce.tasktracker.reduce.tasks.maximum", String.valueOf(machineConf.getMaximumTasks())); aggConf.set("mapreduce.jobtracker.maxtasks.perjob", "-1"); aggConf.set("mapreduce.reduce.shuffle.parallelcopies", "20"); aggConf.set("mapreduce.input.fileinputformat.split.minsize", "0"); aggConf.set("mapreduce.task.io.sort.mb", "200"); aggConf.set("mapreduce.task.io.sort.factor", "100"); machineConf.setMachineConfiguration(aggConf); if (s3) { machineConf.setMachineConfiguration(aggConf); aggConf.set("fs.s3.awsAccessKeyId", awsAccessKeyId); aggConf.set("fs.s3.awsSecretAccessKey", awsSecretAccessKey); } if (snappyCompression) { aggConf.set("mapreduce.map.output.compress", "true"); aggConf.set("mapreduce.map.output.compress.codec", "org.apache.hadoop.io.compress.SnappyCodec"); //aggConf.set("mapreduce.output.fileoutputformat.compress.codec", "org.apache.hadoop.io.compress.SnappyCodec"); } if (bzip2Compression) { aggConf.set("mapreduce.map.output.compress", "true"); aggConf.set("mapreduce.map.output.compress.codec", "org.apache.hadoop.io.compress.BZip2Codec"); //aggConf.set("mapreduce.output.fileoutputformat.compress.codec", "org.apache.hadoop.io.compress.BZip2Codec"); } aggJob = new Job(aggConf); aggJob.setJobName(jobName); aggJob.setMapOutputKeyClass(SpatioTemporalWritable.class); aggJob.setMapOutputValueClass(AggregationArrayWritable.class); aggJob.setOutputKeyClass(SpatioTemporalWritable.class); aggJob.setOutputValueClass(FloatArrayWritable.class); //aggJob.setOutputKeyClass(Text.class); //aggJob.setOutputValueClass(Text.class); aggJob.setMapperClass(AggregationMapper.class); aggJob.setCombinerClass(AggregationCombiner.class); aggJob.setReducerClass(AggregationReducer.class); aggJob.setNumReduceTasks(machineConf.getNumberReduces()); aggJob.setInputFormatClass(SequenceFileInputFormat.class); //aggJob.setOutputFormatClass(SequenceFileOutputFormat.class); LazyOutputFormat.setOutputFormatClass(aggJob, SequenceFileOutputFormat.class); //LazyOutputFormat.setOutputFormatClass(aggJob, TextOutputFormat.class); SequenceFileOutputFormat.setCompressOutput(aggJob, true); SequenceFileOutputFormat.setOutputCompressionType(aggJob, CompressionType.BLOCK); FileInputFormat.setInputDirRecursive(aggJob, true); FileInputFormat.setInputPaths(aggJob, preProcessingDatasets.substring(0, preProcessingDatasets.length() - 1)); FileOutputFormat.setOutputPath(aggJob, new Path(aggregatesOutputDir)); aggJob.setJarByClass(Aggregation.class); long start = System.currentTimeMillis(); aggJob.submit(); aggJob.waitForCompletion(true); System.out.println(jobName + "\t" + (System.currentTimeMillis() - start)); // moving files to right place for (String dataset : shortDatasetAggregation) { String from = s3bucket + FrameworkUtils.aggregatesDir + "/tmp/" + dataset + "/"; String to = s3bucket + FrameworkUtils.aggregatesDir + "/" + dataset + "/"; FrameworkUtils.renameFile(from, to, s3conf, s3); } }
From source file:edu.oregonstate.eecs.mcplan.ml.KernelPrincipalComponentsAnalysis.java
public static void main(final String[] args) throws FileNotFoundException { final File root = new File("test/KernelPrincipalComponentsAnalysis"); root.mkdirs();/*from www . j a v a 2 s. c om*/ final int seed = 42; final int N = 30; final RandomGenerator rng = new MersenneTwister(seed); final ArrayList<RealVector> data = new ArrayList<RealVector>(); final ArrayList<RealVector> shuffled = new ArrayList<RealVector>(); // final double[][] covariance = new double[][] { {1.0, 0.0}, // {0.0, 1.0} }; // final MultivariateNormalDistribution p = new MultivariateNormalDistribution( // rng, new double[] { 0.0, 0.0 }, covariance ); // final MultivariateNormalDistribution q = new MultivariateNormalDistribution( // rng, new double[] { 10.0, 0.0 }, covariance ); // // for( int i = 0; i < N; ++i ) { // data.add( new ArrayRealVector( p.sample() ) ); // data.add( new ArrayRealVector( q.sample() ) ); // } // Fn.shuffle( rng, data ); final double sigma = 0.01; final double[][] covariance = new double[][] { { sigma, 0.0 }, { 0.0, sigma } }; final MultivariateNormalDistribution p = new MultivariateNormalDistribution(rng, new double[] { 0.0, 0.0 }, covariance); for (final double r : new double[] { 1.0, 3.0, 5.0 }) { for (int i = 0; i < N; ++i) { final double theta = i * 2 * Math.PI / N; final double[] noise = p.sample(); final RealVector v = new ArrayRealVector( new double[] { r * Math.cos(theta) + noise[0], r * Math.sin(theta) + noise[1] }); data.add(v); shuffled.add(v); } } Fn.shuffle(rng, shuffled); final Csv.Writer data_writer = new Csv.Writer(new PrintStream(new File(root, "data.csv"))); for (final RealVector v : data) { for (int i = 0; i < v.getDimension(); ++i) { data_writer.cell(v.getEntry(i)); } data_writer.newline(); } data_writer.close(); System.out.println("[Training]"); final int Ncomponents = 2; final KernelPrincipalComponentsAnalysis<RealVector> kpca = new KernelPrincipalComponentsAnalysis<RealVector>( shuffled, new RadialBasisFunctionKernel(0.5), 1e-6); System.out.println("[Finished]"); for (int i = 0; i < Ncomponents; ++i) { System.out.println(kpca.eigenvectors.get(i)); } System.out.println("Transformed data:"); final KernelPrincipalComponentsAnalysis.Transformer<RealVector> transformer = kpca .makeTransformer(Ncomponents); final Csv.Writer transformed_writer = new Csv.Writer(new PrintStream(new File(root, "transformed.csv"))); for (final RealVector u : data) { final RealVector v = transformer.transform(u); System.out.println(v); for (int i = 0; i < v.getDimension(); ++i) { transformed_writer.cell(v.getEntry(i)); } transformed_writer.newline(); } transformed_writer.close(); }
From source file:net.massbank.validator.RecordValidator.java
public static void main(String[] args) { RequestDummy request;//from ww w.j av a2s.c o m PrintStream out = System.out; Options lvOptions = new Options(); lvOptions.addOption("h", "help", false, "show this help."); lvOptions.addOption("r", "recdata", true, "points to the recdata directory containing massbank records. Reads all *.txt files in there."); CommandLineParser lvParser = new BasicParser(); CommandLine lvCmd = null; try { lvCmd = lvParser.parse(lvOptions, args); if (lvCmd.hasOption('h')) { printHelp(lvOptions); return; } } catch (org.apache.commons.cli.ParseException pvException) { System.out.println(pvException.getMessage()); } String recDataPath = lvCmd.getOptionValue("recdata"); // --------------------------------------------- // ???? // --------------------------------------------- final String baseUrl = MassBankEnv.get(MassBankEnv.KEY_BASE_URL); final String dbRootPath = "./"; final String dbHostName = MassBankEnv.get(MassBankEnv.KEY_DB_HOST_NAME); final String tomcatTmpPath = "."; final String tmpPath = (new File(tomcatTmpPath + sdf.format(new Date()))).getPath() + File.separator; GetConfig conf = new GetConfig(baseUrl); int recVersion = 2; String selDbName = ""; Object up = null; // Was: file Upload boolean isResult = true; String upFileName = ""; boolean upResult = false; DatabaseAccess db = null; try { // ---------------------------------------------------- // ??? // ---------------------------------------------------- // if (FileUpload.isMultipartContent(request)) { // (new File(tmpPath)).mkdir(); // String os = System.getProperty("os.name"); // if (os.indexOf("Windows") == -1) { // isResult = FileUtil.changeMode("777", tmpPath); // if (!isResult) { // out.println(msgErr("[" + tmpPath // + "] chmod failed.")); // return; // } // } // up = new FileUpload(request, tmpPath); // } // ---------------------------------------------------- // ?DB???? // ---------------------------------------------------- List<String> dbNameList = Arrays.asList(conf.getDbName()); ArrayList<String> dbNames = new ArrayList<String>(); dbNames.add(""); File[] dbDirs = (new File(dbRootPath)).listFiles(); if (dbDirs != null) { for (File dbDir : dbDirs) { if (dbDir.isDirectory()) { int pos = dbDir.getName().lastIndexOf("\\"); String dbDirName = dbDir.getName().substring(pos + 1); pos = dbDirName.lastIndexOf("/"); dbDirName = dbDirName.substring(pos + 1); if (dbNameList.contains(dbDirName)) { // DB???massbank.conf???DB???? dbNames.add(dbDirName); } } } } if (dbDirs == null || dbNames.size() == 0) { out.println(msgErr("[" + dbRootPath + "] directory not exist.")); return; } Collections.sort(dbNames); // ---------------------------------------------------- // ? // ---------------------------------------------------- // if (FileUpload.isMultipartContent(request)) { // HashMap<String, String[]> reqParamMap = new HashMap<String, // String[]>(); // reqParamMap = up.getRequestParam(); // if (reqParamMap != null) { // for (Map.Entry<String, String[]> req : reqParamMap // .entrySet()) { // if (req.getKey().equals("ver")) { // try { // recVersion = Integer // .parseInt(req.getValue()[0]); // } catch (NumberFormatException nfe) { // } // } else if (req.getKey().equals("db")) { // selDbName = req.getValue()[0]; // } // } // } // } else { // if (request.getParameter("ver") != null) { // try { // recVersion = Integer.parseInt(request // .getParameter("ver")); // } catch (NumberFormatException nfe) { // } // } // selDbName = request.getParameter("db"); // } // if (selDbName == null || selDbName.equals("") // || !dbNames.contains(selDbName)) { // selDbName = dbNames.get(0); // } // --------------------------------------------- // // --------------------------------------------- out.println("Database: "); for (int i = 0; i < dbNames.size(); i++) { String dbName = dbNames.get(i); out.print("dbName"); if (dbName.equals(selDbName)) { out.print(" selected"); } if (i == 0) { out.println("------------------"); } else { out.println(dbName); } } out.println("Record Version : "); out.println(recVersion); out.println("Record Archive :"); // --------------------------------------------- // // --------------------------------------------- // HashMap<String, Boolean> upFileMap = up.doUpload(); // if (upFileMap != null) { // for (Map.Entry<String, Boolean> e : upFileMap.entrySet()) { // upFileName = e.getKey(); // upResult = e.getValue(); // break; // } // if (upFileName.equals("")) { // out.println(msgErr("please select file.")); // isResult = false; // } else if (!upResult) { // out.println(msgErr("[" + upFileName // + "] upload failed.")); // isResult = false; // } else if (!upFileName.endsWith(ZIP_EXTENSION) // && !upFileName.endsWith(MSBK_EXTENSION)) { // out.println(msgErr("please select [" // + UPLOAD_RECDATA_ZIP // + "] or [" // + UPLOAD_RECDATA_MSBK + "].")); // up.deleteFile(upFileName); // isResult = false; // } // } else { // out.println(msgErr("server error.")); // isResult = false; // } // up.deleteFileItem(); // if (!isResult) { // return; // } // --------------------------------------------- // ??? // --------------------------------------------- // final String upFilePath = (new File(tmpPath + File.separator // + upFileName)).getPath(); // isResult = FileUtil.unZip(upFilePath, tmpPath); // if (!isResult) { // out.println(msgErr("[" // + upFileName // + "] extraction failed. possibility of time-out.")); // return; // } // --------------------------------------------- // ?? // --------------------------------------------- final String recPath = (new File(dbRootPath + File.separator + selDbName)).getPath(); File tmpRecDir = new File(recDataPath); if (!tmpRecDir.isDirectory()) { tmpRecDir.mkdirs(); } // --------------------------------------------- // ??? // --------------------------------------------- // data? // final String recDataPath = (new File(tmpPath + File.separator // + RECDATA_DIR_NAME)).getPath() // + File.separator; // // if (!(new File(recDataPath)).isDirectory()) { // if (upFileName.endsWith(ZIP_EXTENSION)) { // out.println(msgErr("[" // + RECDATA_DIR_NAME // + "] directory is not included in the up-loading file.")); // } else if (upFileName.endsWith(MSBK_EXTENSION)) { // out.println(msgErr("The uploaded file is not record data.")); // } // return; // } // --------------------------------------------- // DB // --------------------------------------------- // db = new DatabaseAccess(dbHostName, selDbName); // isResult = db.open(); // if (!isResult) { // db.close(); // out.println(msgErr("not connect to database.")); // return; // } // --------------------------------------------- // ?? // --------------------------------------------- TreeMap<String, String> resultMap = validationRecord(db, out, recDataPath, recPath, recVersion); if (resultMap.size() == 0) { return; } // --------------------------------------------- // ? // --------------------------------------------- isResult = dispResult(out, resultMap); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } finally { if (db != null) { db.close(); } File tmpDir = new File(tmpPath); if (tmpDir.exists()) { FileUtil.removeDir(tmpDir.getPath()); } } }
From source file:mlbench.pagerank.PagerankNaive.java
@SuppressWarnings({ "rawtypes", "unchecked" })
public static void main(String[] args) throws IOException, InterruptedException {
try {/* w w w.j a v a 2s.c o m*/
parseArgs(args);
HashMap<String, String> conf = new HashMap<String, String>();
initConf(conf);
MPI_D.Init(args, MPI_D.Mode.Common, conf);
JobConf jobConf = new JobConf(confPath);
if (MPI_D.COMM_BIPARTITE_O != null) {
// O communicator
int rank = MPI_D.Comm_rank(MPI_D.COMM_BIPARTITE_O);
int size = MPI_D.Comm_size(MPI_D.COMM_BIPARTITE_O);
if (rank == 0) {
LOG.info(PagerankNaive.class.getSimpleName() + " O start.");
}
FileSplit[] inputs1 = DataMPIUtil.HDFSDataLocalLocator.getTaskInputs(MPI_D.COMM_BIPARTITE_O,
jobConf, edgeDir, rank);
FileSplit[] inputs2 = DataMPIUtil.HDFSDataLocalLocator.getTaskInputs(MPI_D.COMM_BIPARTITE_O,
jobConf, vecDir, rank);
FileSplit[] inputs = (FileSplit[]) ArrayUtils.addAll(inputs2, inputs1);
for (int i = 0; i < inputs.length; i++) {
FileSplit fsplit = inputs[i];
LineRecordReader kvrr = new LineRecordReader(jobConf, fsplit);
LongWritable key = kvrr.createKey();
Text value = kvrr.createValue();
{
IntWritable k = new IntWritable();
Text v = new Text();
while (kvrr.next(key, value)) {
String line_text = value.toString();
// ignore comments in edge file
if (line_text.startsWith("#"))
continue;
final String[] line = line_text.split("\t");
if (line.length < 2)
continue;
// vector : ROWID VALUE('vNNNN')
if (line[1].charAt(0) == 'v') {
k.set(Integer.parseInt(line[0]));
v.set(line[1]);
MPI_D.Send(k, v);
} else {
/*
* In other matrix-vector multiplication, we
* output (dst, src) here However, In PageRank,
* the matrix-vector computation formula is M^T
* * v. Therefore, we output (src,dst) here.
*/
int src_id = Integer.parseInt(line[0]);
int dst_id = Integer.parseInt(line[1]);
k.set(src_id);
v.set(line[1]);
MPI_D.Send(k, v);
if (make_symmetric == 1) {
k.set(dst_id);
v.set(line[0]);
MPI_D.Send(k, v);
}
}
}
}
}
} else if (MPI_D.COMM_BIPARTITE_A != null) {
// A communicator
int rank = MPI_D.Comm_rank(MPI_D.COMM_BIPARTITE_A);
if (rank == 0) {
LOG.info(PagerankNaive.class.getSimpleName() + " A start.");
}
HadoopWriter<IntWritable, Text> outrw = HadoopIOUtil.getNewWriter(jobConf, outDir,
IntWritable.class, Text.class, TextOutputFormat.class, null, rank, MPI_D.COMM_BIPARTITE_A);
IntWritable oldKey = null;
int i;
double cur_rank = 0;
ArrayList<Integer> dst_nodes_list = new ArrayList<Integer>();
Object[] keyValue = MPI_D.Recv();
while (keyValue != null) {
IntWritable key = (IntWritable) keyValue[0];
Text value = (Text) keyValue[1];
if (oldKey == null) {
oldKey = key;
}
// A new key arrives
if (!key.equals(oldKey)) {
outrw.write(oldKey, new Text("s" + cur_rank));
int outdeg = dst_nodes_list.size();
if (outdeg > 0) {
cur_rank = cur_rank / (double) outdeg;
}
for (i = 0; i < outdeg; i++) {
outrw.write(new IntWritable(dst_nodes_list.get(i)), new Text("v" + cur_rank));
}
oldKey = key;
cur_rank = 0;
dst_nodes_list = new ArrayList<Integer>();
}
// common record
String line_text = value.toString();
final String[] line = line_text.split("\t");
if (line.length == 1) {
if (line_text.charAt(0) == 'v') { // vector : VALUE
cur_rank = Double.parseDouble(line_text.substring(1));
} else { // edge : ROWID
dst_nodes_list.add(Integer.parseInt(line[0]));
}
}
keyValue = MPI_D.Recv();
}
// write the left part
if (cur_rank != 0) {
outrw.write(oldKey, new Text("s" + cur_rank));
int outdeg = dst_nodes_list.size();
if (outdeg > 0) {
cur_rank = cur_rank / (double) outdeg;
}
for (i = 0; i < outdeg; i++) {
outrw.write(new IntWritable(dst_nodes_list.get(i)), new Text("v" + cur_rank));
}
}
outrw.close();
}
MPI_D.Finalize();
} catch (MPI_D_Exception e) {
e.printStackTrace();
}
}
From source file:de.prozesskraft.pramp.parts.PrampPartUi1.java
/** * @param args//from w ww . ja v a 2 s. c o m */ public static void main(String[] args) { /*---------------------------- get options from ini-file ----------------------------*/ File inifile = new java.io.File( WhereAmI.getInstallDirectoryAbsolutePath(PrampPartUi1.class) + "/" + "../etc/pramp-gui.ini"); if (inifile.exists()) { try { ini = new Ini(inifile); } catch (InvalidFileFormatException e1) { // TODO Auto-generated catch block e1.printStackTrace(); } catch (IOException e1) { // TODO Auto-generated catch block e1.printStackTrace(); } } else { System.err.println("ini file does not exist: " + inifile.getAbsolutePath()); System.exit(1); } /*---------------------------- create boolean options ----------------------------*/ Option help = new Option("help", "print this message"); Option v = new Option("v", "prints version and build-date"); /*---------------------------- create argument options ----------------------------*/ Option dbfile = OptionBuilder.withArgName("dbfile").hasArg().withDescription("[optional] dbfile") // .isRequired() .create("dbfile"); /*---------------------------- create options object ----------------------------*/ Options options = new Options(); options.addOption(help); options.addOption(v); options.addOption(dbfile); /*---------------------------- create the parser ----------------------------*/ CommandLineParser parser = new GnuParser(); try { // parse the command line arguments line = parser.parse(options, args); } catch (Exception exp) { // oops, something went wrong System.err.println("Parsing failed. Reason: " + exp.getMessage()); } /*---------------------------- usage/help ----------------------------*/ if (line.hasOption("help")) { HelpFormatter formatter = new HelpFormatter(); // formatter.printHelp("checkin --version [% version %]", options); formatter.printHelp("pramp-gui", options); System.exit(0); } if (line.hasOption("v")) { System.out.println("author: alexander.vogel@caegroup.de"); System.out.println("version: [% version %]"); System.out.println("date: [% date %]"); System.exit(0); } /*---------------------------- die lizenz ueberpruefen und ggf abbrechen ----------------------------*/ // check for valid license ArrayList<String> allPortAtHost = new ArrayList<String>(); allPortAtHost.add(ini.get("license-server", "license-server-1")); allPortAtHost.add(ini.get("license-server", "license-server-2")); allPortAtHost.add(ini.get("license-server", "license-server-3")); MyLicense lic = new MyLicense(allPortAtHost, "1", "user-edition", "0.1"); // lizenz-logging ausgeben for (String actLine : (ArrayList<String>) lic.getLog()) { System.err.println(actLine); } // abbruch, wenn lizenz nicht valide if (!lic.isValid()) { System.exit(1); } // gui final Display display = new Display(); Realm.runWithDefault(SWTObservables.getRealm(display), new Runnable() { public void run() { try { shell = new Shell(display); shell.setText("pramp " + "[% version %]"); // set an icon if (this.getClass().getResourceAsStream("/logoSymbol50Transp.png") != null) { shell.setImage( new Image(display, this.getClass().getResourceAsStream("/logoSymbol50Transp.png"))); } else if ((new java.io.File("logoSymbol50Transp.png")).exists()) { shell.setImage(new Image(display, "logoSymbol50Transp.png")); } shell.setLayout(new FillLayout()); shell.setSize(1000, 780); Composite composite = new Composite(shell, SWT.NO_FOCUS); GridLayout gl_composite = new GridLayout(2, false); gl_composite.marginWidth = 0; gl_composite.marginHeight = 0; new PrampPartUi1(composite); try { shell.open(); while (!shell.isDisposed()) { if (!display.readAndDispatch()) { display.sleep(); } } } finally { if (!shell.isDisposed()) { shell.dispose(); } } } finally { display.dispose(); } } }); System.exit(0); }
From source file:com.sun.faban.harness.util.CLI.java
/** * The first argument to the CLI is the action. It can be:<ul> * <li>pending</li>/* w w w .j a va 2s. co m*/ * <li>status runId</li> * <li>submit benchmark profile configfile.xml</ul> * </ul> * * @param args The command line arguments. */ public static void main(String[] args) { if (args.length == 0) { printUsage(); System.exit(1); } ArrayList<String> argList = new ArrayList<String>(); // Do the getopt thing. char opt = (char) -1; String master = null; String user = null; String password = null; for (String arg : args) { if (arg.startsWith("-M")) { String optArg = arg.substring(2); if (optArg.length() == 0) { opt = 'M'; continue; } master = optArg; } else if (arg.startsWith("-U")) { String optArg = arg.substring(2); if (optArg.length() == 0) { opt = 'U'; continue; } user = optArg; } else if (arg.startsWith("-P")) { String optArg = arg.substring(2); if (optArg.length() == 0) { opt = 'P'; continue; } password = optArg; } else if (opt != (char) -1) { switch (opt) { case 'M': master = arg; opt = (char) -1; break; case 'U': user = arg; opt = (char) -1; break; case 'P': password = arg; opt = (char) -1; break; } } else { argList.add(arg); opt = (char) -1; } } if (master == null) master = "http://localhost:9980/"; else if (!master.endsWith("/")) master += '/'; CLI cli = new CLI(); String action = argList.get(0); try { if ("pending".equals(action)) { cli.doGet(master + "pending"); } else if ("status".equals(action)) { if (argList.size() > 1) cli.doGet(master + "status/" + argList.get(1)); else printUsage(); } else if ("submit".equals(action)) { if (argList.size() > 3) { cli.doPostSubmit(master, user, password, argList); } else { printUsage(); System.exit(1); } } else if ("kill".equals(action)) { if (argList.size() > 1) { cli.doPostKill(master, user, password, argList); } else { printUsage(); System.exit(1); } } else if ("wait".equals(action)) { if (argList.size() > 1) { cli.pollStatus(master + "status/" + argList.get(1)); } else { printUsage(); System.exit(1); } } else if ("showlogs".equals(action)) { StringBuilder url = new StringBuilder(); if (argList.size() > 1) { url.append(master).append("logs/"); url.append(argList.get(1)); } else { printUsage(); } for (int i = 2; i < argList.size(); i++) { if ("-t".equals(argList.get(i))) url.append("/tail"); if ("-f".equals(argList.get(i))) url.append("/follow"); if ("-ft".equals(argList.get(i))) url.append("/tail/follow"); if ("-tf".equals(argList.get(i))) url.append("/tail/follow"); } cli.doGet(url.toString()); } else { printUsage(); } } catch (IOException e) { System.err.println(e.getMessage()); System.exit(1); } }
From source file:com.zimbra.cs.util.ProxyPurgeUtil.java
public static void main(String[] args) throws ServiceException { CommandLine commandLine;/*from ww w . j a v a 2 s. c o m*/ ArrayList<String> servers; ArrayList<String> accounts; String outputformat; boolean purge = false; Provisioning prov; List<Server> memcachedServers; String logLevel = "ERROR"; /* Parse the command-line arguments, and display usage if necessary */ try { commandLine = parseCommandLine(args); } catch (ParseException pe) { commandLine = null; } if ((commandLine == null) || commandLine.hasOption("h") || commandLine.hasOption("u")) { usage(); System.exit(1); } if (commandLine.hasOption("v")) { logLevel = "DEBUG"; } ZimbraLog.toolSetupLog4j(logLevel, null, false); /* Initialize the logging system and the zimbra environment */ prov = Provisioning.getInstance(); /* Get the list of servers running the memcached service this is equivalent to the $(zmprov gamcs) command */ memcachedServers = prov.getAllServers(Provisioning.SERVICE_MEMCACHED); servers = new ArrayList<String>(); for (Iterator<Server> it = memcachedServers.iterator(); it.hasNext();) { Server s = it.next(); String serverName = s.getAttr(Provisioning.A_zimbraServiceHostname, "localhost"); String servicePort = s.getAttr(Provisioning.A_zimbraMemcachedBindPort, memcachedPort); servers.add(serverName + ":" + servicePort); } accounts = getAccounts(commandLine); servers.addAll(getCacheServers(commandLine)); if (servers.size() == 0) { System.err.println("No memcached servers found, and none specified (--help for help)"); System.exit(1); } if (accounts.size() == 0) { System.err.println("No accounts specified (--help for help)"); System.exit(1); } /* Assume purge unless `-i' is specified */ /* -i (info) indicates that account route info should be printed -p (purge) indicates that account route info should be purged */ purge = (commandLine.hasOption("i") == false); /* parse the format string */ if (commandLine.hasOption("o")) { outputformat = commandLine.getOptionValue("o"); } else { outputformat = "[%1$s] %2$s -- %3$s"; } purgeAccounts(servers, accounts, purge, outputformat); }