List of usage examples for java.util List indexOf
int indexOf(Object o);
From source file:Main.java
public static void main(String[] a) { List list = Arrays.asList(new String[] { "A", "B", "C", "D" }); System.out.println(list.indexOf("A")); }
From source file:Main.java
public static void main(String args[]) { JFrame frame = new JFrame("Reverse Sample"); frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE); frame.setLayout(new GridLayout(3, 3)); for (int i = 0; i < 9; i++) { JButton button = new JButton(Integer.toString(i)); frame.add(button);//from w w w . java 2 s . c o m } final Container contentPane = frame.getContentPane(); Comparator<Component> comp = new Comparator<Component>() { public int compare(Component c1, Component c2) { Component comps[] = contentPane.getComponents(); List list = Arrays.asList(comps); int first = list.indexOf(c1); int second = list.indexOf(c2); return second - first; } }; FocusTraversalPolicy policy = new SortingFocusTraversalPolicy(comp); frame.setFocusTraversalPolicy(policy); frame.setSize(300, 200); frame.setVisible(true); }
From source file:MainClass.java
public static void main(String args[]) { JFrame frame = new JFrame("Reverse Sample"); frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE); frame.setLayout(new GridLayout(3, 3)); for (int i = 9; i > 0; i--) { JButton button = new JButton(Integer.toString(i)); frame.add(button, 0);/*from w ww .j a v a 2 s . c o m*/ } final Container contentPane = frame.getContentPane(); Comparator<Component> comp = new Comparator<Component>() { public int compare(Component c1, Component c2) { Component comps[] = contentPane.getComponents(); List list = Arrays.asList(comps); int first = list.indexOf(c1); int second = list.indexOf(c2); return second - first; } }; FocusTraversalPolicy policy = new SortingFocusTraversalPolicy(comp); frame.setFocusTraversalPolicy(policy); frame.setSize(300, 200); frame.setVisible(true); }
From source file:Main.java
public static void main(String[] a) { Map<String, String> yourMap = new HashMap<String, String>(); yourMap.put("1", "one"); yourMap.put("2", "two"); yourMap.put("3", "three"); Map<String, Object> map = new LinkedHashMap<String, Object>(); List<String> keyList = new ArrayList<String>(yourMap.keySet()); List<String> valueList = new ArrayList<String>(yourMap.values()); Set<String> sortedSet = new TreeSet<String>(valueList); Object[] sortedArray = sortedSet.toArray(); int size = sortedArray.length; for (int i = 0; i < size; i++) { map.put(keyList.get(valueList.indexOf(sortedArray[i])), sortedArray[i]); }//from w w w . ja v a 2s . c o m Set ref = map.keySet(); Iterator it = ref.iterator(); while (it.hasNext()) { String i = (String) it.next(); System.out.println(i); } }
From source file:de.tu.darmstadt.lt.ner.preprocessing.GermaNERMain.java
public static void main(String[] arg) throws Exception { long startTime = System.currentTimeMillis(); String usage = "USAGE: java -jar germanner.jar [-c config.properties] \n" + " [-f trainingFileName] -t testFileName -d ModelOutputDirectory-o outputFile"; long start = System.currentTimeMillis(); ChangeColon c = new ChangeColon(); List<String> argList = Arrays.asList(arg); try {//from w w w . j a v a 2 s . c o m if (argList.contains("-c") && argList.get(argList.indexOf("-c") + 1) != null) { if (!new File(argList.get(argList.indexOf("-c") + 1)).exists()) { LOG.error("Default configuration is read from the system\n"); } else { configFile = new FileInputStream(argList.get(argList.indexOf("-c") + 1)); } } if (argList.contains("-t") && argList.get(argList.indexOf("-t") + 1) != null) { if (!new File(argList.get(argList.indexOf("-t") + 1)).exists()) { LOG.error("There is no test file to tag"); System.exit(1); } Configuration.testFileName = argList.get(argList.indexOf("-t") + 1); } if (argList.contains("-f") && argList.get(argList.indexOf("-f") + 1) != null) { if (!new File(argList.get(argList.indexOf("-f") + 1)).exists()) { LOG.error("The system is running in tagging mode. No training data provided"); } else { Configuration.trainFileName = argList.get(argList.indexOf("-f") + 1); } } if (argList.contains("-d") && argList.get(argList.indexOf("-d") + 1) != null) { if (new File(argList.get(argList.indexOf("-d") + 1)).exists()) { Configuration.modelDir = argList.get(argList.indexOf("-d") + 1); } else { File dir = new File(argList.get(argList.indexOf("-d") + 1)); dir.mkdirs(); Configuration.modelDir = dir.getAbsolutePath(); } } // load a properties file initNERModel(); } catch (IOException ex) { ex.printStackTrace(); } try { setModelDir(); File outputtmpFile = new File(modelDirectory, "result.tmp"); File outputFile = null; if (argList.contains("-o") && argList.get(argList.indexOf("-o") + 1) != null) { outputFile = new File(argList.get(argList.indexOf("-o") + 1)); } else { LOG.error("The directory for this output file does not exist. Output file " + "will be found in the current directury under folder \"output\""); outputFile = new File(modelDirectory, "result.tsv"); } if (Configuration.mode.equals("ft") && (Configuration.trainFileName == null || Configuration.testFileName == null)) { LOG.error(usage); System.exit(1); } if (Configuration.mode.equals("f") && Configuration.trainFileName == null) { LOG.error(usage); System.exit(1); } if (Configuration.mode.equals("t") && Configuration.testFileName == null) { LOG.error(usage); System.exit(1); } if (Configuration.mode.equals("f") && Configuration.trainFileName != null) { c.normalize(Configuration.trainFileName, Configuration.trainFileName + ".normalized"); System.out.println("Start model generation"); writeModel(new File(Configuration.trainFileName + ".normalized"), modelDirectory); System.out.println("Start model generation -- done"); System.out.println("Start training"); trainModel(modelDirectory); System.out.println("Start training ---done"); } else if (Configuration.mode.equals("ft") && Configuration.trainFileName != null && Configuration.testFileName != null) { c.normalize(Configuration.trainFileName, Configuration.trainFileName + ".normalized"); c.normalize(Configuration.testFileName, Configuration.testFileName + ".normalized"); System.out.println("Start model generation"); writeModel(new File(Configuration.trainFileName + ".normalized"), modelDirectory); System.out.println("Start model generation -- done"); System.out.println("Start training"); trainModel(modelDirectory); System.out.println("Start training ---done"); System.out.println("Start tagging"); classifyTestFile(modelDirectory, new File(Configuration.testFileName + ".normalized"), outputtmpFile, null, null); System.out.println("Start tagging ---done"); // re-normalized the colon changed text c.deNormalize(outputtmpFile.getAbsolutePath(), outputFile.getAbsolutePath()); } else { c.normalize(Configuration.testFileName, Configuration.testFileName + ".normalized"); System.out.println("Start tagging"); classifyTestFile(modelDirectory, new File(Configuration.testFileName + ".normalized"), outputtmpFile, null, null); // re-normalized the colon changed text c.deNormalize(outputtmpFile.getAbsolutePath(), outputFile.getAbsolutePath()); System.out.println("Start tagging ---done"); } long now = System.currentTimeMillis(); UIMAFramework.getLogger().log(Level.INFO, "Time: " + (now - start) + "ms"); } catch (Exception e) { LOG.error(usage); e.printStackTrace(); } long endTime = System.currentTimeMillis(); long totalTime = endTime - startTime; System.out.println("NER tarin/test done in " + totalTime / 1000 + " seconds"); }
From source file:main.StratioENEI.java
/** * @param args the command line arguments *///from ww w.j a va2 s. c o m public static void main(String[] args) throws IOException { // TODO code application logic here float[][] custos; List<String> ordem; List<String> cidades; float custoTotal; int nCidades = Integer.parseInt(args[0]); String cidadeInicial = args[1]; String cidadeFinal = args[2]; for (Mode m : Mode.values()) { custos = new float[nCidades][nCidades]; cidades = new ArrayList<>(nCidades); ordem = new ArrayList<>(nCidades); readFromCSV(m.file, custos, cidades); custoTotal = calculate(custos, cidades.indexOf(cidadeInicial), cidades.indexOf(cidadeFinal), ordem); printRota(custoTotal, cidadeInicial, cidadeFinal, cidades, ordem, m); } }
From source file:hydrograph.server.execution.tracking.client.main.HydrographMain.java
/** * The main method.//from w ww . j a va 2 s . c o m * * @param args * the arguments * @throws Exception * the exception */ public static void main(String[] args) throws Exception { HydrographMain hydrographMain = new HydrographMain(); final Timer timer = new Timer(); final CountDownLatch latch = new CountDownLatch(1); try { Session session = null; boolean isExecutionTracking = false; String[] argsList = args; List<String> argumentList = new ArrayList<String>(Arrays.asList(args)); final String jobId = hydrographMain.getJobId(argumentList); getLogLevel(argumentList).ifPresent(x -> { if (!x.equalsIgnoreCase(String.valueOf(logger.getLevel()))) { setLoglevel(x); } else { Optional.empty(); } }); logger.info("Argument List: " + argumentList.toString()); String trackingClientSocketPort = hydrographMain.getTrackingClientSocketPort(argumentList); if (argumentList.contains(Constants.IS_TRACKING_ENABLE)) { int index = argumentList.indexOf(Constants.IS_TRACKING_ENABLE); isExecutionTracking = Boolean.valueOf(argsList[index + 1]); argumentList = removeItemFromIndex(index, argumentList); } if (argumentList.contains(Constants.TRACKING_CLIENT_SOCKET_PORT)) { int index = argumentList.indexOf(Constants.TRACKING_CLIENT_SOCKET_PORT); argumentList = removeItemFromIndex(index, argumentList); } argsList = argumentList.toArray(new String[argumentList.size()]); logger.debug("Execution tracking enabled - " + isExecutionTracking); logger.info("Tracking Client Port: " + trackingClientSocketPort); /** * Start new thread to run job */ final HydrographService execution = new HydrographService(); FutureTask task = hydrographMain.executeGraph(latch, jobId, argsList, execution, isExecutionTracking); hydrographMain.executorService = Executors.newSingleThreadExecutor(); hydrographMain.executorService.submit(task); if (isExecutionTracking) { //If tracking is enabled, start to post execution tracking status. final HydrographEngineCommunicatorSocket socket = new HydrographEngineCommunicatorSocket(execution); session = hydrographMain.connectToServer(socket, jobId, trackingClientSocketPort); hydrographMain.sendExecutionTrackingStatus(latch, session, jobId, timer, execution, socket); } //waiting for execute graph thread task.get(); } catch (Exception exp) { logger.info("Getting exception from HydrographMain"); throw new RuntimeException(exp); } finally { //cleanup threads --> executor thread and timer thread logger.info("HydrographMain releasing resources"); if (!hydrographMain.executorService.isShutdown() && !hydrographMain.executorService.isTerminated()) { hydrographMain.executorService.shutdown(); } timer.cancel(); } }
From source file:com.cws.esolutions.core.main.NetworkUtility.java
public static final void main(final String[] args) { final String methodName = NetworkUtility.CNAME + "#main(final String[] args)"; if (DEBUG) {/*from w w w . j a v a 2s . c o m*/ DEBUGGER.debug(methodName); DEBUGGER.debug("Value: {}", (Object) args); } if (args.length == 0) { new HelpFormatter().printHelp(NetworkUtility.CNAME, options, true); return; } try { CommandLineParser parser = new PosixParser(); CommandLine commandLine = parser.parse(options, args); if (DEBUG) { DEBUGGER.debug("CommandLineParser parser: {}", parser); DEBUGGER.debug("CommandLine commandLine: {}", commandLine); } String coreConfiguration = (StringUtils.isBlank(System.getProperty("coreConfigFile"))) ? NetworkUtility.CORE_SVC_CONFIG : System.getProperty("coreConfigFile"); String securityConfiguration = (StringUtils.isBlank(System.getProperty("secConfigFile"))) ? NetworkUtility.CORE_LOG_CONFIG : System.getProperty("secConfigFile"); String coreLogging = (StringUtils.isBlank(System.getProperty("coreLogConfig"))) ? NetworkUtility.SEC_SVC_CONFIG : System.getProperty("coreLogConfig"); String securityLogging = (StringUtils.isBlank(System.getProperty("secLogConfig"))) ? NetworkUtility.SEC_LOG_CONFIG : System.getProperty("secLogConfig"); if (DEBUG) { DEBUGGER.debug("String coreConfiguration: {}", coreConfiguration); DEBUGGER.debug("String securityConfiguration: {}", securityConfiguration); DEBUGGER.debug("String coreLogging: {}", coreLogging); DEBUGGER.debug("String securityLogging: {}", securityLogging); } SecurityServiceInitializer.initializeService(securityConfiguration, securityLogging, false); CoreServiceInitializer.initializeService(coreConfiguration, coreLogging, false, true); final CoreConfigurationData coreConfigData = NetworkUtility.appBean.getConfigData(); final SecurityConfigurationData secConfigData = NetworkUtility.secBean.getConfigData(); if (DEBUG) { DEBUGGER.debug("CoreConfigurationData coreConfigData: {}", coreConfigData); DEBUGGER.debug("SecurityConfigurationData secConfig: {}", secConfigData); } if (commandLine.hasOption("ssh")) { if (!(commandLine.hasOption("hostname")) || (!(StringUtils.isBlank(commandLine.getOptionValue("hostname"))))) { throw new CoreServiceException("No target host was provided. Unable to process request."); } else if (!(commandLine.hasOption("username")) && (StringUtils.isBlank(commandLine.getOptionValue("username")))) { throw new CoreServiceException("No username was provided. Using default."); } // NetworkUtils.executeSshConnection(commandLine.getOptionValue("hostname"), commandList); } else if (commandLine.hasOption("http")) { if (!(commandLine.hasOption("hostname")) || (!(StringUtils.isBlank(commandLine.getOptionValue("hostname"))))) { throw new CoreServiceException("No target host was provided. Unable to process request."); } else if (!(commandLine.hasOption("method")) && (StringUtils.isBlank(commandLine.getOptionValue("method")))) { throw new CoreServiceException("No HTTP method was provided. Unable to process request."); } NetworkUtils.executeHttpConnection(new URL(commandLine.getOptionValue("hostname")), commandLine.getOptionValue("method")); } else if (commandLine.hasOption("copyFiles")) { if (!(commandLine.hasOption("hostname")) || (StringUtils.isBlank(commandLine.getOptionValue("hostname")))) { throw new CoreServiceException("No target host was provided. Unable to process request."); } else if (!(commandLine.hasOption("sourceFile")) && (StringUtils.isBlank(commandLine.getOptionValue("sourceFile")))) { throw new CoreServiceException("No source file(s) were provided. Unable to process request."); } else if (!(commandLine.hasOption("targetFile")) && (StringUtils.isBlank(commandLine.getOptionValue("targetFile")))) { throw new CoreServiceException("No target file(s) were provided. Unable to process request."); } List<String> filesToTransfer = new ArrayList<String>( Arrays.asList(commandLine.getOptionValues("sourceFile"))); List<String> filesToCreate = new ArrayList<String>( Arrays.asList(commandLine.getOptionValues("targetFile"))); if (DEBUG) { DEBUGGER.debug("List<String> filesToTransfer: {}", filesToTransfer); DEBUGGER.debug("List<String> filesToCreate: {}", filesToCreate); } if (commandLine.hasOption("withSSH")) { for (String fileName : filesToTransfer) { if (DEBUG) { DEBUGGER.debug("String fileName: {}", fileName); } NetworkUtils.executeSftpTransfer(fileName, filesToCreate.get(filesToTransfer.indexOf(fileName)), commandLine.getOptionValue("hostname"), FileUtils.getFile(fileName).exists()); } } else if (commandLine.hasOption("withFTP")) { if ((commandLine.hasOption("withSSL")) && (Boolean.valueOf(commandLine.getOptionValue("withSSL")))) { // ftp/s } // NetworkUtils.executeFtpConnection(sourceFile, targetFile, targetHost, isUpload); } } } catch (ParseException px) { ERROR_RECORDER.error(px.getMessage(), px); System.err.println("An error occurred during processing: " + px.getMessage()); } catch (SecurityException sx) { ERROR_RECORDER.error(sx.getMessage(), sx); System.err.println("An error occurred during processing: " + sx.getMessage()); } catch (SecurityServiceException ssx) { ERROR_RECORDER.error(ssx.getMessage(), ssx); System.err.println("An error occurred during processing: " + ssx.getMessage()); } catch (CoreServiceException csx) { ERROR_RECORDER.error(csx.getMessage(), csx); System.err.println("An error occurred during processing: " + csx.getMessage()); } catch (MalformedURLException mux) { ERROR_RECORDER.error(mux.getMessage(), mux); System.err.println("An error occurred during processing: " + mux.getMessage()); } }
From source file:com.aerospike.load.AerospikeLoad.java
public static void main(String[] args) throws IOException { Thread statPrinter = new Thread(new PrintStat(counters)); try {/*from ww w. ja v a 2 s . com*/ log.info("Aerospike loader started"); Options options = new Options(); options.addOption("h", "host", true, "Server hostname (default: localhost)"); options.addOption("p", "port", true, "Server port (default: 3000)"); options.addOption("n", "namespace", true, "Namespace (default: test)"); options.addOption("s", "set", true, "Set name. (default: null)"); options.addOption("c", "config", true, "Column definition file name"); options.addOption("wt", "write-threads", true, "Number of writer threads (default: Number of cores * 5)"); options.addOption("rt", "read-threads", true, "Number of reader threads (default: Number of cores * 1)"); options.addOption("l", "rw-throttle", true, "Throttling of reader to writer(default: 10k) "); options.addOption("tt", "transaction-timeout", true, "write transaction timeout in miliseconds(default: No timeout)"); options.addOption("et", "expiration-time", true, "Expiration time of records in seconds (default: never expire)"); options.addOption("T", "timezone", true, "Timezone of source where data dump is taken (default: local timezone)"); options.addOption("ec", "abort-error-count", true, "Error count to abort (default: 0)"); options.addOption("wa", "write-action", true, "Write action if key already exists (default: update)"); options.addOption("v", "verbose", false, "Logging all"); options.addOption("u", "usage", false, "Print usage."); CommandLineParser parser = new PosixParser(); CommandLine cl = parser.parse(options, args, false); if (args.length == 0 || cl.hasOption("u")) { logUsage(options); return; } if (cl.hasOption("l")) { rwThrottle = Integer.parseInt(cl.getOptionValue("l")); } else { rwThrottle = Constants.READLOAD; } // Get all command line options params = Utils.parseParameters(cl); //Get client instance AerospikeClient client = new AerospikeClient(params.host, params.port); if (!client.isConnected()) { log.error("Client is not able to connect:" + params.host + ":" + params.port); return; } if (params.verbose) { log.setLevel(Level.DEBUG); } // Get available processors to calculate default number of threads int cpus = Runtime.getRuntime().availableProcessors(); nWriterThreads = cpus * scaleFactor; nReaderThreads = cpus; // Get writer thread count if (cl.hasOption("wt")) { nWriterThreads = Integer.parseInt(cl.getOptionValue("wt")); nWriterThreads = (nWriterThreads > 0 ? (nWriterThreads > Constants.MAX_THREADS ? Constants.MAX_THREADS : nWriterThreads) : 1); log.debug("Using writer Threads: " + nWriterThreads); } writerPool = Executors.newFixedThreadPool(nWriterThreads); // Get reader thread count if (cl.hasOption("rt")) { nReaderThreads = Integer.parseInt(cl.getOptionValue("rt")); nReaderThreads = (nReaderThreads > 0 ? (nReaderThreads > Constants.MAX_THREADS ? Constants.MAX_THREADS : nReaderThreads) : 1); log.debug("Using reader Threads: " + nReaderThreads); } String columnDefinitionFileName = cl.getOptionValue("c", null); log.debug("Column definition files/directory: " + columnDefinitionFileName); if (columnDefinitionFileName == null) { log.error("Column definition files/directory not specified. use -c <file name>"); return; } File columnDefinitionFile = new File(columnDefinitionFileName); if (!columnDefinitionFile.exists()) { log.error("Column definition files/directory does not exist: " + Utils.getFileName(columnDefinitionFileName)); return; } // Get data file list String[] files = cl.getArgs(); if (files.length == 0) { log.error("No data file Specified: add <file/dir name> to end of the command "); return; } List<String> allFileNames = new ArrayList<String>(); allFileNames = Utils.getFileNames(files); if (allFileNames.size() == 0) { log.error("Given datafiles/directory does not exist"); return; } for (int i = 0; i < allFileNames.size(); i++) { log.debug("File names:" + Utils.getFileName(allFileNames.get(i))); File file = new File(allFileNames.get(i)); counters.write.recordTotal = counters.write.recordTotal + file.length(); } //remove column definition file from list allFileNames.remove(columnDefinitionFileName); log.info("Number of data files:" + allFileNames.size()); /** * Process column definition file to get meta data and bin mapping. */ metadataColumnDefs = new ArrayList<ColumnDefinition>(); binColumnDefs = new ArrayList<ColumnDefinition>(); metadataConfigs = new HashMap<String, String>(); if (Parser.processJSONColumnDefinitions(columnDefinitionFile, metadataConfigs, metadataColumnDefs, binColumnDefs, params)) { log.info("Config file processed."); } else { throw new Exception("Config file parsing Error"); } // Add metadata of config to parameters String metadata; if ((metadata = metadataConfigs.get(Constants.INPUT_TYPE)) != null) { params.fileType = metadata; if (params.fileType.equals(Constants.CSV_FILE)) { // Version check metadata = metadataConfigs.get(Constants.VERSION); String[] vNumber = metadata.split("\\."); int v1 = Integer.parseInt(vNumber[0]); int v2 = Integer.parseInt(vNumber[1]); if ((v1 <= Constants.MajorV) && (v2 <= Constants.MinorV)) { log.debug("Config version used:" + metadata); } else throw new Exception("\"" + Constants.VERSION + ":" + metadata + "\" is not Supported"); // Set delimiter if ((metadata = metadataConfigs.get(Constants.DELIMITER)) != null && metadata.length() == 1) { params.delimiter = metadata.charAt(0); } else { log.warn("\"" + Constants.DELIMITER + ":" + metadata + "\" is not properly specified in config file. Default is ','"); } if ((metadata = metadataConfigs.get(Constants.IGNORE_FIRST_LINE)) != null) { params.ignoreFirstLine = metadata.equals("true"); } else { log.warn("\"" + Constants.IGNORE_FIRST_LINE + ":" + metadata + "\" is not properly specified in config file. Default is false"); } if ((metadata = metadataConfigs.get(Constants.COLUMNS)) != null) { counters.write.colTotal = Integer.parseInt(metadata); } else { throw new Exception("\"" + Constants.COLUMNS + ":" + metadata + "\" is not properly specified in config file"); } } else { throw new Exception("\"" + params.fileType + "\" is not supported in config file"); } } else { throw new Exception("\"" + Constants.INPUT_TYPE + "\" is not specified in config file"); } // add config input to column definitions if (params.fileType.equals(Constants.CSV_FILE)) { List<String> binName = null; if (params.ignoreFirstLine) { String line; BufferedReader br = new BufferedReader( new InputStreamReader(new FileInputStream(allFileNames.get(0)), "UTF8")); if ((line = br.readLine()) != null) { binName = Parser.getCSVRawColumns(line, params.delimiter); br.close(); if (binName.size() != counters.write.colTotal) { throw new Exception("Number of column in config file and datafile are mismatch." + " Datafile: " + Utils.getFileName(allFileNames.get(0)) + " Configfile: " + Utils.getFileName(columnDefinitionFileName)); } } } //update columndefs for metadata for (int i = 0; i < metadataColumnDefs.size(); i++) { if (metadataColumnDefs.get(i).staticValue) { } else { if (metadataColumnDefs.get(i).binValuePos < 0) { if (metadataColumnDefs.get(i).columnName == null) { if (metadataColumnDefs.get(i).jsonPath == null) { log.error("dynamic metadata having improper info" + metadataColumnDefs.toString()); //TODO } else { //TODO check for json_path } } else { if (params.ignoreFirstLine) { if (binName.indexOf(metadataColumnDefs.get(i).binValueHeader) != -1) { metadataColumnDefs.get(i).binValuePos = binName .indexOf(metadataColumnDefs.get(i).binValueHeader); } else { throw new Exception("binName missing in data file:" + metadataColumnDefs.get(i).binValueHeader); } } } } else { if (params.ignoreFirstLine) metadataColumnDefs.get(i).binValueHeader = binName .get(metadataColumnDefs.get(i).binValuePos); } } if ((!metadataColumnDefs.get(i).staticValue) && (metadataColumnDefs.get(i).binValuePos < 0)) { throw new Exception("Information for bin mapping is missing in config file:" + metadataColumnDefs.get(i)); } if (metadataColumnDefs.get(i).srcType == null) { throw new Exception( "Source data type is not properly mentioned:" + metadataColumnDefs.get(i)); } if (metadataColumnDefs.get(i).binNameHeader == Constants.SET && !metadataColumnDefs.get(i).srcType.equals(SrcColumnType.STRING)) { throw new Exception("Set name should be string type:" + metadataColumnDefs.get(i)); } if (metadataColumnDefs.get(i).binNameHeader.equalsIgnoreCase(Constants.SET) && params.set != null) { throw new Exception( "Set name is given both in config file and commandline. Provide only once."); } } //update columndefs for bins for (int i = 0; i < binColumnDefs.size(); i++) { if (binColumnDefs.get(i).staticName) { } else { if (binColumnDefs.get(i).binNamePos < 0) { if (binColumnDefs.get(i).columnName == null) { if (binColumnDefs.get(i).jsonPath == null) { log.error("dynamic bin having improper info"); //TODO } else { //TODO check for json_path } } else { if (params.ignoreFirstLine) { if (binName.indexOf(binColumnDefs.get(i).binNameHeader) != -1) { binColumnDefs.get(i).binNamePos = binName .indexOf(binColumnDefs.get(i).binNameHeader); } else { throw new Exception("binName missing in data file:" + binColumnDefs.get(i).binNameHeader); } } } } else { if (params.ignoreFirstLine) binColumnDefs.get(i).binNameHeader = binName.get(binColumnDefs.get(i).binNamePos); } } if (binColumnDefs.get(i).staticValue) { } else { if (binColumnDefs.get(i).binValuePos < 0) { if (binColumnDefs.get(i).columnName == null) { if (binColumnDefs.get(i).jsonPath == null) { log.error("dynamic bin having improper info"); //TODO } else { //TODO check for json_path } } else { if (params.ignoreFirstLine) { if (binName.contains(binColumnDefs.get(i).binValueHeader)) { binColumnDefs.get(i).binValuePos = binName .indexOf(binColumnDefs.get(i).binValueHeader); } else if (!binColumnDefs.get(i).binValueHeader.toLowerCase() .equals(Constants.SYSTEM_TIME)) { throw new Exception("Wrong column name mentioned in config file:" + binColumnDefs.get(i).binValueHeader); } } } } else { if (params.ignoreFirstLine) binColumnDefs.get(i).binValueHeader = binName.get(binColumnDefs.get(i).binValuePos); } //check for missing entries in config file if (binColumnDefs.get(i).binValuePos < 0 && binColumnDefs.get(i).binValueHeader == null) { throw new Exception("Information missing(Value header or bin mapping) in config file:" + binColumnDefs.get(i)); } //check for proper data type in config file. if (binColumnDefs.get(i).srcType == null) { throw new Exception( "Source data type is not properly mentioned:" + binColumnDefs.get(i)); } //check for valid destination type if ((binColumnDefs.get(i).srcType.equals(SrcColumnType.TIMESTAMP) || binColumnDefs.get(i).srcType.equals(SrcColumnType.BLOB)) && binColumnDefs.get(i).dstType == null) { throw new Exception("Destination type is not mentioned: " + binColumnDefs.get(i)); } //check for encoding if (binColumnDefs.get(i).dstType != null && binColumnDefs.get(i).encoding == null) { throw new Exception( "Encoding is not given for src-dst type conversion:" + binColumnDefs.get(i)); } //check for valid encoding if (binColumnDefs.get(i).srcType.equals(SrcColumnType.BLOB) && !binColumnDefs.get(i).encoding.equals(Constants.HEX_ENCODING)) { throw new Exception("Wrong encoding for blob data:" + binColumnDefs.get(i)); } } //Check static bin name mapped to dynamic bin value if ((binColumnDefs.get(i).binNamePos == binColumnDefs.get(i).binValuePos) && (binColumnDefs.get(i).binNamePos != -1)) { throw new Exception("Static bin name mapped to dynamic bin value:" + binColumnDefs.get(i)); } //check for missing entries in config file if (binColumnDefs.get(i).binNameHeader == null && binColumnDefs.get(i).binNameHeader.length() > Constants.BIN_NAME_LENGTH) { throw new Exception("Information missing binName or large binName in config file:" + binColumnDefs.get(i)); } } } log.info(params.toString()); log.debug("MetadataConfig:" + metadataColumnDefs); log.debug("BinColumnDefs:" + binColumnDefs); // Start PrintStat thread statPrinter.start(); // Reader pool size ExecutorService readerPool = Executors.newFixedThreadPool( nReaderThreads > allFileNames.size() ? allFileNames.size() : nReaderThreads); log.info("Reader pool size : " + nReaderThreads); // Submit all tasks to writer threadpool. for (String aFile : allFileNames) { log.debug("Submitting task for: " + aFile); readerPool.submit(new AerospikeLoad(aFile, client, params)); } // Wait for reader pool to complete readerPool.shutdown(); log.info("Shutdown down reader thread pool"); while (!readerPool.isTerminated()) ; //readerPool.awaitTermination(20, TimeUnit.MINUTES); log.info("Reader thread pool terminated"); // Wait for writer pool to complete after getting all tasks from reader pool writerPool.shutdown(); log.info("Shutdown down writer thread pool"); while (!writerPool.isTerminated()) ; log.info("Writer thread pool terminated"); // Print final statistic of aerospike-loader. log.info("Final Statistics of importer: (Succesfull Writes = " + counters.write.writeCount.get() + ", " + "Errors=" + (counters.write.writeErrors.get() + counters.write.readErrors.get() + counters.write.processingErrors.get()) + "(" + (counters.write.writeErrors.get()) + "-Write," + counters.write.readErrors.get() + "-Read," + counters.write.processingErrors.get() + "-Processing)"); } catch (Exception e) { log.error(e); if (log.isDebugEnabled()) { e.printStackTrace(); } } finally { // Stop statistic printer thread. statPrinter.interrupt(); log.info("Aerospike loader completed"); } }
From source file:de.tudarmstadt.ukp.experiments.dip.wp1.documents.Step8GoldDataAggregator.java
public static void main(String[] args) throws Exception { String inputDir = args[0] + "/"; // output dir File outputDir = new File(args[1]); File turkersConfidence = new File(args[2]); if (outputDir.exists()) { outputDir.delete();//from www .ja v a 2 s. c o m } outputDir.mkdir(); List<String> annotatorsIDs = new ArrayList<>(); // for (File f : FileUtils.listFiles(new File(inputDir), new String[] { "xml" }, false)) { // QueryResultContainer queryResultContainer = QueryResultContainer // .fromXML(FileUtils.readFileToString(f, "utf-8")); // for (QueryResultContainer.SingleRankedResult rankedResults : queryResultContainer.rankedResults) { // for (QueryResultContainer.MTurkRelevanceVote relevanceVote : rankedResults.mTurkRelevanceVotes) { // if (!annotatorsIDs.contains(relevanceVote.turkID)) // annotatorsIDs.add(relevanceVote.turkID); // } // } // } HashMap<String, Integer> countVotesForATurker = new HashMap<>(); // creates temporary file with format for mace // Hashmap annotations: key is the id of a document and a sentence // Value is an array votes[] of turkers decisions: true or false (relevant or not) // the length of this array equals the number of annotators in List<String> annotatorsIDs. // If an annotator worked on the task his decision is written in the array otherwise the value is NULL // key: queryID + clueWebID + sentenceID // value: true and false annotations TreeMap<String, Annotations> annotations = new TreeMap<>(); for (File f : FileUtils.listFiles(new File(inputDir), new String[] { "xml" }, false)) { QueryResultContainer queryResultContainer = QueryResultContainer .fromXML(FileUtils.readFileToString(f, "utf-8")); System.out.println("Reading " + f.getName()); for (QueryResultContainer.SingleRankedResult rankedResults : queryResultContainer.rankedResults) { String documentID = rankedResults.clueWebID; for (QueryResultContainer.MTurkRelevanceVote relevanceVote : rankedResults.mTurkRelevanceVotes) { Integer turkerID; if (!annotatorsIDs.contains(relevanceVote.turkID)) { annotatorsIDs.add(relevanceVote.turkID); turkerID = annotatorsIDs.size() - 1; } else { turkerID = annotatorsIDs.indexOf(relevanceVote.turkID); } Integer count = countVotesForATurker.get(relevanceVote.turkID); if (count == null) { count = 0; } count++; countVotesForATurker.put(relevanceVote.turkID, count); String id; List<Integer> trueVotes; List<Integer> falseVotes; for (QueryResultContainer.SingleSentenceRelevanceVote singleSentenceRelevanceVote : relevanceVote.singleSentenceRelevanceVotes) if (!"".equals(singleSentenceRelevanceVote.sentenceID)) { id = f.getName() + "_" + documentID + "_" + singleSentenceRelevanceVote.sentenceID; Annotations turkerVotes = annotations.get(id); if (turkerVotes == null) { trueVotes = new ArrayList<>(); falseVotes = new ArrayList<>(); turkerVotes = new Annotations(trueVotes, falseVotes); } trueVotes = turkerVotes.trueAnnotations; falseVotes = turkerVotes.falseAnnotations; if ("true".equals(singleSentenceRelevanceVote.relevant)) { // votes[turkerID] = true; trueVotes.add(turkerID); } else if ("false".equals(singleSentenceRelevanceVote.relevant)) { // votes[turkerID] = false; falseVotes.add(turkerID); } else { throw new IllegalStateException("Annotation value of sentence " + singleSentenceRelevanceVote.sentenceID + " in " + rankedResults.clueWebID + " equals " + singleSentenceRelevanceVote.relevant); } try { int allVotesCount = trueVotes.size() + falseVotes.size(); if (allVotesCount > 5) { System.err.println(id + " doesn't have 5 annotators: true: " + trueVotes.size() + " false: " + falseVotes.size()); // nasty hack, we're gonna strip some data; true votes first /* we can't do that, it breaks something down the line int toRemove = allVotesCount - 5; if (trueVotes.size() >= toRemove) { trueVotes = trueVotes .subList(0, trueVotes.size() - toRemove); } else if ( falseVotes.size() >= toRemove) { falseVotes = falseVotes .subList(0, trueVotes.size() - toRemove); } */ System.err.println("Adjusted: " + id + " doesn't have 5 annotators: true: " + trueVotes.size() + " false: " + falseVotes.size()); } } catch (IllegalStateException e) { e.printStackTrace(); } turkerVotes.trueAnnotations = trueVotes; turkerVotes.falseAnnotations = falseVotes; annotations.put(id, turkerVotes); } else { throw new IllegalStateException( "Empty Sentence ID in " + f.getName() + " for turker " + turkerID); } } } } File tmp = printHashMap(annotations, annotatorsIDs.size()); String file = TEMP_DIR + "/" + tmp.getName(); MACE.main(new String[] { "--prefix", file }); //gets the keys of the documents and sentences ArrayList<String> lines = (ArrayList<String>) FileUtils.readLines(new File(file + ".prediction")); int i = 0; TreeMap<String, TreeMap<String, ArrayList<HashMap<String, String>>>> ids = new TreeMap<>(); ArrayList<HashMap<String, String>> sentences; if (lines.size() != annotations.size()) { throw new IllegalStateException( "The size of prediction file is " + lines.size() + "but expected " + annotations.size()); } for (Map.Entry entry : annotations.entrySet()) { //1001.xml_clueweb12-1905wb-13-07360_8783 String key = (String) entry.getKey(); String[] IDs = key.split("_"); if (IDs.length > 2) { String queryID = IDs[0]; String clueWebID = IDs[1]; String sentenceID = IDs[2]; TreeMap<String, ArrayList<HashMap<String, String>>> clueWebIDs = ids.get(queryID); if (clueWebIDs == null) { clueWebIDs = new TreeMap<>(); } sentences = clueWebIDs.get(clueWebID); if (sentences == null) { sentences = new ArrayList<>(); } HashMap<String, String> sentence = new HashMap<>(); sentence.put(sentenceID, lines.get(i)); sentences.add(sentence); clueWebIDs.put(clueWebID, sentences); ids.put(queryID, clueWebIDs); } else { throw new IllegalStateException("Wrong ID " + key); } i++; } for (Map.Entry entry : ids.entrySet()) { TreeMap<Integer, String> value = (TreeMap<Integer, String>) entry.getValue(); String queryID = (String) entry.getKey(); QueryResultContainer queryResultContainer = QueryResultContainer .fromXML(FileUtils.readFileToString(new File(inputDir, queryID), "utf-8")); for (QueryResultContainer.SingleRankedResult rankedResults : queryResultContainer.rankedResults) { for (Map.Entry val : value.entrySet()) { String clueWebID = (String) val.getKey(); if (clueWebID.equals(rankedResults.clueWebID)) { List<QueryResultContainer.SingleSentenceRelevanceVote> goldEstimatedLabels = new ArrayList<>(); List<QueryResultContainer.SingleSentenceRelevanceVote> turkersVotes = new ArrayList<>(); int size = 0; int hitSize = 0; String hitID = ""; for (QueryResultContainer.MTurkRelevanceVote vote : rankedResults.mTurkRelevanceVotes) { if (!hitID.equals(vote.hitID)) { hitID = vote.hitID; hitSize = vote.singleSentenceRelevanceVotes.size(); size = size + hitSize; turkersVotes.addAll(vote.singleSentenceRelevanceVotes); } else { if (vote.singleSentenceRelevanceVotes.size() != hitSize) { hitSize = vote.singleSentenceRelevanceVotes.size(); size = size + hitSize; turkersVotes.addAll(vote.singleSentenceRelevanceVotes); } } } ArrayList<HashMap<String, String>> sentenceList = (ArrayList<HashMap<String, String>>) val .getValue(); if (sentenceList.size() != turkersVotes.size()) { try { throw new IllegalStateException("Expected size of annotations is " + turkersVotes.size() + "but found " + sentenceList.size() + " for document " + rankedResults.clueWebID + " in " + queryID); } catch (IllegalStateException ex) { ex.printStackTrace(); } } for (QueryResultContainer.SingleSentenceRelevanceVote s : turkersVotes) { String valSentence = null; for (HashMap<String, String> anno : sentenceList) { if (anno.keySet().contains(s.sentenceID)) { valSentence = anno.get(s.sentenceID); } } QueryResultContainer.SingleSentenceRelevanceVote singleSentenceVote = new QueryResultContainer.SingleSentenceRelevanceVote(); singleSentenceVote.sentenceID = s.sentenceID; if (("false").equals(valSentence)) { singleSentenceVote.relevant = "false"; } else if (("true").equals(valSentence)) { singleSentenceVote.relevant = "true"; } else { throw new IllegalStateException("Annotation value of sentence " + singleSentenceVote.sentenceID + " equals " + val.getValue()); } goldEstimatedLabels.add(singleSentenceVote); } rankedResults.goldEstimatedLabels = goldEstimatedLabels; } } } File outputFile = new File(outputDir, queryID); FileUtils.writeStringToFile(outputFile, queryResultContainer.toXML(), "utf-8"); System.out.println("Finished " + outputFile); } ArrayList<String> annotators = (ArrayList<String>) FileUtils.readLines(new File(file + ".competence")); FileWriter fileWriter; StringBuilder sb = new StringBuilder(); for (int j = 0; j < annotatorsIDs.size(); j++) { String[] s = annotators.get(0).split("\t"); Float score = Float.parseFloat(s[j]); String turkerID = annotatorsIDs.get(j); System.out.println(turkerID + " " + score + " " + countVotesForATurker.get(turkerID)); sb.append(turkerID).append(" ").append(score).append(" ").append(countVotesForATurker.get(turkerID)) .append("\n"); } fileWriter = new FileWriter(turkersConfidence); fileWriter.append(sb.toString()); fileWriter.close(); }