List of usage examples for java.lang String toUpperCase
public String toUpperCase()
From source file:com.oltpbenchmark.multitenancy.MuTeBench.java
/** * @param args/* w w w. j a va2s . co m*/ * @throws Exception */ public static void main(String[] args) throws Exception { String duration = null; String scenarioFile = null; // ------------------------------------------------------------------- // INITIALIZE LOGGING // ------------------------------------------------------------------- String log4jPath = System.getProperty("log4j.configuration"); if (log4jPath != null) { org.apache.log4j.PropertyConfigurator.configure(log4jPath); } else { throw new RuntimeException("Missing log4j.properties file"); } // ------------------------------------------------------------------- // PARSE COMMAND LINE PARAMETERS // ------------------------------------------------------------------- CommandLineParser parser = new PosixParser(); XMLConfiguration pluginConfig = null; try { pluginConfig = new XMLConfiguration("config/plugin.xml"); } catch (ConfigurationException e1) { LOG.info("Plugin configuration file config/plugin.xml is missing"); e1.printStackTrace(); } pluginConfig.setExpressionEngine(new XPathExpressionEngine()); Options options = new Options(); options.addOption("s", "scenario", true, "[required] Workload scenario file"); options.addOption("a", "analysis-buckets", true, "sampling buckets for result aggregation"); options.addOption("r", "runtime", true, "maximum runtime (no events will be started after finishing runtime)"); options.addOption("v", "verbose", false, "Display Messages"); options.addOption("g", "gui", false, "Show controlling GUI"); options.addOption("h", "help", false, "Print this help"); options.addOption("o", "output", true, "Output file (default System.out)"); options.addOption("b", "baseline", true, "Output files of previous baseline run"); options.addOption(null, "histograms", false, "Print txn histograms"); options.addOption("d", "dialects-export", true, "Export benchmark SQL to a dialects file"); // parse the command line arguments CommandLine argsLine = parser.parse(options, args); if (argsLine.hasOption("h")) { printUsage(options); return; } else if (!argsLine.hasOption("scenario")) { INIT_LOG.fatal("Missing scenario description file"); System.exit(-1); } else scenarioFile = argsLine.getOptionValue("scenario"); if (argsLine.hasOption("r")) duration = argsLine.getOptionValue("r"); if (argsLine.hasOption("runtime")) duration = argsLine.getOptionValue("runtime"); // ------------------------------------------------------------------- // CREATE TENANT SCHEDULE // ------------------------------------------------------------------- INIT_LOG.info("Create schedule"); Schedule schedule = new Schedule(duration, scenarioFile); HashMap<Integer, ScheduleEvents> tenantEvents = schedule.getTenantEvents(); ArrayList<Integer> tenantList = schedule.getTenantList(); List<BenchmarkModule> benchList = new ArrayList<BenchmarkModule>(); for (int tenInd = 0; tenInd < tenantList.size(); tenInd++) { int tenantID = tenantList.get(tenInd); for (int tenEvent = 0; tenEvent < tenantEvents.get(tenantID).size(); tenEvent++) { BenchmarkSettings benchmarkSettings = (BenchmarkSettings) tenantEvents.get(tenantID) .getBenchmarkSettings(tenEvent); // update benchmark Settings benchmarkSettings.setTenantID(tenantID); // ------------------------------------------------------------------- // GET PLUGIN LIST // ------------------------------------------------------------------- String plugins = benchmarkSettings.getBenchmark(); String[] pluginList = plugins.split(","); String configFile = benchmarkSettings.getConfigFile(); XMLConfiguration xmlConfig = new XMLConfiguration(configFile); xmlConfig.setExpressionEngine(new XPathExpressionEngine()); int lastTxnId = 0; for (String plugin : pluginList) { // ---------------------------------------------------------------- // WORKLOAD CONFIGURATION // ---------------------------------------------------------------- String pluginTest = ""; pluginTest = "[@bench='" + plugin + "']"; WorkloadConfiguration wrkld = new WorkloadConfiguration(); wrkld.setTenantId(tenantID); wrkld.setBenchmarkName(setTenantIDinString(plugin, tenantID)); wrkld.setXmlConfig(xmlConfig); wrkld.setDBType(DatabaseType.get(setTenantIDinString(xmlConfig.getString("dbtype"), tenantID))); wrkld.setDBDriver(setTenantIDinString(xmlConfig.getString("driver"), tenantID)); wrkld.setDBConnection(setTenantIDinString(xmlConfig.getString("DBUrl"), tenantID)); wrkld.setDBName(setTenantIDinString(xmlConfig.getString("DBName"), tenantID)); wrkld.setDBUsername(setTenantIDinString(xmlConfig.getString("username"), tenantID)); wrkld.setDBPassword(setTenantIDinString(xmlConfig.getString("password"), tenantID)); String terminalString = setTenantIDinString(xmlConfig.getString("terminals[not(@bench)]", "0"), tenantID); int terminals = Integer.parseInt(xmlConfig.getString("terminals" + pluginTest, terminalString)); wrkld.setTerminals(terminals); int taSize = Integer.parseInt(xmlConfig.getString("taSize", "1")); if (taSize < 0) INIT_LOG.fatal("taSize must not be negative!"); wrkld.setTaSize(taSize); wrkld.setProprietaryTaSyntax(xmlConfig.getBoolean("proprietaryTaSyntax", false)); wrkld.setIsolationMode(setTenantIDinString( xmlConfig.getString("isolation", "TRANSACTION_SERIALIZABLE"), tenantID)); wrkld.setScaleFactor(Double .parseDouble(setTenantIDinString(xmlConfig.getString("scalefactor", "1.0"), tenantID))); wrkld.setRecordAbortMessages(xmlConfig.getBoolean("recordabortmessages", false)); int size = xmlConfig.configurationsAt("/works/work").size(); for (int i = 1; i < size + 1; i++) { SubnodeConfiguration work = xmlConfig.configurationAt("works/work[" + i + "]"); List<String> weight_strings; // use a workaround if there multiple workloads or // single // attributed workload if (pluginList.length > 1 || work.containsKey("weights[@bench]")) { weight_strings = get_weights(plugin, work); } else { weight_strings = work.getList("weights[not(@bench)]"); } int rate = 1; boolean rateLimited = true; boolean disabled = false; // can be "disabled", "unlimited" or a number String rate_string; rate_string = setTenantIDinString(work.getString("rate[not(@bench)]", ""), tenantID); rate_string = setTenantIDinString(work.getString("rate" + pluginTest, rate_string), tenantID); if (rate_string.equals(RATE_DISABLED)) { disabled = true; } else if (rate_string.equals(RATE_UNLIMITED)) { rateLimited = false; } else if (rate_string.isEmpty()) { LOG.fatal(String.format( "Tenant " + tenantID + ": Please specify the rate for phase %d and workload %s", i, plugin)); System.exit(-1); } else { try { rate = Integer.parseInt(rate_string); if (rate < 1) { LOG.fatal("Tenant " + tenantID + ": Rate limit must be at least 1. Use unlimited or disabled values instead."); System.exit(-1); } } catch (NumberFormatException e) { LOG.fatal(String.format( "Tenant " + tenantID + ": Rate string must be '%s', '%s' or a number", RATE_DISABLED, RATE_UNLIMITED)); System.exit(-1); } } Phase.Arrival arrival = Phase.Arrival.REGULAR; String arrive = setTenantIDinString(work.getString("@arrival", "regular"), tenantID); if (arrive.toUpperCase().equals("POISSON")) arrival = Phase.Arrival.POISSON; int activeTerminals; activeTerminals = Integer.parseInt(setTenantIDinString( work.getString("active_terminals[not(@bench)]", String.valueOf(terminals)), tenantID)); activeTerminals = Integer.parseInt(setTenantIDinString( work.getString("active_terminals" + pluginTest, String.valueOf(activeTerminals)), tenantID)); if (activeTerminals > terminals) { LOG.fatal("Tenant " + tenantID + ": Configuration error in work " + i + ": number of active terminals" + "" + "is bigger than the total number of terminals"); System.exit(-1); } wrkld.addWork(Integer.parseInt(setTenantIDinString(work.getString("/time"), tenantID)), rate, weight_strings, rateLimited, disabled, activeTerminals, arrival); } // FOR int numTxnTypes = xmlConfig .configurationsAt("transactiontypes" + pluginTest + "/transactiontype").size(); if (numTxnTypes == 0 && pluginList.length == 1) { // if it is a single workload run, <transactiontypes /> // w/o attribute is used pluginTest = "[not(@bench)]"; numTxnTypes = xmlConfig .configurationsAt("transactiontypes" + pluginTest + "/transactiontype").size(); } wrkld.setNumTxnTypes(numTxnTypes); // CHECKING INPUT PHASES int j = 0; for (Phase p : wrkld.getAllPhases()) { j++; if (p.getWeightCount() != wrkld.getNumTxnTypes()) { LOG.fatal(String.format("Tenant " + tenantID + ": Configuration files is inconsistent, phase %d contains %d weights but you defined %d transaction types", j, p.getWeightCount(), wrkld.getNumTxnTypes())); System.exit(-1); } } // FOR // Generate the dialect map wrkld.init(); assert (wrkld.getNumTxnTypes() >= 0); assert (xmlConfig != null); // ---------------------------------------------------------------- // BENCHMARK MODULE // ---------------------------------------------------------------- String classname = pluginConfig.getString("/plugin[@name='" + plugin + "']"); if (classname == null) { throw new ParseException("Plugin " + plugin + " is undefined in config/plugin.xml"); } BenchmarkModule bench = ClassUtil.newInstance(classname, new Object[] { wrkld }, new Class<?>[] { WorkloadConfiguration.class }); assert (benchList.get(0) != null); Map<String, Object> initDebug = new ListOrderedMap<String, Object>(); initDebug.put("Benchmark", String.format("%s {%s}", plugin.toUpperCase(), classname)); initDebug.put("Configuration", configFile); initDebug.put("Type", wrkld.getDBType()); initDebug.put("Driver", wrkld.getDBDriver()); initDebug.put("URL", wrkld.getDBConnection()); initDebug.put("Isolation", setTenantIDinString( xmlConfig.getString("isolation", "TRANSACTION_SERIALIZABLE [DEFAULT]"), tenantID)); initDebug.put("Scale Factor", wrkld.getScaleFactor()); INIT_LOG.info(SINGLE_LINE + "\n\n" + StringUtil.formatMaps(initDebug)); INIT_LOG.info(SINGLE_LINE); // Load TransactionTypes List<TransactionType> ttypes = new ArrayList<TransactionType>(); // Always add an INVALID type for Carlo ttypes.add(TransactionType.INVALID); int txnIdOffset = lastTxnId; for (int i = 1; i < wrkld.getNumTxnTypes() + 1; i++) { String key = "transactiontypes" + pluginTest + "/transactiontype[" + i + "]"; String txnName = setTenantIDinString(xmlConfig.getString(key + "/name"), tenantID); int txnId = i + 1; if (xmlConfig.containsKey(key + "/id")) { txnId = Integer .parseInt(setTenantIDinString(xmlConfig.getString(key + "/id"), tenantID)); } ttypes.add(bench.initTransactionType(txnName, txnId + txnIdOffset)); lastTxnId = i; } // FOR TransactionTypes tt = new TransactionTypes(ttypes); wrkld.setTransTypes(tt); if (benchmarkSettings.getBenchmarkSlaFile() != null) wrkld.setSlaFromFile(benchmarkSettings.getBenchmarkSlaFile()); LOG.debug("Tenant " + tenantID + ": Using the following transaction types: " + tt); bench.setTenantOffset(tenantEvents.get(tenantID).getTime(tenEvent)); bench.setTenantID(tenantID); bench.setBenchmarkSettings(benchmarkSettings); benchList.add(bench); } } } // create result collector ResultCollector rCollector = new ResultCollector(tenantList); // execute benchmarks in parallel ArrayList<Thread> benchThreads = new ArrayList<Thread>(); for (BenchmarkModule benchmark : benchList) { BenchmarkExecutor benchThread = new BenchmarkExecutor(benchmark, argsLine); Thread t = new Thread(benchThread); t.start(); benchThreads.add(t); benchmark.getWorkloadConfiguration().setrCollector(rCollector); } // waiting for completion of all benchmarks for (Thread t : benchThreads) { t.join(); } // print statistics int analysisBuckets = -1; if (argsLine.hasOption("analysis-buckets")) analysisBuckets = Integer.parseInt(argsLine.getOptionValue("analysis-buckets")); String output = null; if (argsLine.hasOption("o")) output = argsLine.getOptionValue("o"); String baseline = null; if (argsLine.hasOption("b")) baseline = argsLine.getOptionValue("b"); rCollector.printStatistics(output, analysisBuckets, argsLine.hasOption("histograms"), baseline); // create GUI if (argsLine.hasOption("g") && (!rCollector.getAllResults().isEmpty())) { try { Gui gui = new Gui(Integer.parseInt(argsLine.getOptionValue("analysis-buckets", "10")), rCollector, output); } catch (Exception e) { e.printStackTrace(); } } }
From source file:com.icantrap.collections.dawg.Dawg.java
public static void main(String[] args) throws IOException { Dawg dawg = Dawg.load(Dawg.class.getResourceAsStream("/twl06.dat")); InputStreamReader isr = new InputStreamReader(System.in); BufferedReader reader = new BufferedReader(isr); StopWatch stopWatch = new StopWatch(); while (true) { System.out.print("letters: "); String letters = reader.readLine(); System.out.print("pattern: "); String pattern = reader.readLine(); stopWatch.reset();/*from w w w .j ava 2 s .co m*/ stopWatch.start(); Result[] results = dawg.subwords(letters.toUpperCase(), pattern.toUpperCase()); stopWatch.stop(); if (results != null) { System.out.println(); for (Result result : results) { StringBuilder message = new StringBuilder(result.word); if (result.wildcardPositions != null) { message.append(" with wildcards at"); for (int position : result.wildcardPositions) message.append(" ").append(position); } System.out.println(message.toString()); System.out.println(); } System.out.println("Found " + results.length + " matches in " + stopWatch.getTime() + " ms."); } System.out.println(); } }
From source file:com.bytelightning.opensource.pokerface.PokerFaceApp.java
public static void main(String[] args) { if (JavaVersionAsFloat() < (1.8f - Float.MIN_VALUE)) { System.err.println("PokerFace requires at least Java v8 to run."); return;/*from w w w .j av a2 s .com*/ } // Configure the command line options parser Options options = new Options(); options.addOption("h", false, "help"); options.addOption("listen", true, "(http,https,secure,tls,ssl,CertAlias)=Address:Port for https."); options.addOption("keystore", true, "Filepath for PokerFace certificate keystore."); options.addOption("storepass", true, "The store password of the keystore."); options.addOption("keypass", true, "The key password of the keystore."); options.addOption("target", true, "Remote Target requestPattern=targetUri"); // NOTE: targetUri may contain user-info and if so will be interpreted as the alias of a cert to be presented to the remote target options.addOption("servercpu", true, "Number of cores the server should use."); options.addOption("targetcpu", true, "Number of cores the http targets should use."); options.addOption("trustany", false, "Ignore certificate identity errors from target servers."); options.addOption("files", true, "Filepath to a directory of static files."); options.addOption("config", true, "Path for XML Configuration file."); options.addOption("scripts", true, "Filepath for root scripts directory."); options.addOption("library", true, "JavaScript library to load into global context."); options.addOption("watch", false, "Dynamically watch scripts directory for changes."); options.addOption("dynamicTargetScripting", false, "WARNING! This option allows scripts to redirect requests to *any* other remote server."); CommandLine cmdLine = null; // parse the command line. try { CommandLineParser parser = new PosixParser(); cmdLine = parser.parse(options, args); if (args.length == 0 || cmdLine.hasOption('h')) { HelpFormatter formatter = new HelpFormatter(); formatter.setWidth(120); formatter.printHelp(PokerFaceApp.class.getSimpleName(), options); return; } } catch (ParseException exp) { System.err.println("Parsing failed. Reason: " + exp.getMessage()); return; } catch (Exception ex) { ex.printStackTrace(System.err); return; } XMLConfiguration config = new XMLConfiguration(); try { if (cmdLine.hasOption("config")) { Path tmp = Utils.MakePath(cmdLine.getOptionValue("config")); if (!Files.exists(tmp)) throw new FileNotFoundException("Configuration file does not exist."); if (Files.isDirectory(tmp)) throw new FileNotFoundException("'config' path is not a file."); // This is a bit of a pain, but but let's make sure we have a valid configuration file before we actually try to use it. config.setEntityResolver(new DefaultEntityResolver() { @Override public InputSource resolveEntity(String publicId, String systemId) throws SAXException { InputSource retVal = super.resolveEntity(publicId, systemId); if ((retVal == null) && (systemId != null)) { try { URL entityURL; if (systemId.endsWith("/PokerFace_v1Config.xsd")) entityURL = PokerFaceApp.class.getResource("/PokerFace_v1Config.xsd"); else entityURL = new URL(systemId); URLConnection connection = entityURL.openConnection(); connection.setUseCaches(false); InputStream stream = connection.getInputStream(); retVal = new InputSource(stream); retVal.setSystemId(entityURL.toExternalForm()); } catch (Throwable e) { return retVal; } } return retVal; } }); config.setSchemaValidation(true); config.setURL(tmp.toUri().toURL()); config.load(); if (cmdLine.hasOption("listen")) System.out.println("IGNORING 'listen' option because a configuration file was supplied."); if (cmdLine.hasOption("target")) System.out.println("IGNORING 'target' option(s) because a configuration file was supplied."); if (cmdLine.hasOption("scripts")) System.out.println("IGNORING 'scripts' option because a configuration file was supplied."); if (cmdLine.hasOption("library")) System.out.println("IGNORING 'library' option(s) because a configuration file was supplied."); } else { String[] serverStrs; String[] addr = { null }; String[] port = { null }; serverStrs = cmdLine.getOptionValues("listen"); if (serverStrs == null) throw new MissingOptionException("No listening addresses specified specified"); for (int i = 0; i < serverStrs.length; i++) { String addrStr; String alias = null; String protocol = null; Boolean https = null; int addrPos = serverStrs[i].indexOf('='); if (addrPos >= 0) { if (addrPos < 2) throw new IllegalArgumentException("Invalid http argument."); else if (addrPos + 1 >= serverStrs[i].length()) throw new IllegalArgumentException("Invalid http argument."); addrStr = serverStrs[i].substring(addrPos + 1, serverStrs[i].length()); String[] types = serverStrs[i].substring(0, addrPos).split(","); for (String type : types) { if (type.equalsIgnoreCase("http")) break; else if (type.equalsIgnoreCase("https") || type.equalsIgnoreCase("secure")) https = true; else if (type.equalsIgnoreCase("tls") || type.equalsIgnoreCase("ssl")) protocol = type.toUpperCase(); else alias = type; } } else addrStr = serverStrs[i]; ParseAddressString(addrStr, addr, port, alias != null ? 443 : 80); config.addProperty("server.listen(" + i + ")[@address]", addr[0]); config.addProperty("server.listen(" + i + ")[@port]", port[0]); if (alias != null) config.addProperty("server.listen(" + i + ")[@alias]", alias); if (protocol != null) config.addProperty("server.listen(" + i + ")[@protocol]", protocol); if (https != null) config.addProperty("server.listen(" + i + ")[@secure]", https); } String servercpu = cmdLine.getOptionValue("servercpu"); if (servercpu != null) config.setProperty("server[@cpu]", servercpu); String clientcpu = cmdLine.getOptionValue("targetcpu"); if (clientcpu != null) config.setProperty("targets[@cpu]", clientcpu); // Configure static files if (cmdLine.hasOption("files")) { Path tmp = Utils.MakePath(cmdLine.getOptionValue("files")); if (!Files.exists(tmp)) throw new FileNotFoundException("Files directory does not exist."); if (!Files.isDirectory(tmp)) throw new FileNotFoundException("'files' path is not a directory."); config.setProperty("files.rootDirectory", tmp.toAbsolutePath().toUri()); } // Configure scripting if (cmdLine.hasOption("scripts")) { Path tmp = Utils.MakePath(cmdLine.getOptionValue("scripts")); if (!Files.exists(tmp)) throw new FileNotFoundException("Scripts directory does not exist."); if (!Files.isDirectory(tmp)) throw new FileNotFoundException("'scripts' path is not a directory."); config.setProperty("scripts.rootDirectory", tmp.toAbsolutePath().toUri()); config.setProperty("scripts.dynamicWatch", cmdLine.hasOption("watch")); String[] libraries = cmdLine.getOptionValues("library"); if (libraries != null) { for (int i = 0; i < libraries.length; i++) { Path lib = Utils.MakePath(libraries[i]); if (!Files.exists(lib)) throw new FileNotFoundException( "Script library does not exist [" + libraries[i] + "]."); if (Files.isDirectory(lib)) throw new FileNotFoundException( "Script library is not a file [" + libraries[i] + "]."); config.setProperty("scripts.library(" + i + ")", lib.toAbsolutePath().toUri()); } } } else if (cmdLine.hasOption("watch")) System.out.println("IGNORING 'watch' option as no 'scripts' directory was specified."); else if (cmdLine.hasOption("library")) System.out.println("IGNORING 'library' option as no 'scripts' directory was specified."); } String keyStorePath = cmdLine.getOptionValue("keystore"); if (keyStorePath != null) config.setProperty("keystore", keyStorePath); String keypass = cmdLine.getOptionValue("keypass"); if (keypass != null) config.setProperty("keypass", keypass); String storepass = cmdLine.getOptionValue("storepass"); if (storepass != null) config.setProperty("storepass", keypass); if (cmdLine.hasOption("trustany")) config.setProperty("targets[@trustAny]", true); config.setProperty("scripts.dynamicTargetScripting", cmdLine.hasOption("dynamicTargetScripting")); String[] targetStrs = cmdLine.getOptionValues("target"); if (targetStrs != null) { for (int i = 0; i < targetStrs.length; i++) { int uriPos = targetStrs[i].indexOf('='); if (uriPos < 2) throw new IllegalArgumentException("Invalid target argument."); else if (uriPos + 1 >= targetStrs[i].length()) throw new IllegalArgumentException("Invalid target argument."); String patternStr = targetStrs[i].substring(0, uriPos); String urlStr = targetStrs[i].substring(uriPos + 1, targetStrs[i].length()); String alias; try { URL url = new URL(urlStr); alias = url.getUserInfo(); String scheme = url.getProtocol(); if ((!"http".equals(scheme)) && (!"https".equals(scheme))) throw new IllegalArgumentException("Invalid target uri scheme."); int port = url.getPort(); if (port < 0) port = url.getDefaultPort(); urlStr = scheme + "://" + url.getHost() + ":" + port + url.getPath(); String ref = url.getRef(); if (ref != null) urlStr += "#" + ref; } catch (MalformedURLException ex) { throw new IllegalArgumentException("Malformed target uri"); } config.addProperty("targets.target(" + i + ")[@pattern]", patternStr); config.addProperty("targets.target(" + i + ")[@url]", urlStr); if (alias != null) config.addProperty("targets.target(" + i + ")[@alias]", alias); } } // config.save(System.out); } catch (Throwable e) { e.printStackTrace(System.err); return; } // If we get here, we have a possibly valid configuration. try { final PokerFace p = new PokerFace(); p.config(config); if (p.start()) { PokerFace.Logger.warn("Started!"); Runtime.getRuntime().addShutdownHook(new Thread() { public void run() { try { PokerFace.Logger.warn("Initiating shutdown..."); p.stop(); PokerFace.Logger.warn("Shutdown completed!"); } catch (Throwable e) { PokerFace.Logger.error("Failed to shutdown cleanly!"); e.printStackTrace(System.err); } } }); } else { PokerFace.Logger.error("Failed to start!"); System.exit(-1); } } catch (Throwable e) { e.printStackTrace(System.err); } }
From source file:com.oltpbenchmark.DBWorkload.java
/** * @param args/*from w ww . j ava2 s . c o m*/ * @throws Exception */ public static void main(String[] args) throws Exception { // Initialize log4j String log4jPath = System.getProperty("log4j.configuration"); if (log4jPath != null) { org.apache.log4j.PropertyConfigurator.configure(log4jPath); } else { throw new RuntimeException("Missing log4j.properties file"); } // create the command line parser CommandLineParser parser = new PosixParser(); XMLConfiguration pluginConfig = null; try { pluginConfig = new XMLConfiguration("config/plugin.xml"); } catch (ConfigurationException e1) { LOG.info("Plugin configuration file config/plugin.xml is missing"); e1.printStackTrace(); } pluginConfig.setExpressionEngine(new XPathExpressionEngine()); Options options = new Options(); options.addOption("b", "bench", true, "[required] Benchmark class. Currently supported: " + pluginConfig.getList("/plugin//@name")); options.addOption("c", "config", true, "[required] Workload configuration file"); options.addOption(null, "create", true, "Initialize the database for this benchmark"); options.addOption(null, "clear", true, "Clear all records in the database for this benchmark"); options.addOption(null, "load", true, "Load data using the benchmark's data loader"); options.addOption(null, "execute", true, "Execute the benchmark workload"); options.addOption(null, "runscript", true, "Run an SQL script"); options.addOption(null, "upload", true, "Upload the result"); options.addOption("v", "verbose", false, "Display Messages"); options.addOption("h", "help", false, "Print this help"); options.addOption("s", "sample", true, "Sampling window"); options.addOption("ss", false, "Verbose Sampling per Transaction"); options.addOption("o", "output", true, "Output file (default System.out)"); options.addOption("d", "directory", true, "Base directory for the result files, default is current directory"); options.addOption("t", "timestamp", false, "Each result file is prepended with a timestamp for the beginning of the experiment"); options.addOption(null, "histograms", false, "Print txn histograms"); options.addOption(null, "dialects-export", true, "Export benchmark SQL to a dialects file"); // parse the command line arguments CommandLine argsLine = parser.parse(options, args); if (argsLine.hasOption("h")) { printUsage(options); return; } else if (argsLine.hasOption("c") == false) { LOG.error("Missing Configuration file"); printUsage(options); return; } else if (argsLine.hasOption("b") == false) { LOG.fatal("Missing Benchmark Class to load"); printUsage(options); return; } // If an output directory is used, store the information String outputDirectory = ""; if (argsLine.hasOption("d")) { outputDirectory = argsLine.getOptionValue("d"); } String timestampValue = ""; if (argsLine.hasOption("t")) { timestampValue = String.valueOf(TimeUtil.getCurrentTime().getTime()) + "_"; } // ------------------------------------------------------------------- // GET PLUGIN LIST // ------------------------------------------------------------------- String plugins = argsLine.getOptionValue("b"); String[] pluginList = plugins.split(","); List<BenchmarkModule> benchList = new ArrayList<BenchmarkModule>(); // Use this list for filtering of the output List<TransactionType> activeTXTypes = new ArrayList<TransactionType>(); String configFile = argsLine.getOptionValue("c"); XMLConfiguration xmlConfig = new XMLConfiguration(configFile); xmlConfig.setExpressionEngine(new XPathExpressionEngine()); int lastTxnId = 0; for (String plugin : pluginList) { // ---------------------------------------------------------------- // WORKLOAD CONFIGURATION // ---------------------------------------------------------------- String pluginTest = ""; pluginTest = "[@bench='" + plugin + "']"; WorkloadConfiguration wrkld = new WorkloadConfiguration(); wrkld.setBenchmarkName(plugin); wrkld.setXmlConfig(xmlConfig); wrkld.setDBType(DatabaseType.get(xmlConfig.getString("dbtype"))); wrkld.setDBDriver(xmlConfig.getString("driver")); wrkld.setDBConnection(xmlConfig.getString("DBUrl")); wrkld.setDBName(xmlConfig.getString("DBName")); wrkld.setDBUsername(xmlConfig.getString("username")); wrkld.setDBPassword(xmlConfig.getString("password")); int terminals = xmlConfig.getInt("terminals[not(@bench)]", 0); terminals = xmlConfig.getInt("terminals" + pluginTest, terminals); wrkld.setTerminals(terminals); wrkld.setIsolationMode(xmlConfig.getString("isolation", "TRANSACTION_SERIALIZABLE")); wrkld.setScaleFactor(xmlConfig.getDouble("scalefactor", 1.0)); wrkld.setRecordAbortMessages(xmlConfig.getBoolean("recordabortmessages", false)); int size = xmlConfig.configurationsAt("/works/work").size(); for (int i = 1; i < size + 1; i++) { SubnodeConfiguration work = xmlConfig.configurationAt("works/work[" + i + "]"); List<String> weight_strings; // use a workaround if there multiple workloads or single // attributed workload if (pluginList.length > 1 || work.containsKey("weights[@bench]")) { weight_strings = get_weights(plugin, work); } else { weight_strings = work.getList("weights[not(@bench)]"); } int rate = 1; boolean rateLimited = true; boolean disabled = false; // can be "disabled", "unlimited" or a number String rate_string; rate_string = work.getString("rate[not(@bench)]", ""); rate_string = work.getString("rate" + pluginTest, rate_string); if (rate_string.equals(RATE_DISABLED)) { disabled = true; } else if (rate_string.equals(RATE_UNLIMITED)) { rateLimited = false; } else if (rate_string.isEmpty()) { LOG.fatal(String.format("Please specify the rate for phase %d and workload %s", i, plugin)); System.exit(-1); } else { try { rate = Integer.parseInt(rate_string); if (rate < 1) { LOG.fatal("Rate limit must be at least 1. Use unlimited or disabled values instead."); System.exit(-1); } } catch (NumberFormatException e) { LOG.fatal(String.format("Rate string must be '%s', '%s' or a number", RATE_DISABLED, RATE_UNLIMITED)); System.exit(-1); } } Phase.Arrival arrival = Phase.Arrival.REGULAR; String arrive = work.getString("@arrival", "regular"); if (arrive.toUpperCase().equals("POISSON")) arrival = Phase.Arrival.POISSON; int activeTerminals; activeTerminals = work.getInt("active_terminals[not(@bench)]", terminals); activeTerminals = work.getInt("active_terminals" + pluginTest, activeTerminals); if (activeTerminals > terminals) { System.out.println("Configuration error in work " + i + ": number of active terminals" + "" + "is bigger than the total number of terminals"); System.exit(-1); } wrkld.addWork(work.getInt("/time"), rate, weight_strings, rateLimited, disabled, activeTerminals, arrival); } // FOR int numTxnTypes = xmlConfig.configurationsAt("transactiontypes" + pluginTest + "/transactiontype") .size(); if (numTxnTypes == 0 && pluginList.length == 1) { //if it is a single workload run, <transactiontypes /> w/o attribute is used pluginTest = "[not(@bench)]"; numTxnTypes = xmlConfig.configurationsAt("transactiontypes" + pluginTest + "/transactiontype") .size(); } wrkld.setNumTxnTypes(numTxnTypes); // CHECKING INPUT PHASES int j = 0; for (Phase p : wrkld.getAllPhases()) { j++; if (p.getWeightCount() != wrkld.getNumTxnTypes()) { LOG.fatal(String.format( "Configuration files is inconsistent, phase %d contains %d weights but you defined %d transaction types", j, p.getWeightCount(), wrkld.getNumTxnTypes())); System.exit(-1); } } // FOR // Generate the dialect map wrkld.init(); assert (wrkld.getNumTxnTypes() >= 0); assert (xmlConfig != null); // ---------------------------------------------------------------- // BENCHMARK MODULE // ---------------------------------------------------------------- String classname = pluginConfig.getString("/plugin[@name='" + plugin + "']"); if (classname == null) { throw new ParseException("Plugin " + plugin + " is undefined in config/plugin.xml"); } BenchmarkModule bench = ClassUtil.newInstance(classname, new Object[] { wrkld }, new Class<?>[] { WorkloadConfiguration.class }); assert (benchList.get(0) != null); Map<String, Object> initDebug = new ListOrderedMap<String, Object>(); initDebug.put("Benchmark", String.format("%s {%s}", plugin.toUpperCase(), classname)); initDebug.put("Configuration", configFile); initDebug.put("Type", wrkld.getDBType()); initDebug.put("Driver", wrkld.getDBDriver()); initDebug.put("URL", wrkld.getDBConnection()); initDebug.put("Isolation", xmlConfig.getString("isolation", "TRANSACTION_SERIALIZABLE [DEFAULT]")); initDebug.put("Scale Factor", wrkld.getScaleFactor()); INIT_LOG.info(SINGLE_LINE + "\n\n" + StringUtil.formatMaps(initDebug)); INIT_LOG.info(SINGLE_LINE); // Load TransactionTypes List<TransactionType> ttypes = new ArrayList<TransactionType>(); // Always add an INVALID type for Carlo ttypes.add(TransactionType.INVALID); int txnIdOffset = lastTxnId; for (int i = 1; i < wrkld.getNumTxnTypes() + 1; i++) { String key = "transactiontypes" + pluginTest + "/transactiontype[" + i + "]"; String txnName = xmlConfig.getString(key + "/name"); int txnId = i + 1; if (xmlConfig.containsKey(key + "/id")) { txnId = xmlConfig.getInt(key + "/id"); } TransactionType tmpType = bench.initTransactionType(txnName, txnId + txnIdOffset); // Keep a reference for filtering activeTXTypes.add(tmpType); // Add a reference for the active TTypes in this benchmark ttypes.add(tmpType); lastTxnId = i; } // FOR TransactionTypes tt = new TransactionTypes(ttypes); wrkld.setTransTypes(tt); LOG.debug("Using the following transaction types: " + tt); benchList.add(bench); } // Export StatementDialects if (isBooleanOptionSet(argsLine, "dialects-export")) { BenchmarkModule bench = benchList.get(0); if (bench.getStatementDialects() != null) { LOG.info("Exporting StatementDialects for " + bench); String xml = bench.getStatementDialects().export(bench.getWorkloadConfiguration().getDBType(), bench.getProcedures().values()); System.out.println(xml); System.exit(0); } throw new RuntimeException("No StatementDialects is available for " + bench); } @Deprecated boolean verbose = argsLine.hasOption("v"); // Create the Benchmark's Database if (isBooleanOptionSet(argsLine, "create")) { for (BenchmarkModule benchmark : benchList) { CREATE_LOG.info("Creating new " + benchmark.getBenchmarkName().toUpperCase() + " database..."); runCreator(benchmark, verbose); CREATE_LOG.info("Finished!"); CREATE_LOG.info(SINGLE_LINE); } } else if (CREATE_LOG.isDebugEnabled()) { CREATE_LOG.debug("Skipping creating benchmark database tables"); CREATE_LOG.info(SINGLE_LINE); } // Clear the Benchmark's Database if (isBooleanOptionSet(argsLine, "clear")) { for (BenchmarkModule benchmark : benchList) { CREATE_LOG.info("Resetting " + benchmark.getBenchmarkName().toUpperCase() + " database..."); benchmark.clearDatabase(); CREATE_LOG.info("Finished!"); CREATE_LOG.info(SINGLE_LINE); } } else if (CREATE_LOG.isDebugEnabled()) { CREATE_LOG.debug("Skipping creating benchmark database tables"); CREATE_LOG.info(SINGLE_LINE); } // Execute Loader if (isBooleanOptionSet(argsLine, "load")) { for (BenchmarkModule benchmark : benchList) { LOAD_LOG.info("Loading data into " + benchmark.getBenchmarkName().toUpperCase() + " database..."); runLoader(benchmark, verbose); LOAD_LOG.info("Finished!"); LOAD_LOG.info(SINGLE_LINE); } } else if (LOAD_LOG.isDebugEnabled()) { LOAD_LOG.debug("Skipping loading benchmark database records"); LOAD_LOG.info(SINGLE_LINE); } // Execute a Script if (argsLine.hasOption("runscript")) { for (BenchmarkModule benchmark : benchList) { String script = argsLine.getOptionValue("runscript"); SCRIPT_LOG.info("Running a SQL script: " + script); runScript(benchmark, script); SCRIPT_LOG.info("Finished!"); SCRIPT_LOG.info(SINGLE_LINE); } } // Execute Workload if (isBooleanOptionSet(argsLine, "execute")) { // Bombs away! Results r = null; try { r = runWorkload(benchList, verbose); } catch (Throwable ex) { LOG.error("Unexpected error when running benchmarks.", ex); System.exit(1); } assert (r != null); PrintStream ps = System.out; PrintStream rs = System.out; ResultUploader ru = new ResultUploader(r, xmlConfig, argsLine); if (argsLine.hasOption("o")) { // Check if directory needs to be created if (outputDirectory.length() > 0) { FileUtil.makeDirIfNotExists(outputDirectory.split("/")); } // Build the complex path String baseFile = timestampValue + argsLine.getOptionValue("o"); // Increment the filename for new results String nextName = FileUtil.getNextFilename(FileUtil.joinPath(outputDirectory, baseFile + ".res")); ps = new PrintStream(new File(nextName)); EXEC_LOG.info("Output into file: " + nextName); nextName = FileUtil.getNextFilename(FileUtil.joinPath(outputDirectory, baseFile + ".raw")); rs = new PrintStream(new File(nextName)); EXEC_LOG.info("Output Raw data into file: " + nextName); nextName = FileUtil.getNextFilename(FileUtil.joinPath(outputDirectory, baseFile + ".summary")); PrintStream ss = new PrintStream(new File(nextName)); EXEC_LOG.info("Output summary data into file: " + nextName); ru.writeSummary(ss); ss.close(); nextName = FileUtil.getNextFilename(FileUtil.joinPath(outputDirectory, baseFile + ".db.cnf")); ss = new PrintStream(new File(nextName)); EXEC_LOG.info("Output db config into file: " + nextName); ru.writeDBParameters(ss); ss.close(); nextName = FileUtil.getNextFilename(FileUtil.joinPath(outputDirectory, baseFile + ".ben.cnf")); ss = new PrintStream(new File(nextName)); EXEC_LOG.info("Output benchmark config into file: " + nextName); ru.writeBenchmarkConf(ss); ss.close(); } else if (EXEC_LOG.isDebugEnabled()) { EXEC_LOG.debug("No output file specified"); } if (argsLine.hasOption("s")) { int windowSize = Integer.parseInt(argsLine.getOptionValue("s")); EXEC_LOG.info("Grouped into Buckets of " + windowSize + " seconds"); r.writeCSV(windowSize, ps); if (isBooleanOptionSet(argsLine, "upload")) { ru.uploadResult(); } // Allow more detailed reporting by transaction to make it easier to check if (argsLine.hasOption("ss")) { for (TransactionType t : activeTXTypes) { PrintStream ts = ps; if (ts != System.out) { // Get the actual filename for the output String baseFile = timestampValue + argsLine.getOptionValue("o") + "_" + t.getName(); String prepended = outputDirectory + timestampValue; String nextName = FileUtil .getNextFilename(FileUtil.joinPath(outputDirectory, baseFile + ".res")); ts = new PrintStream(new File(nextName)); r.writeCSV(windowSize, ts, t); ts.close(); } } } } else if (EXEC_LOG.isDebugEnabled()) { EXEC_LOG.warn("No bucket size specified"); } if (argsLine.hasOption("histograms")) { EXEC_LOG.info(SINGLE_LINE); EXEC_LOG.info("Completed Transactions:\n" + r.getTransactionSuccessHistogram() + "\n"); EXEC_LOG.info("Aborted Transactions:\n" + r.getTransactionAbortHistogram() + "\n"); EXEC_LOG.info("Rejected Transactions:\n" + r.getTransactionRetryHistogram()); EXEC_LOG.info("Unexpected Errors:\n" + r.getTransactionErrorHistogram()); if (r.getTransactionAbortMessageHistogram().isEmpty() == false) EXEC_LOG.info( "User Aborts:\n" + StringUtil.formatMaps(r.getTransactionAbortMessageHistogram())); } else if (EXEC_LOG.isDebugEnabled()) { EXEC_LOG.warn("No bucket size specified"); } r.writeAllCSVAbsoluteTiming(rs); ps.close(); rs.close(); } else { EXEC_LOG.info("Skipping benchmark workload execution"); } }
From source file:com.crawler.app.run.CrawlSiteController.java
public static void main(String[] args) throws Exception { logger.info("Start...: "); /*//from w w w .j a v a 2 s . c om * if (args.length != 2) { logger.info("Needed parameters: "); * logger.info * ("\t rootFolder (it will contain intermediate crawl data)"); * logger.info("\t numberOfCralwers (number of concurrent threads)"); * return; } */ /* * crawlStorageFolder is a folder where intermediate crawl data is * stored. */ String crawlStorageFolder = "D:\\/Java\\/storage";//"/crawler4j/storage";// args[0]; /* * numberOfCrawlers shows the number of concurrent threads that should * be initiated for crawling. */ // int numberOfCrawlers = Integer.parseInt(args[1]); int numberOfCrawlers = 1; CrawlConfig config = new CrawlConfig(); config.setCrawlStorageFolder(crawlStorageFolder); /* * Be polite: Make sure that we don't send more than 1 request per * second (1000 milliseconds between requests). */ config.setPolitenessDelay(1000); // config.setFollowRedirects(false); /* * You can set the maximum crawl depth here. The default value is -1 for * unlimited depth */ config.setMaxDepthOfCrawling(1);// ( use -1 for unlimited depth ) /* * You can set the maximum number of pages to crawl. The default value * is -1 for unlimited number of pages */ config.setMaxPagesToFetch(-1);// ( use -1 for unlimited pages ) /** * Do you want crawler4j to crawl also binary data ? example: the * contents of pdf, or the metadata of images etc */ config.setIncludeBinaryContentInCrawling(false); /* * Do you need to set a proxy? If so, you can use: * config.setProxyHost("proxyserver.example.com"); * config.setProxyPort(8080); * * If your proxy also needs authentication: * config.setProxyUsername(username); config.getProxyPassword(password); */ /* * This config parameter can be used to set your crawl to be resumable * (meaning that you can resume the crawl from a previously * interrupted/crashed crawl). Note: if you enable resuming feature and * want to start a fresh crawl, you need to delete the contents of * rootFolder manually. */ config.setResumableCrawling(false); /* * Overwrite user ddagent */ config.setUserAgentString("Crawler"); /* * Instantiate the controller for this crawl. */ PageFetcher pageFetcher = new PageFetcher(config); RobotstxtConfig robotstxtConfig = new RobotstxtConfig(); // by me robotstxtConfig.setEnabled(false); RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig, pageFetcher); // by me CrawlController controller = new CrawlController(config, pageFetcher, robotstxtServer); /* * For each crawl, you need to add some seed urls. These are the first * URLs that are fetched and then the crawler starts following links * which are found in these pages */ // controller.addSeed("http://careerbuilder.vn/"); try { String tag_size = "site102"; int sizeIDXML = -1; String provinceYESNO, linkCrawlerBegin, linkCrawlerPage; int pageNumberBegin = -1, pageNumberEnd = -1, pageLoopInit = -1, pageLoop = -1; File fXmlFile = new File(pathXmlFile); DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); DocumentBuilder dBuilder = dbFactory.newDocumentBuilder(); org.w3c.dom.Document doc = dBuilder.parse(fXmlFile); org.w3c.dom.NodeList nList = doc.getElementsByTagName(tag_size); org.w3c.dom.Node nNode = nList.item(0); if (nNode.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) { org.w3c.dom.Element eElement = (org.w3c.dom.Element) nNode; sizeIDXML = Integer.parseInt(eElement.getAttribute("id")); String pageDefine = eElement.getElementsByTagName("pageDefine").item(0).getTextContent(); // read config ReadConfigPageNumberEnd(eElement); // if define then get define value to using if (!pageDefine.isEmpty() && pageDefine.toUpperCase().equals("YES")) { org.w3c.dom.NodeList nListOnewebsite = eElement.getElementsByTagName("website"); org.w3c.dom.Element eElementOnewebsite = (org.w3c.dom.Element) nListOnewebsite.item(0); linkCrawlerBegin = eElementOnewebsite.getElementsByTagName("linkCrawlerBegin").item(0) .getTextContent(); linkCrawlerPage = eElementOnewebsite.getElementsByTagName("linkCrawlerPage").item(0) .getTextContent(); int pageNumberTotal = Integer.parseInt( eElementOnewebsite.getElementsByTagName("pageNumberTotal").item(0).getTextContent()); pageNumberBegin = Integer.parseInt( eElementOnewebsite.getElementsByTagName("pageNumberBegin").item(0).getTextContent()); pageLoopInit = Integer.parseInt( eElementOnewebsite.getElementsByTagName("pageLoopInit").item(0).getTextContent()); pageLoop = Integer .parseInt(eElementOnewebsite.getElementsByTagName("pageLoop").item(0).getTextContent()); if (!linkCrawlerBegin.isEmpty()) { controller.addSeed(linkCrawlerBegin); //pageNumberEnd = getPageNumberEnd(linkCrawlerBegin); if (pageNumberTotal > 1) { int i = 0; for (; pageNumberTotal >= pageNumberBegin; pageNumberBegin++) { String convertlinkCrawlerPage = linkCrawlerPage.replace("%s", String.valueOf(pageLoopInit)); controller.addSeed(convertlinkCrawlerPage); pageLoopInit += pageLoop; i++; System.out.println(i); } } } } else { provinceYESNO = eElement.getElementsByTagName("provinceYESNO").item(0).getTextContent(); if (!provinceYESNO.isEmpty() && provinceYESNO.toUpperCase().equals("YES")) { // have sevent province org.w3c.dom.NodeList nListProvince = eElement.getElementsByTagName("province"); for (int index = 0; index < nListProvince.getLength(); index++) { org.w3c.dom.Element eElementProvince = (org.w3c.dom.Element) nListProvince.item(index); linkCrawlerBegin = eElementProvince.getElementsByTagName("linkCrawlerBegin").item(0) .getTextContent(); linkCrawlerPage = eElementProvince.getElementsByTagName("linkCrawlerPage").item(0) .getTextContent(); if (!eElementProvince.getElementsByTagName("pageNumberBegin").item(0).getTextContent() .isEmpty()) { pageNumberBegin = Integer.parseInt(eElementProvince .getElementsByTagName("pageNumberBegin").item(0).getTextContent()); } if (!eElementProvince.getElementsByTagName("pageLoopInit").item(0).getTextContent() .isEmpty()) { pageLoopInit = Integer.parseInt(eElementProvince .getElementsByTagName("pageLoopInit").item(0).getTextContent()); } if (!eElementProvince.getElementsByTagName("pageLoop").item(0).getTextContent() .isEmpty()) { pageLoop = Integer.parseInt( eElementProvince.getElementsByTagName("pageLoop").item(0).getTextContent()); } if (!linkCrawlerBegin.isEmpty()) { controller.addSeed(linkCrawlerBegin); pageNumberEnd = getPageNumberEnd(linkCrawlerBegin); if (pageNumberEnd > 1) { for (; pageNumberBegin <= pageNumberEnd; pageNumberBegin++) { String convertlinkCrawlerPage = linkCrawlerPage.replace("%s", String.valueOf(pageLoopInit)); controller.addSeed(convertlinkCrawlerPage); pageLoopInit += pageLoop; } } } } } else if (!provinceYESNO.isEmpty() && provinceYESNO.toUpperCase().equals("NO")) { // don't have sevent province org.w3c.dom.NodeList nListOnewebsite = eElement.getElementsByTagName("website"); org.w3c.dom.Element eElementOnewebsite = (org.w3c.dom.Element) nListOnewebsite.item(0); // read config of pagenumber end linkCrawlerBegin = eElementOnewebsite.getElementsByTagName("linkCrawlerBegin").item(0) .getTextContent(); linkCrawlerPage = eElementOnewebsite.getElementsByTagName("linkCrawlerPage").item(0) .getTextContent(); if (!eElementOnewebsite.getElementsByTagName("pageNumberBegin").item(0).getTextContent() .isEmpty()) { pageNumberBegin = Integer.parseInt(eElementOnewebsite .getElementsByTagName("pageNumberBegin").item(0).getTextContent()); } if (!eElementOnewebsite.getElementsByTagName("pageLoopInit").item(0).getTextContent() .isEmpty()) { pageLoopInit = Integer.parseInt(eElementOnewebsite.getElementsByTagName("pageLoopInit") .item(0).getTextContent()); } if (!eElementOnewebsite.getElementsByTagName("pageLoop").item(0).getTextContent() .isEmpty()) { pageLoop = Integer.parseInt( eElementOnewebsite.getElementsByTagName("pageLoop").item(0).getTextContent()); } if (!linkCrawlerBegin.isEmpty()) { controller.addSeed(linkCrawlerBegin); pageNumberEnd = getPageNumberEnd(linkCrawlerBegin); if (pageNumberEnd > 1) { for (; pageNumberBegin <= pageNumberEnd; pageNumberBegin++) { String convertlinkCrawlerPage = linkCrawlerPage.replace("%s", String.valueOf(pageLoopInit)); controller.addSeed(convertlinkCrawlerPage); pageLoopInit += pageLoop; } } } } } CrawlSite.tag_size = tag_size; CrawlSite.siteIDXML = sizeIDXML; controller.start(CrawlSite.class, numberOfCrawlers); } } catch (Exception ex) { System.out.print("can't read config xml, review xml file !!"); System.out.print(ex.getMessage()); } }
From source file:gov.nih.nci.ncicb.tcga.dcc.QCLiveTestDataGenerator.java
/** * Main entry point for the application. Configures the Spring context and calls the {@link QCLiveTestDataGenerator} * bean to load and generate test data for a specific archive name. * /*from www. jav a2s .c o m*/ * @param args - list of arguments to be passed to the {@link QCLiveTestDataGenerator} bean */ public static void main(final String[] args) { // Display help if no arguments are provided, otherwise parse the arguments if (args.length == 0) displayHelp(); else { try { // Parse the command line arguments final CommandLine commandLine = new GnuParser().parse(CommandLineOptionType.getOptions(), args); // If the command line instance contains the -? (--help) option display help, otherwise call the QCLiveTestDataGenerator // to process the command line arguments if (commandLine.hasOption(CommandLineOptionType.HELP.name().toLowerCase())) { displayHelp(); } else { final String archiveNameOption = CommandLineOptionType.ARCHIVE_NAME.getOptionValue().getOpt(); final String sqlScriptFileOption = CommandLineOptionType.SQL_SCRIPT_FILE.getOptionValue() .getOpt(); final String schemaOption = CommandLineOptionType.SCHEMA.getOptionValue().getOpt(); // Initialize the Spring context final ApplicationContext appCtx = new ClassPathXmlApplicationContext(APP_CONTEXT_FILE_NAME); // Retrieve the QCLiveTestDataGenerator from the Spring context final QCLiveTestDataGenerator qcLiveTestDataGenerator = (QCLiveTestDataGenerator) appCtx .getBean("qcLiveTestDataGenerator"); // Get the archive name from the command line argument(s) (if provided) and generate the test data if (commandLine.hasOption(archiveNameOption)) { qcLiveTestDataGenerator.generateTestData(commandLine.getOptionValue(archiveNameOption)); } // If the SQL script file and schema options are provided, execute the script if (commandLine.hasOption(sqlScriptFileOption)) { if (commandLine.hasOption(schemaOption)) { // Try to resolve the schema type from the provided schema name. If it cannot be resolved, throw an exception that // indicates the supported schema types final String schemaOptionValue = commandLine.getOptionValue(schemaOption); SchemaType schemaTpye = null; try { schemaTpye = SchemaType.valueOf(schemaOptionValue.toUpperCase()); } catch (IllegalArgumentException iae) { throw new ParseException("Could not resolve schema name '" + schemaOptionValue + "' to a supported schema type " + "when attempting to execute SQL script file '" + commandLine.getOptionValue(sqlScriptFileOption) + "'. " + "Supported types are '" + SchemaType.getSupportedSchemaTypes() + "'"); } qcLiveTestDataGenerator.executeSQLScriptFile(schemaTpye, new FileSystemResource(commandLine.getOptionValue(sqlScriptFileOption))); } else throw new ParseException( "Setting the -f (or -sql_script_file) option also requires the -s (or -schema) to be set."); } } } catch (ParseException pe) { System.err.println("\nParsing failed. Reason: " + pe.getMessage()); displayHelp(); } catch (IOException ioe) { logger.error(ioe.getMessage()); } catch (SQLException sqle) { logger.error(sqle.getMessage()); } } }
From source file:Gen.java
public static void main(String[] args) throws Exception { try {// w ww . j a va 2 s.co m File[] files = null; if (System.getProperty("dir") != null && !System.getProperty("dir").equals("")) { files = new File(System.getProperty("dir")).listFiles(new FilenameFilter() { public boolean accept(File dir, String name) { return name.toUpperCase().endsWith(".XML"); }; }); } else { String fileName = System.getProperty("file") != null && !System.getProperty("file").equals("") ? System.getProperty("file") : "rjmap.xml"; files = new File[] { new File(fileName) }; } log.info("files : " + Arrays.toString(files)); if (files == null || files.length == 0) { log.info("no files to parse"); System.exit(0); } boolean formatsource = true; if (System.getProperty("formatsource") != null && !System.getProperty("formatsource").equals("") && System.getProperty("formatsource").equalsIgnoreCase("false")) { formatsource = false; } GEN_ROOT = System.getProperty("outputdir"); if (GEN_ROOT == null || GEN_ROOT.equals("")) { GEN_ROOT = new File(files[0].getAbsolutePath()).getParent() + FILE_SEPARATOR + "distrib"; } GEN_ROOT = new File(GEN_ROOT).getAbsolutePath().replace('\\', '/'); if (GEN_ROOT.endsWith("/")) GEN_ROOT = GEN_ROOT.substring(0, GEN_ROOT.length() - 1); System.out.println("GEN ROOT:" + GEN_ROOT); MAPPING_JAR_NAME = System.getProperty("mappingjar") != null && !System.getProperty("mappingjar").equals("") ? System.getProperty("mappingjar") : "mapping.jar"; if (!MAPPING_JAR_NAME.endsWith(".jar")) MAPPING_JAR_NAME += ".jar"; GEN_ROOT_SRC = GEN_ROOT + FILE_SEPARATOR + "src"; GEN_ROOT_LIB = GEN_ROOT + FILE_SEPARATOR + ""; DocumentBuilderFactory domFactory = DocumentBuilderFactory.newInstance(); domFactory.setNamespaceAware(true); domFactory.setValidating(false); DocumentBuilder documentBuilder = domFactory.newDocumentBuilder(); for (int f = 0; f < files.length; ++f) { log.info("parsing file : " + files[f]); Document document = documentBuilder.parse(files[f]); Vector<Node> initNodes = new Vector<Node>(); Utils.catchNodes(Utils.catchNode(document.getDocumentElement(), "scripts"), "initScript", initNodes); for (int i = 0; i < initNodes.size(); ++i) { NamedNodeMap attrs = initNodes.elementAt(i).getAttributes(); boolean embed = attrs.getNamedItem("embed") != null && attrs.getNamedItem("embed").getNodeValue().equalsIgnoreCase("true"); StringBuffer vbuffer = new StringBuffer(); if (attrs.getNamedItem("inline") != null) { vbuffer.append(attrs.getNamedItem("inline").getNodeValue()); vbuffer.append('\n'); } else { String fname = attrs.getNamedItem("name").getNodeValue(); if (!fname.startsWith("\\") && !fname.startsWith("/") && fname.toCharArray()[1] != ':') { String path = files[f].getAbsolutePath(); path = path.substring(0, path.lastIndexOf(FILE_SEPARATOR)); fname = new File(path + FILE_SEPARATOR + fname).getCanonicalPath(); } vbuffer.append(Utils.getFileAsStringBuffer(fname)); } initScriptBuffer.append(vbuffer); if (embed) embedScriptBuffer.append(vbuffer); } Vector<Node> packageInitNodes = new Vector<Node>(); Utils.catchNodes(Utils.catchNode(document.getDocumentElement(), "scripts"), "packageScript", packageInitNodes); for (int i = 0; i < packageInitNodes.size(); ++i) { NamedNodeMap attrs = packageInitNodes.elementAt(i).getAttributes(); String packageName = attrs.getNamedItem("package").getNodeValue(); if (packageName.equals("")) packageName = "rGlobalEnv"; if (!packageName.endsWith("Function")) packageName += "Function"; if (packageEmbedScriptHashMap.get(packageName) == null) { packageEmbedScriptHashMap.put(packageName, new StringBuffer()); } StringBuffer vbuffer = packageEmbedScriptHashMap.get(packageName); // if (!packageName.equals("rGlobalEnvFunction")) { // vbuffer.append("library("+packageName.substring(0,packageName.lastIndexOf("Function"))+")\n"); // } if (attrs.getNamedItem("inline") != null) { vbuffer.append(attrs.getNamedItem("inline").getNodeValue() + "\n"); initScriptBuffer.append(attrs.getNamedItem("inline").getNodeValue() + "\n"); } else { String fname = attrs.getNamedItem("name").getNodeValue(); if (!fname.startsWith("\\") && !fname.startsWith("/") && fname.toCharArray()[1] != ':') { String path = files[f].getAbsolutePath(); path = path.substring(0, path.lastIndexOf(FILE_SEPARATOR)); fname = new File(path + FILE_SEPARATOR + fname).getCanonicalPath(); } StringBuffer fileBuffer = Utils.getFileAsStringBuffer(fname); vbuffer.append(fileBuffer); initScriptBuffer.append(fileBuffer); } } Vector<Node> functionsNodes = new Vector<Node>(); Utils.catchNodes(Utils.catchNode(document.getDocumentElement(), "functions"), "function", functionsNodes); for (int i = 0; i < functionsNodes.size(); ++i) { NamedNodeMap attrs = functionsNodes.elementAt(i).getAttributes(); String functionName = attrs.getNamedItem("name").getNodeValue(); boolean forWeb = attrs.getNamedItem("forWeb") != null && attrs.getNamedItem("forWeb").getNodeValue().equalsIgnoreCase("true"); String signature = (attrs.getNamedItem("signature") == null ? "" : attrs.getNamedItem("signature").getNodeValue() + ","); String renameTo = (attrs.getNamedItem("renameTo") == null ? null : attrs.getNamedItem("renameTo").getNodeValue()); HashMap<String, FAttributes> sigMap = Globals._functionsToPublish.get(functionName); if (sigMap == null) { sigMap = new HashMap<String, FAttributes>(); Globals._functionsToPublish.put(functionName, sigMap); if (attrs.getNamedItem("returnType") == null) { _functionsVector.add(new String[] { functionName }); } else { _functionsVector.add( new String[] { functionName, attrs.getNamedItem("returnType").getNodeValue() }); } } sigMap.put(signature, new FAttributes(renameTo, forWeb)); if (forWeb) _webPublishingEnabled = true; } if (System.getProperty("targetjdk") != null && !System.getProperty("targetjdk").equals("") && System.getProperty("targetjdk").compareTo("1.5") < 0) { if (_webPublishingEnabled || (System.getProperty("ws.r.api") != null && System.getProperty("ws.r.api").equalsIgnoreCase("true"))) { log.info("be careful, web publishing disabled beacuse target JDK<1.5"); } _webPublishingEnabled = false; } else { if (System.getProperty("ws.r.api") == null || System.getProperty("ws.r.api").equals("") || !System.getProperty("ws.r.api").equalsIgnoreCase("false")) { _webPublishingEnabled = true; } if (_webPublishingEnabled && System.getProperty("java.version").compareTo("1.5") < 0) { log.info("be careful, web publishing disabled beacuse a JDK<1.5 is in use"); _webPublishingEnabled = false; } } Vector<Node> s4Nodes = new Vector<Node>(); Utils.catchNodes(Utils.catchNode(document.getDocumentElement(), "s4classes"), "class", s4Nodes); if (s4Nodes.size() > 0) { String formalArgs = ""; String signature = ""; for (int i = 0; i < s4Nodes.size(); ++i) { NamedNodeMap attrs = s4Nodes.elementAt(i).getAttributes(); String s4Name = attrs.getNamedItem("name").getNodeValue(); formalArgs += "p" + i + (i == s4Nodes.size() - 1 ? "" : ","); signature += "'" + s4Name + "'" + (i == s4Nodes.size() - 1 ? "" : ","); } String genBeansScriptlet = "setGeneric('" + PUBLISH_S4_HEADER + "', function(" + formalArgs + ") standardGeneric('" + PUBLISH_S4_HEADER + "'));" + "setMethod('" + PUBLISH_S4_HEADER + "', signature(" + signature + ") , function(" + formalArgs + ") { })"; initScriptBuffer.append(genBeansScriptlet); _functionsVector.add(new String[] { PUBLISH_S4_HEADER, "numeric" }); } } if (!new File(GEN_ROOT_LIB).exists()) regenerateDir(GEN_ROOT_LIB); else { clean(GEN_ROOT_LIB, true); } for (int i = 0; i < rwebservicesScripts.length; ++i) DirectJNI.getInstance().getRServices().sourceFromResource(rwebservicesScripts[i]); String lastStatus = DirectJNI.getInstance().runR(new ExecutionUnit() { public void run(Rengine e) { DirectJNI.getInstance().toggleMarker(); DirectJNI.getInstance().sourceFromBuffer(initScriptBuffer.toString()); log.info(" init script status : " + DirectJNI.getInstance().cutStatusSinceMarker()); for (int i = 0; i < _functionsVector.size(); ++i) { String[] functionPair = _functionsVector.elementAt(i); log.info("dealing with : " + functionPair[0]); regenerateDir(GEN_ROOT_SRC); String createMapStr = "createMap("; boolean isGeneric = e.rniGetBoolArrayI( e.rniEval(e.rniParse("isGeneric(\"" + functionPair[0] + "\")", 1), 0))[0] == 1; log.info("is Generic : " + isGeneric); if (isGeneric) { createMapStr += functionPair[0]; } else { createMapStr += "\"" + functionPair[0] + "\""; } createMapStr += ", outputDirectory=\"" + GEN_ROOT_SRC .substring(0, GEN_ROOT_SRC.length() - "/src".length()).replace('\\', '/') + "\""; createMapStr += ", typeMode=\"robject\""; createMapStr += (functionPair.length == 1 || functionPair[1] == null || functionPair[1].trim().equals("") ? "" : ", S4DefaultTypedSig=TypedSignature(returnType=\"" + functionPair[1] + "\")"); createMapStr += ")"; log.info("------------------------------------------"); log.info("-- createMapStr=" + createMapStr); DirectJNI.getInstance().toggleMarker(); e.rniEval(e.rniParse(createMapStr, 1), 0); String createMapStatus = DirectJNI.getInstance().cutStatusSinceMarker(); log.info(" createMap status : " + createMapStatus); log.info("------------------------------------------"); deleteDir(GEN_ROOT_SRC + "/org/kchine/r/rserviceJms"); compile(GEN_ROOT_SRC); jar(GEN_ROOT_SRC, GEN_ROOT_LIB + FILE_SEPARATOR + TEMP_JARS_PREFIX + i + ".jar", null); URL url = null; try { url = new URL( "jar:file:" + (GEN_ROOT_LIB + FILE_SEPARATOR + TEMP_JARS_PREFIX + i + ".jar") .replace('\\', '/') + "!/"); } catch (Exception ex) { ex.printStackTrace(); } DirectJNI.generateMaps(url, true); } } }); log.info(lastStatus); log.info(DirectJNI._rPackageInterfacesHash); regenerateDir(GEN_ROOT_SRC); for (int i = 0; i < _functionsVector.size(); ++i) { unjar(GEN_ROOT_LIB + FILE_SEPARATOR + TEMP_JARS_PREFIX + i + ".jar", GEN_ROOT_SRC); } regenerateRPackageClass(true); generateS4BeanRef(); if (formatsource) applyJalopy(GEN_ROOT_SRC); compile(GEN_ROOT_SRC); for (String k : DirectJNI._rPackageInterfacesHash.keySet()) { Rmic rmicTask = new Rmic(); rmicTask.setProject(_project); rmicTask.setTaskName("rmic_packages"); rmicTask.setClasspath(new Path(_project, GEN_ROOT_SRC)); rmicTask.setBase(new File(GEN_ROOT_SRC)); rmicTask.setClassname(k + "ImplRemote"); rmicTask.init(); rmicTask.execute(); } // DirectJNI._rPackageInterfacesHash=new HashMap<String, // Vector<Class<?>>>(); // DirectJNI._rPackageInterfacesHash.put("org.bioconductor.packages.rGlobalEnv.rGlobalEnvFunction",new // Vector<Class<?>>()); if (_webPublishingEnabled) { jar(GEN_ROOT_SRC, GEN_ROOT_LIB + FILE_SEPARATOR + "__temp.jar", null); URL url = new URL( "jar:file:" + (GEN_ROOT_LIB + FILE_SEPARATOR + "__temp.jar").replace('\\', '/') + "!/"); ClassLoader cl = new URLClassLoader(new URL[] { url }, Globals.class.getClassLoader()); for (String className : DirectJNI._rPackageInterfacesHash.keySet()) { if (cl.loadClass(className + "Web").getDeclaredMethods().length == 0) continue; log.info("######## " + className); WsGen wsgenTask = new WsGen(); wsgenTask.setProject(_project); wsgenTask.setTaskName("wsgen"); FileSet rjb_fileSet = new FileSet(); rjb_fileSet.setProject(_project); rjb_fileSet.setDir(new File(".")); rjb_fileSet.setIncludes("RJB.jar"); DirSet src_dirSet = new DirSet(); src_dirSet.setDir(new File(GEN_ROOT_LIB + FILE_SEPARATOR + "src/")); Path classPath = new Path(_project); classPath.addFileset(rjb_fileSet); classPath.addDirset(src_dirSet); wsgenTask.setClasspath(classPath); wsgenTask.setKeep(true); wsgenTask.setDestdir(new File(GEN_ROOT_LIB + FILE_SEPARATOR + "src/")); wsgenTask.setResourcedestdir(new File(GEN_ROOT_LIB + FILE_SEPARATOR + "src/")); wsgenTask.setSei(className + "Web"); wsgenTask.init(); wsgenTask.execute(); } new File(GEN_ROOT_LIB + FILE_SEPARATOR + "__temp.jar").delete(); } embedRScripts(); HashMap<String, String> marker = new HashMap<String, String>(); marker.put("RJBMAPPINGJAR", "TRUE"); Properties props = new Properties(); props.put("PACKAGE_NAMES", PoolUtils.objectToHex(DirectJNI._packageNames)); props.put("S4BEANS_MAP", PoolUtils.objectToHex(DirectJNI._s4BeansMapping)); props.put("S4BEANS_REVERT_MAP", PoolUtils.objectToHex(DirectJNI._s4BeansMappingRevert)); props.put("FACTORIES_MAPPING", PoolUtils.objectToHex(DirectJNI._factoriesMapping)); props.put("S4BEANS_HASH", PoolUtils.objectToHex(DirectJNI._s4BeansHash)); props.put("R_PACKAGE_INTERFACES_HASH", PoolUtils.objectToHex(DirectJNI._rPackageInterfacesHash)); props.put("ABSTRACT_FACTORIES", PoolUtils.objectToHex(DirectJNI._abstractFactories)); new File(GEN_ROOT_SRC + "/" + "maps").mkdirs(); FileOutputStream fos = new FileOutputStream(GEN_ROOT_SRC + "/" + "maps/rjbmaps.xml"); props.storeToXML(fos, null); fos.close(); jar(GEN_ROOT_SRC, GEN_ROOT_LIB + FILE_SEPARATOR + MAPPING_JAR_NAME, marker); if (_webPublishingEnabled) genWeb(); DirectJNI._mappingClassLoader = null; } finally { System.exit(0); } }
From source file:com.ailk.oci.ocnosql.tools.load.mutiple.MutipleColumnImportTsv.java
/** * Main entry point.// w w w . ja va2 s .c o m * * @param args The command line parameters. * @throws Exception When running the job fails. */ public static void main(String[] args) throws Exception { long inputLineNum = 0L; long badLineNum = 0L; long outputLineNum = 0L; Configuration conf = HBaseConfiguration.create(); String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); if (otherArgs.length < 2) { usage("Wrong number of arguments: " + otherArgs.length); System.exit(-1); } // Make sure columns are specified String columns = conf.get(CommonConstants.COLUMNS); if (columns == null) { usage("No columns specified. Please specify with -D" + CommonConstants.COLUMNS + "=..."); System.exit(-1); } String seperator = conf.get(CommonConstants.SEPARATOR); if (StringUtils.isEmpty(seperator)) { conf.set(CommonConstants.SEPARATOR, CommonConstants.DEFAULT_SEPARATOR); seperator = CommonConstants.DEFAULT_SEPARATOR; } // Make sure one or more columns are specified if (columns.split(",").length < 2) { usage("One or more columns in addition to the row key are required"); System.exit(-1); } //make sure tableName and columns are upper to used by phoenix. columns = columns.toUpperCase(); String notNeedLoadColumnsStr = conf.get(CommonConstants.NOTNEEDLOADCOLUMNS); String notNeedLoadColumns = null; if (!StringUtils.isEmpty(notNeedLoadColumnsStr)) { notNeedLoadColumns = notNeedLoadColumnsStr.toUpperCase(); conf.set(CommonConstants.NOTNEEDLOADCOLUMNS, notNeedLoadColumns); } String writeTableConfigColumns = getWriteConfigColumn(columns, notNeedLoadColumns); hbaseAdmin = new HBaseAdmin(conf); String tableName = otherArgs[0].toUpperCase(); String inputPath = otherArgs[1]; String tmpOutputPath = conf.get(CommonConstants.IMPORT_TMP_OUTPUT); conf.set(CommonConstants.TABLE_NAME, tableName); conf.set(CommonConstants.COLUMNS, columns); String pathStr = conf.get(CommonConstants.HDFS_URL) + inputPath; FileSystem fs = FileSystem.get(URI.create(conf.get(CommonConstants.HDFS_URL)), conf); FileStatus[] fileStatusArr = fs.listStatus(new Path(pathStr)); if (fileStatusArr != null && fileStatusArr.length > 0) { TableConfiguration.getInstance().writeTableConfiguration(tableName, writeTableConfigColumns, seperator, conf); if (fileStatusArr[0].isFile()) { Object[] resObjs = runJob(conf, tableName, inputPath, tmpOutputPath); inputLineNum = (Long) resObjs[1]; outputLineNum = (Long) resObjs[2]; badLineNum = (Long) resObjs[3]; LOG.info("Bulkload Result={inputLine:" + inputLineNum + ",outputLine:" + outputLineNum + ",badLine:" + badLineNum + "}"); boolean result = (Boolean) resObjs[0]; if (result) { System.exit(0); } System.exit(-1); } for (FileStatus everyInputPath : fileStatusArr) { Path inputPathStr = everyInputPath.getPath(); String absoluteInputPathStr = inputPath + "/" + inputPathStr.getName(); FileStatus[] subFileStatusArr = fs .listStatus(new Path(conf.get(CommonConstants.HDFS_URL) + absoluteInputPathStr)); if (subFileStatusArr == null || subFileStatusArr.length == 0)//?job continue; Object[] resObjs = runJob(conf, tableName, absoluteInputPathStr, tmpOutputPath + "/" + inputPathStr.getName()); boolean ret = (Boolean) resObjs[0]; if (ret) { inputLineNum += (Long) resObjs[1]; outputLineNum += (Long) resObjs[2]; badLineNum += (Long) resObjs[3]; String seperatorStr = conf.get(CommonConstants.SEPARATOR); conf.set(CommonConstants.SEPARATOR, new String(Base64.decode(seperatorStr))); //?separator continue; } else { // LOG.error("Bulkload Result={inputLine:" + inputLineNum + ",outputLine:" + outputLineNum + ",badLine:" + badLineNum + "}"); System.exit(-1); } } LOG.info("Bulkload Result={inputLine:" + inputLineNum + ",outputLine:" + outputLineNum + ",badLine:" + badLineNum + "}"); } LOG.info("Bulkload Result={inputLine:" + inputLineNum + ",outputLine:" + outputLineNum + ",badLine:" + badLineNum + "}"); System.exit(0);// }
From source file:com.ailk.oci.ocnosql.tools.load.single.SingleColumnImportTsv.java
/** * Main entry point./* ww w . ja va 2s . c o m*/ * * @param args The command line parameters. * @throws Exception When running the job fails. */ public static void main(String[] args) throws Exception { Map<String, String> map = getProperty(); if (map == null || map.size() == 0) { System.err.println("Error: read conf file " + CONF_FILE + " occur error."); System.exit(0); } Configuration conf = Connection.getInstance().getConf(); String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); if (otherArgs.length < 2) { usage("Wrong number of arguments: " + otherArgs.length); System.exit(-1); } // Make sure columns are specified String columns = conf.get(CommonConstants.COLUMNS); if (columns == null) { usage("No columns specified. Please specify with -D" + CommonConstants.COLUMNS + "=..."); System.exit(-1); } String seperator = conf.get(CommonConstants.SEPARATOR); if (seperator == null) { conf.set(CommonConstants.SEPARATOR, CommonConstants.DEFAULT_SEPARATOR); seperator = CommonConstants.DEFAULT_SEPARATOR; } // Make sure one or more columns are specified if (columns.split(",").length < 2) { usage("One or more columns in addition to the row key are required"); System.exit(-1); } //make sure tableName and columns are upper to used by phoenix. columns = columns.toUpperCase(); String tableName = otherArgs[0].toUpperCase(); String inputPath = otherArgs[1]; hbaseAdmin = new HBaseAdmin(conf); String tmpOutputPath = conf.get(CommonConstants.IMPORT_TMP_OUTPUT); conf.set(CommonConstants.TABLE_NAME, tableName); conf.set(CommonConstants.COLUMNS, columns); String hdfs_url = conf.get(CommonConstants.HDFS_URL); FileSystem fs = FileSystem.get(URI.create(hdfs_url), conf); FileStatus[] fileStatusArr = fs.listStatus(new Path(hdfs_url + inputPath)); if (fileStatusArr != null && fileStatusArr.length > 0) { TableConfiguration.getInstance().writeTableConfiguration(tableName, columns, seperator, conf); if (fileStatusArr[0].isFile()) { //?? boolean result = runJob(conf, tableName, inputPath, tmpOutputPath); if (result) { System.exit(0); } System.exit(-1); } for (FileStatus everyInputPath : fileStatusArr) { //?? Path inputPathStr = everyInputPath.getPath(); String absoluteInputPathStr = inputPath + "/" + inputPathStr.getName(); FileStatus[] subFileStatusArr = fs.listStatus(new Path(hdfs_url + absoluteInputPathStr)); if (subFileStatusArr == null || subFileStatusArr.length == 0)//?job continue; boolean ret = runJob(conf, tableName, absoluteInputPathStr, tmpOutputPath + "/" + inputPathStr.getName()); if (ret) { String base64Seperator = conf.get(CommonConstants.SEPARATOR); conf.set(CommonConstants.SEPARATOR, new String(Base64.decode(base64Seperator))); //?separator continue; } else // System.exit(-1); } } System.exit(0); // }
From source file:licenseUtil.LicenseUtil.java
public static void main(String[] args) throws IOException, IncompleteLicenseObjectException { if (args.length == 0) { logger.error("Missing parameters. Use --help to get a list of the possible options."); } else if (args[0].equals("--addPomToTsv")) { if (args.length < 4) logger.error(/* w ww . j a v a 2 s. c om*/ "Missing arguments for option --addPomToTsv. Please specify <pomFileName> <licenses.stub.tsv> <currentVersion> or use the option --help for further information."); String pomFN = args[1]; String spreadSheetFN = args[2]; String currentVersion = args[3]; MavenProject project = null; try { project = Utils.readPom(new File(pomFN)); } catch (XmlPullParserException e) { logger.error("Could not parse pom file: \"" + pomFN + "\""); } LicensingList licensingList = new LicensingList(); File f = new File(spreadSheetFN); if (f.exists() && !f.isDirectory()) { licensingList.readFromSpreadsheet(spreadSheetFN); } licensingList.addMavenProject(project, currentVersion); licensingList.writeToSpreadsheet(spreadSheetFN); } else if (args[0].equals("--writeLicense3rdParty")) { if (args.length < 4) logger.error( "Missing arguments for option --writeLicense3rdParty. Please provide <licenses.enhanced.tsv> <processModule> <currentVersion> [and <targetDir>] or use the option --help for further information."); String spreadSheetFN = args[1]; String processModule = args[2]; String currentVersion = args[3]; HashMap<String, String> targetDirs = new HashMap<>(); if (args.length > 4) { File targetDir = new File(args[4]); logger.info("scan pom files in direct subdirectories of \"" + targetDir.getPath() + "\" to obtain target locations for 3rd party license files..."); File[] subdirs = targetDir.listFiles((FileFilter) DirectoryFileFilter.DIRECTORY); for (File subdir : subdirs) { File pomFile = new File(subdir.getPath() + File.separator + POM_FN); if (!pomFile.exists()) continue; MavenProject mavenProject; try { mavenProject = Utils.readPom(pomFile); } catch (Exception e) { logger.warn("Could not read from pom file: \"" + pomFile.getPath() + "\" because of " + e.getMessage()); continue; } targetDirs.put(mavenProject.getModel().getArtifactId(), subdir.getAbsolutePath()); } } LicensingList licensingList = new LicensingList(); licensingList.readFromSpreadsheet(spreadSheetFN); if (processModule.toUpperCase().equals("ALL")) { for (String module : licensingList.getNonFixedHeaders()) { try { writeLicense3rdPartyFile(module, licensingList, currentVersion, targetDirs.get(module)); } catch (NoLicenseTemplateSetException e) { logger.error("Could not write file for module \"" + module + "\". There is no template specified for \"" + e.getLicensingObject() + "\". Please add an existing template filename to the column \"" + LicensingObject.ColumnHeader.LICENSE_TEMPLATE.value() + "\" of \"" + spreadSheetFN + "\"."); } } } else { try { writeLicense3rdPartyFile(processModule, licensingList, currentVersion, targetDirs.get(processModule)); } catch (NoLicenseTemplateSetException e) { logger.error("Could not write file for module \"" + processModule + "\". There is no template specified for \"" + e.getLicensingObject() + "\". Please add an existing template filename to the column \"" + LicensingObject.ColumnHeader.LICENSE_TEMPLATE.value() + "\" of \"" + spreadSheetFN + "\"."); } } } else if (args[0].equals("--buildEffectivePom")) { Utils.writeEffectivePom(new File(args[1]), (new File(EFFECTIVE_POM_FN)).getAbsolutePath()); } else if (args[0].equals("--updateTsvWithProjectsInFolder")) { if (args.length < 4) logger.error( "Missing arguments for option --processProjectsInFolder. Please provide <superDirectory> <licenses.stub.tsv> and <currentVersion> or use the option --help for further information."); File directory = new File(args[1]); String spreadSheetFN = args[2]; String currentVersion = args[3]; LicensingList licensingList = new LicensingList(); File f = new File(spreadSheetFN); if (f.exists() && !f.isDirectory()) { licensingList.readFromSpreadsheet(spreadSheetFN); } licensingList.addAll(processProjectsInFolder(directory, currentVersion, false)); licensingList.writeToSpreadsheet(spreadSheetFN); } else if (args[0].equals("--purgeTsv")) { if (args.length < 3) logger.error( "Missing arguments for option --purgeTsv. Please provide <spreadSheetIN.tsv>, <spreadSheetOUT.tsv> and <currentVersion> or use the option --help for further information."); String spreadSheetIN = args[1]; String spreadSheetOUT = args[2]; String currentVersion = args[3]; LicensingList licensingList = new LicensingList(); licensingList.readFromSpreadsheet(spreadSheetIN); licensingList.purge(currentVersion); licensingList.writeToSpreadsheet(spreadSheetOUT); } else if (args[0].equals("--help")) { InputStream in = LicenseUtil.class.getClassLoader().getResourceAsStream(README_PATH); BufferedReader reader = new BufferedReader(new InputStreamReader(in)); String line; while ((line = reader.readLine()) != null) { System.out.println(line); } } else { logger.error("Unknown parameter: " + args[0] + ". Use --help to get a list of the possible options."); } }