List of usage examples for java.util.concurrent ExecutorService submit
Future<?> submit(Runnable task);
From source file:com.sludev.mssqlapplylog.MSSQLApplyLogMain.java
public static void main(String[] args) { CommandLineParser parser = new DefaultParser(); Options options = new Options(); // Most of the following defaults should be changed in // the --conf or "conf.properties" file String sqlURL = null;/* w w w. ja v a2s .c o m*/ String sqlUser = null; String sqlPass = null; String sqlDb = null; String sqlHost = "127.0.0.1"; String backupDirStr = null; String laterThanStr = ""; String fullBackupPathStr = null; String fullBackupPatternStr = "(?:[\\w_-]+?)(\\d+)\\.bak"; String fullBackupDatePatternStr = "yyyyMMddHHmm"; String sqlProcessUser = null; String logBackupPatternStr = "(.*)\\.trn"; String logBackupDatePatternStr = "yyyyMMddHHmmss"; boolean doFullRestore = false; Boolean useLogFileLastMode = null; Boolean monitorLogBackupDir = null; options.addOption(Option.builder().longOpt("conf").desc("Configuration file.").hasArg().build()); options.addOption(Option.builder().longOpt("laterthan").desc("'Later Than' file filter.").hasArg().build()); options.addOption(Option.builder().longOpt("restore-full") .desc("Restore the full backup before continuing.").build()); options.addOption(Option.builder().longOpt("use-lastmod") .desc("Sort/filter the log backups using their File-System 'Last Modified' date.").build()); options.addOption(Option.builder().longOpt("monitor-backup-dir") .desc("Monitor the backup directory for new log backups, and apply them.").build()); CommandLine line = null; try { try { line = parser.parse(options, args); } catch (ParseException ex) { throw new MSSQLApplyLogException(String.format("Error parsing command line.'%s'", ex.getMessage()), ex); } String confFile = null; // Process the command line arguments Iterator cmdI = line.iterator(); while (cmdI.hasNext()) { Option currOpt = (Option) cmdI.next(); String currOptName = currOpt.getLongOpt(); switch (currOptName) { case "conf": // Parse the configuration file confFile = currOpt.getValue(); break; case "laterthan": // "Later Than" file date filter laterThanStr = currOpt.getValue(); break; case "restore-full": // Do a full backup restore before restoring logs doFullRestore = true; break; case "monitor-backup-dir": // Monitor the backup directory for new logs monitorLogBackupDir = true; break; case "use-lastmod": // Use the last-modified date on Log Backup files for sorting/filtering useLogFileLastMode = true; break; } } Properties confProperties = null; if (StringUtils.isBlank(confFile) || Files.isReadable(Paths.get(confFile)) == false) { throw new MSSQLApplyLogException( "Missing or unreadable configuration file. Please specify --conf"); } else { // Process the conf.properties file confProperties = new Properties(); try { confProperties.load(Files.newBufferedReader(Paths.get(confFile))); } catch (IOException ex) { throw new MSSQLApplyLogException("Error loading properties file", ex); } sqlURL = confProperties.getProperty("sqlURL", ""); sqlUser = confProperties.getProperty("sqlUser", ""); sqlPass = confProperties.getProperty("sqlPass", ""); sqlDb = confProperties.getProperty("sqlDb", ""); sqlHost = confProperties.getProperty("sqlHost", ""); backupDirStr = confProperties.getProperty("backupDir", ""); if (StringUtils.isBlank(laterThanStr)) { laterThanStr = confProperties.getProperty("laterThan", ""); } fullBackupPathStr = confProperties.getProperty("fullBackupPath", fullBackupPathStr); fullBackupPatternStr = confProperties.getProperty("fullBackupPattern", fullBackupPatternStr); fullBackupDatePatternStr = confProperties.getProperty("fullBackupDatePattern", fullBackupDatePatternStr); sqlProcessUser = confProperties.getProperty("sqlProcessUser", ""); logBackupPatternStr = confProperties.getProperty("logBackupPattern", logBackupPatternStr); logBackupDatePatternStr = confProperties.getProperty("logBackupDatePattern", logBackupDatePatternStr); if (useLogFileLastMode == null) { String useLogFileLastModeStr = confProperties.getProperty("useLogFileLastMode", "false"); useLogFileLastMode = Boolean .valueOf(StringUtils.lowerCase(StringUtils.trim(useLogFileLastModeStr))); } if (monitorLogBackupDir == null) { String monitorBackupDirStr = confProperties.getProperty("monitorBackupDir", "false"); monitorLogBackupDir = Boolean .valueOf(StringUtils.lowerCase(StringUtils.trim(monitorBackupDirStr))); } } } catch (MSSQLApplyLogException ex) { try (StringWriter sw = new StringWriter(); PrintWriter pw = new PrintWriter(sw)) { pw.append(String.format("Error : '%s'\n\n", ex.getMessage())); HelpFormatter formatter = new HelpFormatter(); formatter.printHelp(pw, 80, "\njava -jar mssqlapplylog.jar ", "\nThe MSSQLApplyLog application can be used in a variety of options and modes.\n", options, 0, 2, " All Rights Reserved.", true); System.out.println(sw.toString()); } catch (IOException iex) { LOGGER.debug("Error processing usage", iex); } System.exit(1); } MSSQLApplyLogConfig config = MSSQLApplyLogConfig.from(backupDirStr, fullBackupPathStr, fullBackupDatePatternStr, laterThanStr, fullBackupPatternStr, logBackupPatternStr, logBackupDatePatternStr, sqlHost, sqlDb, sqlUser, sqlPass, sqlURL, sqlProcessUser, useLogFileLastMode, doFullRestore, monitorLogBackupDir); MSSQLApplyLog logProc = MSSQLApplyLog.from(config); BasicThreadFactory thFactory = new BasicThreadFactory.Builder().namingPattern("restoreThread-%d").build(); ExecutorService mainThreadExe = Executors.newSingleThreadExecutor(thFactory); Future<Integer> currRunTask = mainThreadExe.submit(logProc); mainThreadExe.shutdown(); Integer resp = 0; try { resp = currRunTask.get(); } catch (InterruptedException ex) { LOGGER.error("Application 'main' thread was interrupted", ex); } catch (ExecutionException ex) { LOGGER.error("Application 'main' thread execution error", ex); } finally { // If main leaves for any reason, shutdown all threads mainThreadExe.shutdownNow(); } System.exit(resp); }
From source file:Pong.java
public static void main(String... args) throws Exception { System.setProperty("os.max.pid.bits", "16"); Options options = new Options(); options.addOption("i", true, "Input chronicle path"); options.addOption("n", true, "Number of entries to write"); options.addOption("w", true, "Number of writer threads"); options.addOption("r", true, "Number of reader threads"); options.addOption("x", false, "Delete the output chronicle at startup"); CommandLine cmd = new DefaultParser().parse(options, args); final Path output = Paths.get(cmd.getOptionValue("o", "/tmp/__test/chr")); final long maxCount = Long.parseLong(cmd.getOptionValue("n", "10000000")); final int writerThreadCount = Integer.parseInt(cmd.getOptionValue("w", "4")); final int readerThreadCount = Integer.parseInt(cmd.getOptionValue("r", "4")); final boolean deleteOnStartup = cmd.hasOption("x"); if (deleteOnStartup) { FileUtil.removeRecursive(output); }//from w ww.jav a 2s . co m final Chronicle chr = ChronicleQueueBuilder.vanilla(output.toFile()).build(); final ExecutorService executor = Executors.newFixedThreadPool(4); final List<Future<?>> futures = new ArrayList<>(); final long totalCount = writerThreadCount * maxCount; final long t0 = System.nanoTime(); for (int i = 0; i != readerThreadCount; ++i) { final int tid = i; futures.add(executor.submit((Runnable) () -> { try { IntLongMap counts = HashIntLongMaps.newMutableMap(); ExcerptTailer tailer = chr.createTailer(); final StringBuilder sb1 = new StringBuilder(); final StringBuilder sb2 = new StringBuilder(); long count = 0; while (count != totalCount) { if (!tailer.nextIndex()) continue; final int id = tailer.readInt(); final long val = tailer.readStopBit(); final long longValue = tailer.readLong(); sb1.setLength(0); sb2.setLength(0); tailer.read8bitText(sb1); tailer.read8bitText(sb2); if (counts.addValue(id, 1) - 1 != val || longValue != 0x0badcafedeadbeefL || !StringInterner.isEqual("FooBar", sb1) || !StringInterner.isEqual("AnotherFooBar", sb2)) { System.out.println("Unexpected value " + id + ", " + val + ", " + Long.toHexString(longValue) + ", " + sb1.toString() + ", " + sb2.toString()); return; } ++count; if (count % 1_000_000 == 0) { long t1 = System.nanoTime(); System.out.println(tid + " " + (t1 - t0) / 1e6 + " ms"); } } } catch (IOException e) { e.printStackTrace(); } })); } for (Future f : futures) { f.get(); } executor.shutdownNow(); final long t1 = System.nanoTime(); System.out.println("Done. Rough time=" + (t1 - t0) / 1e6 + " ms"); }
From source file:alluxio.cli.MiniBenchmark.java
/** * @param args there are no arguments needed * @throws Exception if error occurs during tests *//*from w w w . ja va2s . com*/ public static void main(String[] args) throws Exception { if (!parseInputArgs(args)) { usage(); System.exit(-1); } if (sHelp) { usage(); System.exit(0); } CommonUtils.warmUpLoop(); for (int i = 0; i < sIterations; ++i) { final AtomicInteger count = new AtomicInteger(0); final CyclicBarrier barrier = new CyclicBarrier(sConcurrency); ExecutorService executorService = Executors.newFixedThreadPool(sConcurrency); final AtomicLong runtime = new AtomicLong(0); for (int j = 0; j < sConcurrency; ++j) { switch (sType) { case READ: executorService.submit(new Runnable() { @Override public void run() { try { readFile(barrier, runtime, count.addAndGet(1)); } catch (Exception e) { LOG.error("Failed to read file.", e); System.exit(-1); } } }); break; case WRITE: executorService.submit(new Runnable() { @Override public void run() { try { writeFile(barrier, runtime, count.addAndGet(1)); } catch (Exception e) { LOG.error("Failed to write file.", e); System.exit(-1); } } }); break; default: throw new RuntimeException("Unsupported type."); } } executorService.shutdown(); Preconditions.checkState(executorService.awaitTermination(1, TimeUnit.HOURS)); double time = runtime.get() * 1.0 / sConcurrency / Constants.SECOND_NANO; System.out.printf("Iteration: %d; Duration: %f seconds; Aggregated throughput: %f GB/second.%n", i, time, sConcurrency * 1.0 * sFileSize / time / Constants.GB); } }
From source file:com.weibo.motan.demo.client.DemoRpcClient.java
public static void main(String[] args) throws Exception { final DescriptiveStatistics stats = new SynchronizedDescriptiveStatistics(); int threads = Integer.parseInt(args[0]); DubboBenchmark.BenchmarkMessage msg = prepareArgs(); final byte[] msgBytes = msg.toByteArray(); int n = 1000000; final CountDownLatch latch = new CountDownLatch(n); ExecutorService es = Executors.newFixedThreadPool(threads); final AtomicInteger trans = new AtomicInteger(0); final AtomicInteger transOK = new AtomicInteger(0); ApplicationContext ctx = new ClassPathXmlApplicationContext( new String[] { "classpath:motan_demo_client.xml" }); MotanDemoService service = (MotanDemoService) ctx.getBean("motanDemoReferer"); long start = System.currentTimeMillis(); for (int i = 0; i < n; i++) { es.submit(() -> { try { long t = System.currentTimeMillis(); DubboBenchmark.BenchmarkMessage m = testSay(service, msgBytes); t = System.currentTimeMillis() - t; stats.addValue(t);/*from ww w . j a v a 2 s.c o m*/ trans.incrementAndGet(); if (m != null && m.getField1().equals("OK")) { transOK.incrementAndGet(); } } finally { latch.countDown(); } }); } latch.await(); start = System.currentTimeMillis() - start; System.out.printf("sent requests : %d\n", n); System.out.printf("received requests : %d\n", trans.get()); System.out.printf("received requests_OK : %d\n", transOK.get()); System.out.printf("throughput (TPS) : %d\n", n * 1000 / start); System.out.printf("mean: %f\n", stats.getMean()); System.out.printf("median: %f\n", stats.getPercentile(50)); System.out.printf("max: %f\n", stats.getMax()); System.out.printf("min: %f\n", stats.getMin()); System.out.printf("99P: %f\n", stats.getPercentile(90)); }
From source file:ParallelizedMatrixProduct.java
public static void main(String args[]) throws Exception { System.setSecurityManager(new YesSecurityManager()); double[][] matrix1 = new double[MATRIX_SIZE][MATRIX_SIZE]; double[][] matrix2 = new double[MATRIX_SIZE][MATRIX_SIZE]; for (int i = 0; i < MATRIX_SIZE; ++i) for (int j = 0; j < MATRIX_SIZE; ++j) { matrix1[i][j] = Math.round(Math.random() * MATRIX_ELEMENT_MAX_VALUE); matrix2[i][j] = Math.round(Math.random() * MATRIX_ELEMENT_MAX_VALUE); }/*from ww w . j a va 2 s . c o m*/ ExecutorService exec = Executors.newFixedThreadPool(THREAD_POOL_SIZE); Future<Double>[][] futures = new Future[MATRIX_SIZE][MATRIX_SIZE]; for (int i = 0; i < MATRIX_SIZE; ++i) { for (int j = 0; j < MATRIX_SIZE; ++j) { final double[] v1 = getRow(matrix1, i); final double[] v2 = getColumn(matrix2, j); if (i % 2 == 0) { futures[i][j] = exec.submit(new Callable<Double>() { public Double call() { RPFSessionInfo.get().put("USER", "USER FOR " + Thread.currentThread().getName()); RServices rp = null; int replayCounter = NBR_REPLAY_ON_FAILURE; while (replayCounter >= 0) { try { rp = (RServices) org.kchine.rpf.ServantProviderFactory.getFactory() .getServantProvider().borrowServantProxy(); rp.putAndAssign(new RNumeric(v1), "rv1"); rp.putAndAssign(new RNumeric(v2), "rv2"); RMatrix res = ((RMatrix) rp.getObject("rv1%*%rv2")); return ((RNumeric) res.getValue()).getValue()[0]; } catch (TimeoutException e) { e.printStackTrace(); return null; } catch (RemoteException re) { re.printStackTrace(); --replayCounter; } finally { try { if (rp != null) { ServantProviderFactory.getFactory().getServantProvider() .returnServantProxy(rp); log.info("<" + Thread.currentThread().getName() + "> returned resource : " + rp.getServantName()); } } catch (Exception e) { e.printStackTrace(); } } } return null; } }); } else { futures[i][j] = exec.submit(new Callable<Double>() { public Double call() { try { return vecprod(v1, v2); } finally { log.info("<" + Thread.currentThread().getName() + "> Java task ended successfully"); } } }); } } } while (true) { if (countDone(futures) == (MATRIX_SIZE * MATRIX_SIZE)) break; try { Thread.sleep(20); } catch (Exception e) { } } log.info(" done -- product matrix -->"); Double[][] matrix1_x_matrix2 = new Double[MATRIX_SIZE][MATRIX_SIZE]; for (int i = 0; i < MATRIX_SIZE; ++i) for (int j = 0; j < MATRIX_SIZE; ++j) matrix1_x_matrix2[i][j] = futures[i][j].get(); System.out.println(showMatrix(matrix1, "M1")); System.out.println(showMatrix(matrix2, "M2")); System.out.println(showMatrix(matrix1_x_matrix2, "M1 x M2")); System.exit(0); }
From source file:com.amazonaws.services.kinesis.samples.datavis.HttpReferrerStreamWriter.java
/** * Start a number of threads and send randomly generated {@link HttpReferrerPair}s to a Kinesis Stream until the * program is terminated.//from w ww .ja v a 2 s . c o m * * @param args Expecting 3 arguments: A numeric value indicating the number of threads to use to send * data to Kinesis and the name of the stream to send records to, and the AWS region in which these resources * exist or should be created. * @throws InterruptedException If this application is interrupted while sending records to Kinesis. */ public static void main(String[] args) throws InterruptedException { if (args.length != 3) { System.err.println("Usage: " + HttpReferrerStreamWriter.class.getSimpleName() + " <number of threads> <stream name> <region>"); System.exit(1); } int numberOfThreads = Integer.parseInt(args[0]); String streamName = args[1]; Region region = SampleUtils.parseRegion(args[2]); AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain(); ClientConfiguration clientConfig = SampleUtils.configureUserAgentForSample(new ClientConfiguration()); AmazonKinesis kinesis = new AmazonKinesisClient(credentialsProvider, clientConfig); kinesis.setRegion(region); // The more resources we declare the higher write IOPS we need on our DynamoDB table. // We write a record for each resource every interval. // If interval = 500ms, resource count = 7 we need: (1000/500 * 7) = 14 write IOPS minimum. List<String> resources = new ArrayList<>(); resources.add("/index.html"); // These are the possible referrers to use when generating pairs List<String> referrers = new ArrayList<>(); referrers.add("http://www.amazon.com"); referrers.add("http://www.google.com"); referrers.add("http://www.yahoo.com"); referrers.add("http://www.bing.com"); referrers.add("http://www.stackoverflow.com"); referrers.add("http://www.reddit.com"); HttpReferrerPairFactory pairFactory = new HttpReferrerPairFactory(resources, referrers); // Creates a stream to write to with 2 shards if it doesn't exist StreamUtils streamUtils = new StreamUtils(kinesis); streamUtils.createStreamIfNotExists(streamName, 2); LOG.info(String.format("%s stream is ready for use", streamName)); final HttpReferrerKinesisPutter putter = new HttpReferrerKinesisPutter(pairFactory, kinesis, streamName); ExecutorService es = Executors.newCachedThreadPool(); Runnable pairSender = new Runnable() { @Override public void run() { try { putter.sendPairsIndefinitely(DELAY_BETWEEN_RECORDS_IN_MILLIS, TimeUnit.MILLISECONDS); } catch (Exception ex) { LOG.warn( "Thread encountered an error while sending records. Records will no longer be put by this thread.", ex); } } }; for (int i = 0; i < numberOfThreads; i++) { es.submit(pairSender); } LOG.info(String.format("Sending pairs with a %dms delay between records with %d thread(s).", DELAY_BETWEEN_RECORDS_IN_MILLIS, numberOfThreads)); es.shutdown(); es.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS); }
From source file:com.alertlogic.aws.kinesis.test1.StreamWriter.java
/** * Start a number of threads and send randomly generated {@link HttpReferrerPair}s to a Kinesis Stream until the * program is terminated./*from w ww . jav a 2 s .c om*/ * * @param args Expecting 3 arguments: A numeric value indicating the number of threads to use to send * data to Kinesis and the name of the stream to send records to, and the AWS region in which these resources * exist or should be created. * @throws InterruptedException If this application is interrupted while sending records to Kinesis. */ public static void main(String[] args) throws InterruptedException { if (args.length != 3) { System.err.println( "Usage: " + StreamWriter.class.getSimpleName() + " <number of threads> <stream name> <region>"); System.exit(1); } int numberOfThreads = Integer.parseInt(args[0]); String streamName = args[1]; Region region = SampleUtils.parseRegion(args[2]); AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain(); ClientConfiguration clientConfig = SampleUtils.configureUserAgentForSample(new ClientConfiguration()); AmazonKinesis kinesis = new AmazonKinesisClient(credentialsProvider, clientConfig); kinesis.setRegion(region); // The more resources we declare the higher write IOPS we need on our DynamoDB table. // We write a record for each resource every interval. // If interval = 500ms, resource count = 7 we need: (1000/500 * 7) = 14 write IOPS minimum. List<String> resources = new ArrayList<>(); resources.add("/index.html"); // These are the possible referrers to use when generating pairs List<String> referrers = new ArrayList<>(); referrers.add("http://www.amazon.com"); referrers.add("http://www.google.com"); referrers.add("http://www.yahoo.com"); referrers.add("http://www.bing.com"); referrers.add("http://www.stackoverflow.com"); referrers.add("http://www.reddit.com"); HttpReferrerPairFactory pairFactory = new HttpReferrerPairFactory(resources, referrers); // Creates a stream to write to with 2 shards if it doesn't exist StreamUtils streamUtils = new StreamUtils(kinesis); streamUtils.createStreamIfNotExists(streamName, 2); LOG.info(String.format("%s stream is ready for use", streamName)); final HttpReferrerKinesisPutter putter = new HttpReferrerKinesisPutter(pairFactory, kinesis, streamName); ExecutorService es = Executors.newCachedThreadPool(); Runnable pairSender = new Runnable() { @Override public void run() { try { putter.sendPairsIndefinitely(DELAY_BETWEEN_RECORDS_IN_MILLIS, TimeUnit.MILLISECONDS); } catch (Exception ex) { LOG.warn( "Thread encountered an error while sending records. Records will no longer be put by this thread.", ex); } } }; for (int i = 0; i < numberOfThreads; i++) { es.submit(pairSender); } LOG.info(String.format("Sending pairs with a %dms delay between records with %d thread(s).", DELAY_BETWEEN_RECORDS_IN_MILLIS, numberOfThreads)); es.shutdown(); es.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS); }
From source file:org.openbaton.monitoring.interfaces.MonitoringPluginCaller.java
public static void main(String[] args) throws IOException, TimeoutException, NotFoundException, ExecutionException, InterruptedException { ExecutorService executor = Executors.newFixedThreadPool(3); class Exec implements Callable<Object> { @Override//from w w w . j a v a 2s . co m public Object call() throws Exception { List<String> hosts = new ArrayList<>(); hosts.add("hostname1"); hosts.add("hostname2"); return null /* new MonitoringPluginCaller("zabbix").getMeasurementResults(hosts, new ArrayList<String>(), "")*/; } } long time = new Date().getTime(); System.out.println("Time ---> " + time); Future fut1 = executor.submit(new Exec()); Future fut2 = executor.submit(new Exec()); System.out.println("2nd call"); System.out.println(fut2.get()); System.out.println("1st call"); System.out.println(fut1.get()); System.out.println("Time ---> " + (new Date().getTime() - time) / 1000); }
From source file:locking.LockingExample.java
public static void main(String[] args) throws Exception { // all of the useful sample code is in ExampleClientThatLocks.java // FakeLimitedResource simulates some external resource that can only be access by one process at a time final FakeLimitedResource resource = new FakeLimitedResource(); ExecutorService service = Executors.newFixedThreadPool(QTY); final TestingServer server = new TestingServer(); try {//from w w w .j a va 2 s .com for (int i = 0; i < QTY; ++i) { final int index = i; Callable<Void> task = new Callable<Void>() { @Override public Void call() throws Exception { CuratorFramework client = CuratorFrameworkFactory.newClient(server.getConnectString(), new ExponentialBackoffRetry(1000, 3)); try { client.start(); ExampleClientThatLocks example = new ExampleClientThatLocks(client, PATH, resource, "Client " + index); for (int j = 0; j < REPETITIONS; ++j) { example.doWork(10, TimeUnit.SECONDS); } } catch (Throwable e) { e.printStackTrace(); } finally { IOUtils.closeQuietly(client); } return null; } }; service.submit(task); } service.shutdown(); service.awaitTermination(10, TimeUnit.MINUTES); } finally { IOUtils.closeQuietly(server); } }
From source file:com.aerospike.load.AerospikeLoad.java
public static void main(String[] args) throws IOException { Thread statPrinter = new Thread(new PrintStat(counters)); try {// w w w.ja v a2s. c o m log.info("Aerospike loader started"); Options options = new Options(); options.addOption("h", "host", true, "Server hostname (default: localhost)"); options.addOption("p", "port", true, "Server port (default: 3000)"); options.addOption("n", "namespace", true, "Namespace (default: test)"); options.addOption("s", "set", true, "Set name. (default: null)"); options.addOption("c", "config", true, "Column definition file name"); options.addOption("wt", "write-threads", true, "Number of writer threads (default: Number of cores * 5)"); options.addOption("rt", "read-threads", true, "Number of reader threads (default: Number of cores * 1)"); options.addOption("l", "rw-throttle", true, "Throttling of reader to writer(default: 10k) "); options.addOption("tt", "transaction-timeout", true, "write transaction timeout in miliseconds(default: No timeout)"); options.addOption("et", "expiration-time", true, "Expiration time of records in seconds (default: never expire)"); options.addOption("T", "timezone", true, "Timezone of source where data dump is taken (default: local timezone)"); options.addOption("ec", "abort-error-count", true, "Error count to abort (default: 0)"); options.addOption("wa", "write-action", true, "Write action if key already exists (default: update)"); options.addOption("v", "verbose", false, "Logging all"); options.addOption("u", "usage", false, "Print usage."); CommandLineParser parser = new PosixParser(); CommandLine cl = parser.parse(options, args, false); if (args.length == 0 || cl.hasOption("u")) { logUsage(options); return; } if (cl.hasOption("l")) { rwThrottle = Integer.parseInt(cl.getOptionValue("l")); } else { rwThrottle = Constants.READLOAD; } // Get all command line options params = Utils.parseParameters(cl); //Get client instance AerospikeClient client = new AerospikeClient(params.host, params.port); if (!client.isConnected()) { log.error("Client is not able to connect:" + params.host + ":" + params.port); return; } if (params.verbose) { log.setLevel(Level.DEBUG); } // Get available processors to calculate default number of threads int cpus = Runtime.getRuntime().availableProcessors(); nWriterThreads = cpus * scaleFactor; nReaderThreads = cpus; // Get writer thread count if (cl.hasOption("wt")) { nWriterThreads = Integer.parseInt(cl.getOptionValue("wt")); nWriterThreads = (nWriterThreads > 0 ? (nWriterThreads > Constants.MAX_THREADS ? Constants.MAX_THREADS : nWriterThreads) : 1); log.debug("Using writer Threads: " + nWriterThreads); } writerPool = Executors.newFixedThreadPool(nWriterThreads); // Get reader thread count if (cl.hasOption("rt")) { nReaderThreads = Integer.parseInt(cl.getOptionValue("rt")); nReaderThreads = (nReaderThreads > 0 ? (nReaderThreads > Constants.MAX_THREADS ? Constants.MAX_THREADS : nReaderThreads) : 1); log.debug("Using reader Threads: " + nReaderThreads); } String columnDefinitionFileName = cl.getOptionValue("c", null); log.debug("Column definition files/directory: " + columnDefinitionFileName); if (columnDefinitionFileName == null) { log.error("Column definition files/directory not specified. use -c <file name>"); return; } File columnDefinitionFile = new File(columnDefinitionFileName); if (!columnDefinitionFile.exists()) { log.error("Column definition files/directory does not exist: " + Utils.getFileName(columnDefinitionFileName)); return; } // Get data file list String[] files = cl.getArgs(); if (files.length == 0) { log.error("No data file Specified: add <file/dir name> to end of the command "); return; } List<String> allFileNames = new ArrayList<String>(); allFileNames = Utils.getFileNames(files); if (allFileNames.size() == 0) { log.error("Given datafiles/directory does not exist"); return; } for (int i = 0; i < allFileNames.size(); i++) { log.debug("File names:" + Utils.getFileName(allFileNames.get(i))); File file = new File(allFileNames.get(i)); counters.write.recordTotal = counters.write.recordTotal + file.length(); } //remove column definition file from list allFileNames.remove(columnDefinitionFileName); log.info("Number of data files:" + allFileNames.size()); /** * Process column definition file to get meta data and bin mapping. */ metadataColumnDefs = new ArrayList<ColumnDefinition>(); binColumnDefs = new ArrayList<ColumnDefinition>(); metadataConfigs = new HashMap<String, String>(); if (Parser.processJSONColumnDefinitions(columnDefinitionFile, metadataConfigs, metadataColumnDefs, binColumnDefs, params)) { log.info("Config file processed."); } else { throw new Exception("Config file parsing Error"); } // Add metadata of config to parameters String metadata; if ((metadata = metadataConfigs.get(Constants.INPUT_TYPE)) != null) { params.fileType = metadata; if (params.fileType.equals(Constants.CSV_FILE)) { // Version check metadata = metadataConfigs.get(Constants.VERSION); String[] vNumber = metadata.split("\\."); int v1 = Integer.parseInt(vNumber[0]); int v2 = Integer.parseInt(vNumber[1]); if ((v1 <= Constants.MajorV) && (v2 <= Constants.MinorV)) { log.debug("Config version used:" + metadata); } else throw new Exception("\"" + Constants.VERSION + ":" + metadata + "\" is not Supported"); // Set delimiter if ((metadata = metadataConfigs.get(Constants.DELIMITER)) != null && metadata.length() == 1) { params.delimiter = metadata.charAt(0); } else { log.warn("\"" + Constants.DELIMITER + ":" + metadata + "\" is not properly specified in config file. Default is ','"); } if ((metadata = metadataConfigs.get(Constants.IGNORE_FIRST_LINE)) != null) { params.ignoreFirstLine = metadata.equals("true"); } else { log.warn("\"" + Constants.IGNORE_FIRST_LINE + ":" + metadata + "\" is not properly specified in config file. Default is false"); } if ((metadata = metadataConfigs.get(Constants.COLUMNS)) != null) { counters.write.colTotal = Integer.parseInt(metadata); } else { throw new Exception("\"" + Constants.COLUMNS + ":" + metadata + "\" is not properly specified in config file"); } } else { throw new Exception("\"" + params.fileType + "\" is not supported in config file"); } } else { throw new Exception("\"" + Constants.INPUT_TYPE + "\" is not specified in config file"); } // add config input to column definitions if (params.fileType.equals(Constants.CSV_FILE)) { List<String> binName = null; if (params.ignoreFirstLine) { String line; BufferedReader br = new BufferedReader( new InputStreamReader(new FileInputStream(allFileNames.get(0)), "UTF8")); if ((line = br.readLine()) != null) { binName = Parser.getCSVRawColumns(line, params.delimiter); br.close(); if (binName.size() != counters.write.colTotal) { throw new Exception("Number of column in config file and datafile are mismatch." + " Datafile: " + Utils.getFileName(allFileNames.get(0)) + " Configfile: " + Utils.getFileName(columnDefinitionFileName)); } } } //update columndefs for metadata for (int i = 0; i < metadataColumnDefs.size(); i++) { if (metadataColumnDefs.get(i).staticValue) { } else { if (metadataColumnDefs.get(i).binValuePos < 0) { if (metadataColumnDefs.get(i).columnName == null) { if (metadataColumnDefs.get(i).jsonPath == null) { log.error("dynamic metadata having improper info" + metadataColumnDefs.toString()); //TODO } else { //TODO check for json_path } } else { if (params.ignoreFirstLine) { if (binName.indexOf(metadataColumnDefs.get(i).binValueHeader) != -1) { metadataColumnDefs.get(i).binValuePos = binName .indexOf(metadataColumnDefs.get(i).binValueHeader); } else { throw new Exception("binName missing in data file:" + metadataColumnDefs.get(i).binValueHeader); } } } } else { if (params.ignoreFirstLine) metadataColumnDefs.get(i).binValueHeader = binName .get(metadataColumnDefs.get(i).binValuePos); } } if ((!metadataColumnDefs.get(i).staticValue) && (metadataColumnDefs.get(i).binValuePos < 0)) { throw new Exception("Information for bin mapping is missing in config file:" + metadataColumnDefs.get(i)); } if (metadataColumnDefs.get(i).srcType == null) { throw new Exception( "Source data type is not properly mentioned:" + metadataColumnDefs.get(i)); } if (metadataColumnDefs.get(i).binNameHeader == Constants.SET && !metadataColumnDefs.get(i).srcType.equals(SrcColumnType.STRING)) { throw new Exception("Set name should be string type:" + metadataColumnDefs.get(i)); } if (metadataColumnDefs.get(i).binNameHeader.equalsIgnoreCase(Constants.SET) && params.set != null) { throw new Exception( "Set name is given both in config file and commandline. Provide only once."); } } //update columndefs for bins for (int i = 0; i < binColumnDefs.size(); i++) { if (binColumnDefs.get(i).staticName) { } else { if (binColumnDefs.get(i).binNamePos < 0) { if (binColumnDefs.get(i).columnName == null) { if (binColumnDefs.get(i).jsonPath == null) { log.error("dynamic bin having improper info"); //TODO } else { //TODO check for json_path } } else { if (params.ignoreFirstLine) { if (binName.indexOf(binColumnDefs.get(i).binNameHeader) != -1) { binColumnDefs.get(i).binNamePos = binName .indexOf(binColumnDefs.get(i).binNameHeader); } else { throw new Exception("binName missing in data file:" + binColumnDefs.get(i).binNameHeader); } } } } else { if (params.ignoreFirstLine) binColumnDefs.get(i).binNameHeader = binName.get(binColumnDefs.get(i).binNamePos); } } if (binColumnDefs.get(i).staticValue) { } else { if (binColumnDefs.get(i).binValuePos < 0) { if (binColumnDefs.get(i).columnName == null) { if (binColumnDefs.get(i).jsonPath == null) { log.error("dynamic bin having improper info"); //TODO } else { //TODO check for json_path } } else { if (params.ignoreFirstLine) { if (binName.contains(binColumnDefs.get(i).binValueHeader)) { binColumnDefs.get(i).binValuePos = binName .indexOf(binColumnDefs.get(i).binValueHeader); } else if (!binColumnDefs.get(i).binValueHeader.toLowerCase() .equals(Constants.SYSTEM_TIME)) { throw new Exception("Wrong column name mentioned in config file:" + binColumnDefs.get(i).binValueHeader); } } } } else { if (params.ignoreFirstLine) binColumnDefs.get(i).binValueHeader = binName.get(binColumnDefs.get(i).binValuePos); } //check for missing entries in config file if (binColumnDefs.get(i).binValuePos < 0 && binColumnDefs.get(i).binValueHeader == null) { throw new Exception("Information missing(Value header or bin mapping) in config file:" + binColumnDefs.get(i)); } //check for proper data type in config file. if (binColumnDefs.get(i).srcType == null) { throw new Exception( "Source data type is not properly mentioned:" + binColumnDefs.get(i)); } //check for valid destination type if ((binColumnDefs.get(i).srcType.equals(SrcColumnType.TIMESTAMP) || binColumnDefs.get(i).srcType.equals(SrcColumnType.BLOB)) && binColumnDefs.get(i).dstType == null) { throw new Exception("Destination type is not mentioned: " + binColumnDefs.get(i)); } //check for encoding if (binColumnDefs.get(i).dstType != null && binColumnDefs.get(i).encoding == null) { throw new Exception( "Encoding is not given for src-dst type conversion:" + binColumnDefs.get(i)); } //check for valid encoding if (binColumnDefs.get(i).srcType.equals(SrcColumnType.BLOB) && !binColumnDefs.get(i).encoding.equals(Constants.HEX_ENCODING)) { throw new Exception("Wrong encoding for blob data:" + binColumnDefs.get(i)); } } //Check static bin name mapped to dynamic bin value if ((binColumnDefs.get(i).binNamePos == binColumnDefs.get(i).binValuePos) && (binColumnDefs.get(i).binNamePos != -1)) { throw new Exception("Static bin name mapped to dynamic bin value:" + binColumnDefs.get(i)); } //check for missing entries in config file if (binColumnDefs.get(i).binNameHeader == null && binColumnDefs.get(i).binNameHeader.length() > Constants.BIN_NAME_LENGTH) { throw new Exception("Information missing binName or large binName in config file:" + binColumnDefs.get(i)); } } } log.info(params.toString()); log.debug("MetadataConfig:" + metadataColumnDefs); log.debug("BinColumnDefs:" + binColumnDefs); // Start PrintStat thread statPrinter.start(); // Reader pool size ExecutorService readerPool = Executors.newFixedThreadPool( nReaderThreads > allFileNames.size() ? allFileNames.size() : nReaderThreads); log.info("Reader pool size : " + nReaderThreads); // Submit all tasks to writer threadpool. for (String aFile : allFileNames) { log.debug("Submitting task for: " + aFile); readerPool.submit(new AerospikeLoad(aFile, client, params)); } // Wait for reader pool to complete readerPool.shutdown(); log.info("Shutdown down reader thread pool"); while (!readerPool.isTerminated()) ; //readerPool.awaitTermination(20, TimeUnit.MINUTES); log.info("Reader thread pool terminated"); // Wait for writer pool to complete after getting all tasks from reader pool writerPool.shutdown(); log.info("Shutdown down writer thread pool"); while (!writerPool.isTerminated()) ; log.info("Writer thread pool terminated"); // Print final statistic of aerospike-loader. log.info("Final Statistics of importer: (Succesfull Writes = " + counters.write.writeCount.get() + ", " + "Errors=" + (counters.write.writeErrors.get() + counters.write.readErrors.get() + counters.write.processingErrors.get()) + "(" + (counters.write.writeErrors.get()) + "-Write," + counters.write.readErrors.get() + "-Read," + counters.write.processingErrors.get() + "-Processing)"); } catch (Exception e) { log.error(e); if (log.isDebugEnabled()) { e.printStackTrace(); } } finally { // Stop statistic printer thread. statPrinter.interrupt(); log.info("Aerospike loader completed"); } }