Example usage for java.util.concurrent ExecutorService submit

List of usage examples for java.util.concurrent ExecutorService submit

  1. HOME
  2. Java
  3. java
  4. java.util.concurrent.*
  5. ExecutorService
  6. submit

Introduction

In this page you can find the example usage for java.util.concurrent ExecutorService submit.

Prototype

Future<?> submit(Runnable task);

Source Link

Document

Submits a Runnable task for execution and returns a Future representing that task.

Usage

From source file:com.uniteddev.Unity.Downloader.java

public static void removeRedundancies() throws InterruptedException {
    class removeRedundancy implements Runnable {
        private int i;

        removeRedundancy(int i) {
            this.i = i;
        }//  w w w . j  a  v a 2  s  . c  o  m

        public void run() {
            files.remove(this.i);
        }
    }

    int i = 0;
    ExecutorService pool = Executors.newFixedThreadPool(cores);
    while (i < files.size()) {
        String file = files.get(i);
        file = namefix(file);
        if (new File(Minecraft.getWorkingDirectory(), file).exists())
            pool.submit(new removeRedundancy(i)); // this was originally a while loop
        else
            i++;
    }
    pool.shutdown();
    pool.awaitTermination(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
}

From source file:edu.iu.daal_pca.PCAUtil.java

/**
 * Generate data and upload to the data dir.
 *
 * @param numOfDataPoints// ww  w .  j  a v a  2s.  c  o  m
 * @param vectorSize
 * @param numPointFiles
 * @param localInputDir
 * @param fs
 * @param dataDir
 * @throws IOException
 * @throws InterruptedException
 * @throws ExecutionException
 */
static void generatePoints(int numOfDataPoints, int vectorSize, int numPointFiles, String localInputDir,
        FileSystem fs, Path dataDir) throws IOException, InterruptedException, ExecutionException {
    int pointsPerFile = numOfDataPoints / numPointFiles;
    System.out.println("Writing " + pointsPerFile + " vectors to a file");
    // Check data directory
    if (fs.exists(dataDir)) {
        fs.delete(dataDir, true);
    }
    // Check local directory
    File localDir = new File(localInputDir);
    // If existed, regenerate data
    if (localDir.exists() && localDir.isDirectory()) {
        for (File file : localDir.listFiles()) {
            file.delete();
        }
        localDir.delete();
    }
    boolean success = localDir.mkdir();
    if (success) {
        System.out.println("Directory: " + localInputDir + " created");
    }
    if (pointsPerFile == 0) {
        throw new IOException("No point to write.");
    }
    // Create random data points
    int poolSize = Runtime.getRuntime().availableProcessors();
    ExecutorService service = Executors.newFixedThreadPool(poolSize);
    List<Future<?>> futures = new LinkedList<Future<?>>();
    for (int k = 0; k < numPointFiles; k++) {
        // Future<?> f = service.submit(new DataGenRunnable(pointsPerFile, localInputDir, Integer.toString(k), vectorSize));
        Future<?> f = service
                .submit(new DataGenMMDense(pointsPerFile, localInputDir, Integer.toString(k), vectorSize));
        futures.add(f); // add a new thread
    }
    for (Future<?> f : futures) {
        f.get();
    }
    // Shut down the executor service so that this
    // thread can exit
    service.shutdownNow();
    // Wrap to path object
    Path localInput = new Path(localInputDir);
    fs.copyFromLocalFile(localInput, dataDir);
    DeleteFileFolder(localInputDir);
}

From source file:com.antsdb.saltedfish.util.UberUtil.java

public static <T> List<T> runParallel(int nThreads, Callable<T> callback) throws Exception {
    ExecutorService pool = Executors.newFixedThreadPool(nThreads);
    List<Future<T>> futures = new ArrayList<>();
    for (int i = 0; i < nThreads; i++) {
        Callable<T> run = (Callable<T>) UberUtil.clone(callback);
        futures.add(pool.submit(run));
    }//  www  .j a va 2  s  . c  o  m
    List<T> result = new ArrayList<>();
    for (Future<T> i : futures) {
        result.add(i.get());
    }
    pool.shutdown();
    return result;
}

From source file:edu.iu.daal_svd.SVDUtil.java

/**
 * Generate data and upload to the data dir.
 * /*from  ww  w  .ja v a2  s  .  c o m*/
 * @param numOfDataPoints
 * @param vectorSize
 * @param numPointFiles
 * @param localInputDir
 * @param fs
 * @param dataDir
 * @throws IOException
 * @throws InterruptedException
 * @throws ExecutionException
 */
static void generatePoints(int numOfDataPoints, int vectorSize, int numPointFiles, String localInputDir,
        FileSystem fs, Path dataDir) throws IOException, InterruptedException, ExecutionException {
    int pointsPerFile = numOfDataPoints / numPointFiles;
    System.out.println("Writing " + pointsPerFile + " vectors to a file");
    // Check data directory
    if (fs.exists(dataDir)) {
        fs.delete(dataDir, true);
    }
    // Check local directory
    File localDir = new File(localInputDir);
    // If existed, regenerate data
    if (localDir.exists() && localDir.isDirectory()) {
        for (File file : localDir.listFiles()) {
            file.delete();
        }
        localDir.delete();
    }
    boolean success = localDir.mkdir();
    if (success) {
        System.out.println("Directory: " + localInputDir + " created");
    }
    if (pointsPerFile == 0) {
        throw new IOException("No point to write.");
    }
    // Create random data points
    int poolSize = Runtime.getRuntime().availableProcessors();
    ExecutorService service = Executors.newFixedThreadPool(poolSize);
    List<Future<?>> futures = new LinkedList<Future<?>>();
    for (int k = 0; k < numPointFiles; k++) {
        // Future<?> f =
        //   service.submit(new DataGenRunnable(
        //     pointsPerFile, localInputDir, Integer
        //       .toString(k), vectorSize));
        Future<?> f = service
                .submit(new DataGenMMDense(pointsPerFile, localInputDir, Integer.toString(k), vectorSize));

        futures.add(f); // add a new thread
    }
    for (Future<?> f : futures) {
        f.get();
    }
    // Shut down the executor service so that this
    // thread can exit
    service.shutdownNow();
    // Wrap to path object
    Path localInput = new Path(localInputDir);
    fs.copyFromLocalFile(localInput, dataDir);
    DeleteFileFolder(localInputDir);
}

From source file:com.linkedin.pinot.perf.QueryRunner.java

/**
 * Use multiple threads to run queries as fast as possible.
 *
 * Start {numThreads} worker threads to send queries (blocking call) back to back, and use the main thread to collect
 * the statistic information and log them periodically.
 *
 * @param conf perf benchmark driver config.
 * @param queryFile query file.//from  www  .  j a v  a 2 s.  c o m
 * @param numThreads number of threads sending queries.
 * @throws Exception
 */
@SuppressWarnings("InfiniteLoopStatement")
public static void multiThreadedsQueryRunner(PerfBenchmarkDriverConf conf, String queryFile,
        final int numThreads) throws Exception {
    final long randomSeed = 123456789L;
    final Random random = new Random(randomSeed);
    final int reportIntervalMillis = 3000;

    final List<String> queries;
    try (FileInputStream input = new FileInputStream(new File(queryFile))) {
        queries = IOUtils.readLines(input);
    }

    final int numQueries = queries.size();
    final PerfBenchmarkDriver driver = new PerfBenchmarkDriver(conf);
    final AtomicInteger counter = new AtomicInteger(0);
    final AtomicLong totalResponseTime = new AtomicLong(0L);
    final ExecutorService executorService = Executors.newFixedThreadPool(numThreads);

    final DescriptiveStatistics stats = new DescriptiveStatistics();
    final CountDownLatch latch = new CountDownLatch(numThreads);

    for (int i = 0; i < numThreads; i++) {
        executorService.submit(new Runnable() {
            @Override
            public void run() {
                for (int j = 0; j < numQueries; j++) {
                    String query = queries.get(random.nextInt(numQueries));
                    long startTime = System.currentTimeMillis();
                    try {
                        driver.postQuery(query);
                        long clientTime = System.currentTimeMillis() - startTime;
                        synchronized (stats) {
                            stats.addValue(clientTime);
                        }

                        counter.getAndIncrement();
                        totalResponseTime.getAndAdd(clientTime);
                    } catch (Exception e) {
                        LOGGER.error("Caught exception while running query: {}", query, e);
                        return;
                    }
                }
                latch.countDown();
            }
        });
    }

    executorService.shutdown();

    int iter = 0;
    long startTime = System.currentTimeMillis();
    while (latch.getCount() > 0) {
        Thread.sleep(reportIntervalMillis);
        double timePassedSeconds = ((double) (System.currentTimeMillis() - startTime)) / MILLIS_PER_SECOND;
        int count = counter.get();
        double avgResponseTime = ((double) totalResponseTime.get()) / count;
        LOGGER.info("Time Passed: {}s, Query Executed: {}, QPS: {}, Avg Response Time: {}ms", timePassedSeconds,
                count, count / timePassedSeconds, avgResponseTime);

        iter++;
        if (iter % 10 == 0) {
            printStats(stats);
        }
    }

    printStats(stats);
}

From source file:com.dsclab.loader.app.Loader.java

public static void load(Configs prop)
        throws SQLException, ClassNotFoundException, InterruptedException, ExecutionException {
    int readThread = prop.getReadThread();
    int writeThread = prop.getWriteThread();
    ExecutorService readExecutor = Executors.newFixedThreadPool(readThread);
    ExecutorService writeExecutor = Executors.newFixedThreadPool(writeThread);
    LOG.info("Start load: writeThread:" + writeThread + ", readThread:" + readThread);
    BlockingQueue<List<String>> contentQueue = new LinkedBlockingQueue<>();

    int tableCount = tableTask.size();
    int sum = 0;/*from w ww.  j  a v  a  2  s.  com*/
    for (int i = 0; i < tableCount; i++) {
        sum = sum + tableTask.get(i).getTaskSqlList().size();
    }
    for (int i = 0; i < sum; i++) {
        readExecutor.submit(new ProducerThread(prop.getInputURL(), contentQueue));
        writeExecutor.submit(new ConsumerThread(prop.getOutputURL(), contentQueue));
    }

    readExecutor.shutdown();
    readExecutor.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS);
    System.out.println("[CHIA7712] read threads end");
    writeExecutor.shutdown();
    writeExecutor.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS);
    System.out.println("[CHIA7712] write threads end");
}

From source file:com.linkedin.pinot.perf.QueryRunner.java

/**
 * Use multiple threads to run query at an increasing target QPS.
 *
 * Use a concurrent linked queue to buffer the queries to be sent. Use the main thread to insert queries into the
 * queue at the target QPS, and start {numThreads} worker threads to fetch queries from the queue and send them.
 * We start with the start QPS, and keep adding delta QPS to the start QPS during the test. The main thread is
 * responsible for collecting the statistic information and log them periodically.
 *
 * @param conf perf benchmark driver config.
 * @param queryFile query file./*from   w  w  w . j ava 2s .com*/
 * @param numThreads number of threads sending queries.
 * @param startQPS start QPS
 * @param deltaQPS delta QPS
 * @throws Exception
 */
@SuppressWarnings("InfiniteLoopStatement")
public static void targetQPSQueryRunner(PerfBenchmarkDriverConf conf, String queryFile, int numThreads,
        double startQPS, double deltaQPS) throws Exception {
    final long randomSeed = 123456789L;
    final Random random = new Random(randomSeed);
    final int timePerTargetQPSMillis = 60000;
    final int queueLengthThreshold = Math.max(20, (int) startQPS);

    final List<String> queries;
    try (FileInputStream input = new FileInputStream(new File(queryFile))) {
        queries = IOUtils.readLines(input);
    }
    final int numQueries = queries.size();

    final PerfBenchmarkDriver driver = new PerfBenchmarkDriver(conf);
    final AtomicInteger counter = new AtomicInteger(0);
    final AtomicLong totalResponseTime = new AtomicLong(0L);
    final ExecutorService executorService = Executors.newFixedThreadPool(numThreads);

    final ConcurrentLinkedQueue<String> queryQueue = new ConcurrentLinkedQueue<>();
    double currentQPS = startQPS;
    int intervalMillis = (int) (MILLIS_PER_SECOND / currentQPS);

    for (int i = 0; i < numThreads; i++) {
        executorService.submit(new Runnable() {
            @Override
            public void run() {
                while (true) {
                    String query = queryQueue.poll();
                    if (query == null) {
                        try {
                            Thread.sleep(1);
                            continue;
                        } catch (InterruptedException e) {
                            LOGGER.error("Interrupted.", e);
                            return;
                        }
                    }
                    long startTime = System.currentTimeMillis();
                    try {
                        driver.postQuery(query);
                        counter.getAndIncrement();
                        totalResponseTime.getAndAdd(System.currentTimeMillis() - startTime);
                    } catch (Exception e) {
                        LOGGER.error("Caught exception while running query: {}", query, e);
                        return;
                    }
                }
            }
        });
    }

    LOGGER.info("Start with QPS: {}, delta QPS: {}", startQPS, deltaQPS);
    while (true) {
        long startTime = System.currentTimeMillis();
        while (System.currentTimeMillis() - startTime <= timePerTargetQPSMillis) {
            if (queryQueue.size() > queueLengthThreshold) {
                executorService.shutdownNow();
                throw new RuntimeException("Cannot achieve target QPS of: " + currentQPS);
            }
            queryQueue.add(queries.get(random.nextInt(numQueries)));
            Thread.sleep(intervalMillis);
        }
        double timePassedSeconds = ((double) (System.currentTimeMillis() - startTime)) / MILLIS_PER_SECOND;
        int count = counter.getAndSet(0);
        double avgResponseTime = ((double) totalResponseTime.getAndSet(0)) / count;
        LOGGER.info("Target QPS: {}, Interval: {}ms, Actual QPS: {}, Avg Response Time: {}ms", currentQPS,
                intervalMillis, count / timePassedSeconds, avgResponseTime);

        // Find a new interval
        int newIntervalMillis;
        do {
            currentQPS += deltaQPS;
            newIntervalMillis = (int) (MILLIS_PER_SECOND / currentQPS);
        } while (newIntervalMillis == intervalMillis);
        intervalMillis = newIntervalMillis;
    }
}

From source file:loadTest.loadTestLib.LUtil.java

public static boolean startDockerBuild() throws IOException, InterruptedException {
    if (useDocker) {
        if (runInDockerCluster) {

            HashMap<String, Integer> dockerNodes = getDockerNodes();
            String dockerTar = LUtil.class.getClassLoader().getResource("docker/loadTest.tar").toString()
                    .substring(5);//w w  w.  j  a v  a 2s.  c o m
            ExecutorService exec = Executors.newFixedThreadPool(dockerNodes.size());

            dockerError = false;
            doneDockers = 0;

            for (Entry<String, Integer> entry : dockerNodes.entrySet()) {
                exec.submit(new Runnable() {
                    @Override
                    public void run() {
                        try {
                            String url = entry.getKey() + "/images/vauvenal5/loadtest";
                            URL obj = new URL(url);
                            HttpURLConnection con = (HttpURLConnection) obj.openConnection();
                            con.setRequestMethod("DELETE");
                            con.setRequestProperty("force", "true");
                            int responseCode = con.getResponseCode();
                            con.disconnect();

                            url = entry.getKey()
                                    + "/build?t=vauvenal5/loadtest&dockerfile=./loadTest/Dockerfile";
                            obj = new URL(url);
                            con = (HttpURLConnection) obj.openConnection();
                            con.setRequestMethod("POST");
                            con.setRequestProperty("Content-type", "application/tar");
                            con.setDoOutput(true);
                            File file = new File(dockerTar);
                            FileInputStream fileStr = new FileInputStream(file);
                            byte[] b = new byte[(int) file.length()];
                            fileStr.read(b);
                            con.getOutputStream().write(b);
                            con.getOutputStream().flush();
                            con.getOutputStream().close();

                            responseCode = con.getResponseCode();

                            if (responseCode != 200) {
                                dockerError = true;
                            }

                            String msg = "";

                            do {
                                Thread.sleep(5000);
                                msg = "";
                                url = entry.getKey() + "/images/json";
                                obj = new URL(url);
                                HttpURLConnection con2 = (HttpURLConnection) obj.openConnection();
                                con2.setRequestMethod("GET");
                                responseCode = con2.getResponseCode();

                                BufferedReader in = new BufferedReader(
                                        new InputStreamReader(con2.getInputStream()));
                                String line = null;

                                while ((line = in.readLine()) != null) {
                                    msg += line;
                                }

                                con2.disconnect();
                            } while (!msg.contains("vauvenal5/loadtest"));

                            con.disconnect();
                            doneDockers++;
                        } catch (MalformedURLException ex) {
                            dockerError = true;
                        } catch (FileNotFoundException ex) {
                            dockerError = true;
                        } catch (IOException ex) {
                            dockerError = true;
                        } catch (InterruptedException ex) {
                            dockerError = true;
                        }
                    }
                });
            }

            while (doneDockers < dockerNodes.size()) {
                Thread.sleep(5000);
            }

            return !dockerError;
        }

        //get the path and substring the 'file:' from the URI
        String dockerfile = LUtil.class.getClassLoader().getResource("docker/loadTest").toString().substring(5);

        ProcessBuilder processBuilder = new ProcessBuilder("docker", "rmi", "-f", "vauvenal5/loadtest");
        processBuilder.redirectOutput(ProcessBuilder.Redirect.INHERIT);
        Process docker = processBuilder.start();

        docker.waitFor();

        processBuilder = new ProcessBuilder("docker", "build", "-t", "vauvenal5/loadtest", dockerfile);
        processBuilder.redirectOutput(ProcessBuilder.Redirect.INHERIT);
        Process proc = processBuilder.start();
        return proc.waitFor() == 0;
    }
    return true;
}

From source file:edu.cornell.med.icb.R.RUtils.java

/**
 * Can be used to start a rserve instance.
 * @param threadPool The ExecutorService used to start the Rserve process
 * @param rServeCommand Full path to command used to start Rserve process
 * @param host Host where the command should be sent
 * @param port Port number where the command should be sent
 * @param username Username to send to the server if authentication is required
 * @param password Password to send to the server if authentication is required
 * @return The return value from the Rserve instance
 *///from  w  w  w.  j  a v a  2  s . c o  m
static Future<Integer> startup(final ExecutorService threadPool, final String rServeCommand, final String host,
        final int port, final String username, final String password) {
    if (LOG.isInfoEnabled()) {
        LOG.info("Attempting to start Rserve on " + host + ":" + port);
    }

    return threadPool.submit(new Callable<Integer>() {
        public Integer call() throws IOException {
            final List<String> commands = new ArrayList<String>();

            // if the host is not local, use ssh to exec the command
            if (!"localhost".equals(host) && !"127.0.0.1".equals(host)
                    && !InetAddress.getLocalHost().equals(InetAddress.getByName(host))) {
                commands.add("ssh");
                commands.add(host);
            }

            // TODO - this will fail when spaces are in the the path to the executable
            CollectionUtils.addAll(commands, rServeCommand.split(" "));
            commands.add("--RS-port");
            commands.add(Integer.toString(port));

            final String[] command = commands.toArray(new String[commands.size()]);
            LOG.debug(ArrayUtils.toString(commands));

            final ProcessBuilder builder = new ProcessBuilder(command);
            builder.redirectErrorStream(true);
            final Process process = builder.start();
            BufferedReader br = null;
            try {
                final InputStream is = process.getInputStream();
                final InputStreamReader isr = new InputStreamReader(is);
                br = new BufferedReader(isr);
                String line;
                while ((line = br.readLine()) != null) {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug(host + ":" + port + "> " + line);
                    }
                }

                process.waitFor();
                if (LOG.isInfoEnabled()) {
                    LOG.info("Rserve on " + host + ":" + port + " terminated");
                }
            } catch (InterruptedException e) {
                LOG.error("Interrupted!", e);
                process.destroy();
                Thread.currentThread().interrupt();
            } finally {
                IOUtils.closeQuietly(br);
            }

            final int exitValue = process.exitValue();
            if (LOG.isInfoEnabled()) {
                LOG.info("Rserve on " + host + ":" + port + " returned " + exitValue);
            }
            return exitValue;
        }
    });
}

From source file:it.units.malelab.ege.util.Utils.java

public static void broadcast(final EvolutionEvent event, List<? extends EvolverListener> listeners,
        final ExecutorService executor) {
    for (final EvolverListener listener : listeners) {
        if (listener.getEventClasses().contains(event.getClass())) {
            executor.submit(new Runnable() {
                @Override/*from w ww . ja v  a  2 s. co m*/
                public void run() {
                    listener.listen(event);
                }
            });
        }
    }
}