Example usage for java.util.concurrent ExecutorService shutdownNow

List of usage examples for java.util.concurrent ExecutorService shutdownNow

Introduction

In this page you can find the example usage for java.util.concurrent ExecutorService shutdownNow.

Prototype

List<Runnable> shutdownNow();

Source Link

Document

Attempts to stop all actively executing tasks, halts the processing of waiting tasks, and returns a list of the tasks that were awaiting execution.

Usage

From source file:io.druid.data.input.impl.PrefetchableTextFilesFirehoseFactory.java

@Override
public Firehose connect(StringInputRowParser firehoseParser, File temporaryDirectory) throws IOException {
    if (maxCacheCapacityBytes == 0 && maxFetchCapacityBytes == 0) {
        return super.connect(firehoseParser, temporaryDirectory);
    }/* www  . j a va  2s  .co  m*/

    if (objects == null) {
        objects = ImmutableList.copyOf(Preconditions.checkNotNull(initObjects(), "objects"));
    }

    Preconditions.checkState(temporaryDirectory.exists(), "temporaryDirectory[%s] does not exist",
            temporaryDirectory);
    Preconditions.checkState(temporaryDirectory.isDirectory(), "temporaryDirectory[%s] is not a directory",
            temporaryDirectory);

    // fetchExecutor is responsible for background data fetching
    final ExecutorService fetchExecutor = createFetchExecutor();

    return new FileIteratingFirehose(new Iterator<LineIterator>() {
        // When prefetching is enabled, fetchFiles and nextFetchIndex are updated by the fetchExecutor thread, but
        // read by both the main thread (in hasNext()) and the fetchExecutor thread (in fetch()). To guarantee that
        // fetchFiles and nextFetchIndex are updated atomically, this lock must be held before updating
        // them.
        private final Object fetchLock = new Object();
        private final LinkedBlockingQueue<FetchedFile> fetchFiles = new LinkedBlockingQueue<>();

        // Number of bytes currently fetched files.
        // This is updated when a file is successfully fetched or a fetched file is deleted.
        private final AtomicLong fetchedBytes = new AtomicLong(0);
        private final boolean cacheInitialized;
        private final boolean prefetchEnabled;

        private Future<Void> fetchFuture;
        private int cacheIterateIndex;
        // nextFetchIndex indicates which object should be downloaded when fetch is triggered.
        private int nextFetchIndex;

        {
            cacheInitialized = totalCachedBytes > 0;
            prefetchEnabled = maxFetchCapacityBytes > 0;

            if (cacheInitialized) {
                nextFetchIndex = cacheFiles.size();
            }
            if (prefetchEnabled) {
                fetchIfNeeded(totalCachedBytes);
            }
        }

        private void fetchIfNeeded(long remainingBytes) {
            if ((fetchFuture == null || fetchFuture.isDone()) && remainingBytes <= prefetchTriggerBytes) {
                fetchFuture = fetchExecutor.submit(() -> {
                    fetch();
                    return null;
                });
            }
        }

        /**
         * Fetch objects to a local disk up to {@link PrefetchableTextFilesFirehoseFactory#maxFetchCapacityBytes}.
         * This method is not thread safe and must be called by a single thread.  Note that even
         * {@link PrefetchableTextFilesFirehoseFactory#maxFetchCapacityBytes} is 0, at least 1 file is always fetched.
         * This is for simplifying design, and should be improved when our client implementations for cloud storages
         * like S3 support range scan.
         */
        private void fetch() throws Exception {
            for (int i = nextFetchIndex; i < objects.size()
                    && fetchedBytes.get() <= maxFetchCapacityBytes; i++) {
                final ObjectType object = objects.get(i);
                LOG.info("Fetching object[%s], fetchedBytes[%d]", object, fetchedBytes.get());
                final File outFile = File.createTempFile(FETCH_FILE_PREFIX, null, temporaryDirectory);
                fetchedBytes.addAndGet(download(object, outFile, 0));
                synchronized (fetchLock) {
                    fetchFiles.put(new FetchedFile(object, outFile));
                    nextFetchIndex++;
                }
            }
        }

        /**
         * Downloads an object. It retries downloading {@link PrefetchableTextFilesFirehoseFactory#maxFetchRetry}
         * times and throws an exception.
         *
         * @param object   an object to be downloaded
         * @param outFile  a file which the object data is stored
         * @param tryCount current retry count
         *
         * @return number of downloaded bytes
         *
         * @throws IOException
         */
        private long download(ObjectType object, File outFile, int tryCount) throws IOException {
            try (final InputStream is = openObjectStream(object);
                    final CountingOutputStream cos = new CountingOutputStream(new FileOutputStream(outFile))) {
                IOUtils.copy(is, cos);
                return cos.getCount();
            } catch (IOException e) {
                final int nextTry = tryCount + 1;
                if (!Thread.currentThread().isInterrupted() && nextTry < maxFetchRetry) {
                    LOG.error(e, "Failed to download object[%s], retrying (%d of %d)", object, nextTry,
                            maxFetchRetry);
                    outFile.delete();
                    return download(object, outFile, nextTry);
                } else {
                    LOG.error(e, "Failed to download object[%s], retries exhausted, aborting", object);
                    throw e;
                }
            }
        }

        @Override
        public boolean hasNext() {
            synchronized (fetchLock) {
                return (cacheInitialized && cacheIterateIndex < cacheFiles.size()) || !fetchFiles.isEmpty()
                        || nextFetchIndex < objects.size();
            }
        }

        @Override
        public LineIterator next() {
            if (!hasNext()) {
                throw new NoSuchElementException();
            }

            // If fetch() fails, hasNext() always returns true because nextFetchIndex must be smaller than the number
            // of objects, which means next() is always called. The below method checks that fetch() threw an exception
            // and propagates it if exists.
            checkFetchException();

            final OpenedObject openedObject;

            try {
                // Check cache first
                if (cacheInitialized && cacheIterateIndex < cacheFiles.size()) {
                    final FetchedFile fetchedFile = cacheFiles.get(cacheIterateIndex++);
                    openedObject = new OpenedObject(fetchedFile, getNoopCloser());
                } else if (prefetchEnabled) {
                    openedObject = openObjectFromLocal();
                } else {
                    openedObject = openObjectFromRemote();
                }

                final InputStream stream = wrapObjectStream(openedObject.object, openedObject.objectStream);

                return new ResourceCloseableLineIterator(new InputStreamReader(stream, Charsets.UTF_8),
                        openedObject.resourceCloser);
            } catch (IOException e) {
                throw Throwables.propagate(e);
            }
        }

        private void checkFetchException() {
            if (fetchFuture != null && fetchFuture.isDone()) {
                try {
                    fetchFuture.get();
                    fetchFuture = null;
                } catch (InterruptedException | ExecutionException e) {
                    throw Throwables.propagate(e);
                }
            }
        }

        private OpenedObject openObjectFromLocal() throws IOException {
            final FetchedFile fetchedFile;
            final Closeable resourceCloser;

            if (!fetchFiles.isEmpty()) {
                // If there are already fetched files, use them
                fetchedFile = fetchFiles.poll();
                resourceCloser = cacheIfPossibleAndGetCloser(fetchedFile, fetchedBytes);
                fetchIfNeeded(fetchedBytes.get());
            } else {
                // Otherwise, wait for fetching
                try {
                    fetchIfNeeded(fetchedBytes.get());
                    fetchedFile = fetchFiles.poll(fetchTimeout, TimeUnit.MILLISECONDS);
                    if (fetchedFile == null) {
                        // Check the latest fetch is failed
                        checkFetchException();
                        // Or throw a timeout exception
                        throw new RuntimeException(new TimeoutException());
                    }
                    resourceCloser = cacheIfPossibleAndGetCloser(fetchedFile, fetchedBytes);
                    // trigger fetch again for subsequent next() calls
                    fetchIfNeeded(fetchedBytes.get());
                } catch (InterruptedException e) {
                    throw Throwables.propagate(e);
                }
            }
            return new OpenedObject(fetchedFile, resourceCloser);
        }

        private OpenedObject openObjectFromRemote() throws IOException {
            final OpenedObject openedObject;
            final Closeable resourceCloser = getNoopCloser();

            if (totalCachedBytes < maxCacheCapacityBytes) {
                LOG.info("Caching object[%s]", objects.get(nextFetchIndex));
                try {
                    // Since maxFetchCapacityBytes is 0, at most one file is fetched.
                    fetch();
                    FetchedFile fetchedFile = fetchFiles.poll();
                    if (fetchedFile == null) {
                        throw new ISE("Cannot fetch object[%s]", objects.get(nextFetchIndex));
                    }
                    cacheIfPossible(fetchedFile);
                    fetchedBytes.addAndGet(-fetchedFile.length());
                    openedObject = new OpenedObject(fetchedFile, resourceCloser);
                } catch (Exception e) {
                    throw Throwables.propagate(e);
                }
            } else {
                final ObjectType object = objects.get(nextFetchIndex++);
                LOG.info("Reading object[%s]", object);
                openedObject = new OpenedObject(object, openObjectStream(object), resourceCloser);
            }
            return openedObject;
        }
    }, firehoseParser, () -> {
        fetchExecutor.shutdownNow();
        try {
            Preconditions.checkState(fetchExecutor.awaitTermination(fetchTimeout, TimeUnit.MILLISECONDS));
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            throw new ISE("Failed to shutdown fetch executor during close");
        }
    });
}