ddf.catalog.resource.download.ReliableResourceDownloadManagerTest.java Source code

Java tutorial

Introduction

Here is the source code for ddf.catalog.resource.download.ReliableResourceDownloadManagerTest.java

Source

/**
 * Copyright (c) Codice Foundation
 * <p>
 * This is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser
 * General Public License as published by the Free Software Foundation, either version 3 of the
 * License, or any later version.
 * <p>
 * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without
 * even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * Lesser General Public License for more details. A copy of the GNU Lesser General Public License
 * is distributed along with this program and can be found at
 * <http://www.gnu.org/licenses/lgpl.html>.
 */
package ddf.catalog.resource.download;

import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.notNullValue;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.anyLong;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.reset;
import static org.mockito.Mockito.timeout;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;

import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.Serializable;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;

import javax.activation.MimeType;
import javax.activation.MimeTypeParseException;

import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.MethodRule;
import org.junit.rules.TestWatchman;
import org.junit.runners.model.FrameworkMethod;
import org.mockito.ArgumentCaptor;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import ddf.catalog.cache.MockInputStream;
import ddf.catalog.cache.impl.CacheKey;
import ddf.catalog.cache.impl.ResourceCache;
import ddf.catalog.data.Metacard;
import ddf.catalog.data.impl.BasicTypes;
import ddf.catalog.event.retrievestatus.DownloadStatusInfo;
import ddf.catalog.event.retrievestatus.DownloadStatusInfoImpl;
import ddf.catalog.event.retrievestatus.DownloadsStatusEventListener;
import ddf.catalog.event.retrievestatus.DownloadsStatusEventPublisher;
import ddf.catalog.operation.ResourceRequest;
import ddf.catalog.operation.ResourceResponse;
import ddf.catalog.resource.Resource;
import ddf.catalog.resource.ResourceNotFoundException;
import ddf.catalog.resource.ResourceNotSupportedException;
import ddf.catalog.resource.data.ReliableResource;
import ddf.catalog.resourceretriever.ResourceRetriever;

public class ReliableResourceDownloadManagerTest {
    public static final int MONITOR_PERIOD = 5;

    public static final String EXPECTED_METACARD_ID = "abc123";

    public static final String EXPECTED_METACARD_SOURCE_ID = "ddf-1";

    public static final String EXPECTED_CACHE_KEY = EXPECTED_METACARD_SOURCE_ID + "-" + EXPECTED_METACARD_ID;

    private static final Logger LOGGER = LoggerFactory.getLogger(ReliableResourceDownloadManagerTest.class);

    private static String productCacheDirectory;

    private static String productInputFilename;

    private static long expectedFileSize;

    private static String expectedFileContents;

    @Rule
    public MethodRule watchman = new TestWatchman() {
        public void starting(FrameworkMethod method) {
            LOGGER.debug("***************************  STARTING: {}  **************************\n",
                    method.getName());
        }

        public void finished(FrameworkMethod method) {
            LOGGER.debug("***************************  END: {}  **************************\n", method.getName());
        }
    };

    private ResourceCache resourceCache;

    private DownloadsStatusEventPublisher eventPublisher;

    private DownloadsStatusEventListener eventListener;

    private ReliableResourceDownloadManager downloadMgr;

    private ResourceRequest resourceRequest;

    private ResourceResponse resourceResponse;

    private Resource resource;

    private MockInputStream mis;

    private InputStream productInputStream;

    private ExecutorService executor;

    private Future<ByteArrayOutputStream> future;

    private DownloadStatusInfo downloadStatusInfo;

    @BeforeClass
    public static void oneTimeSetup() throws IOException {
        String workingDir = System.getProperty("user.dir");
        productCacheDirectory = workingDir + "/target/tests/product-cache";
        productInputFilename = workingDir + "/src/test/resources/foo_10_lines.txt";
        File productInputFile = new File(productInputFilename);
        expectedFileSize = productInputFile.length();
        expectedFileContents = FileUtils.readFileToString(productInputFile);
    }

    @Before
    public void setup() {
        resourceCache = mock(ResourceCache.class);
        when(resourceCache.getProductCacheDirectory()).thenReturn(productCacheDirectory);
        eventPublisher = mock(DownloadsStatusEventPublisher.class);
        eventListener = mock(DownloadsStatusEventListener.class);
        downloadStatusInfo = new DownloadStatusInfoImpl();

        downloadMgr = new ReliableResourceDownloadManager(resourceCache, eventPublisher, eventListener,
                downloadStatusInfo, Executors.newSingleThreadExecutor());

    }

    @Test(expected = DownloadException.class)
    public void testDownloadWithNullMetacard() throws Exception {
        resourceRequest = mock(ResourceRequest.class);
        ResourceRetriever retriever = mock(ResourceRetriever.class);

        downloadMgr.download(resourceRequest, null, retriever);
    }

    @Test(expected = DownloadException.class)
    public void testDownloadWithEmptyMetacardId() throws Exception {
        Metacard metacard = getMockMetacard("", EXPECTED_METACARD_SOURCE_ID);
        resourceRequest = mock(ResourceRequest.class);
        ResourceRetriever retriever = mock(ResourceRetriever.class);

        downloadMgr.download(resourceRequest, metacard, retriever);
    }

    @Test(expected = DownloadException.class)
    public void testDownloadWithNullResourceRetriever() throws Exception {
        Metacard metacard = getMockMetacard(EXPECTED_METACARD_ID, EXPECTED_METACARD_SOURCE_ID);
        resourceRequest = mock(ResourceRequest.class);

        downloadMgr.download(resourceRequest, metacard, null);
    }

    @Test(expected = DownloadException.class)
    public void testDownloadWithNullResourceRequest() throws Exception {
        Metacard metacard = getMockMetacard(EXPECTED_METACARD_ID, EXPECTED_METACARD_SOURCE_ID);
        ResourceRetriever retriever = mock(ResourceRetriever.class);

        downloadMgr.download(null, metacard, retriever);
    }

    @Test(expected = DownloadException.class)
    public void testDownloadResourceNotFound() throws Exception {
        Metacard metacard = getMockMetacard(EXPECTED_METACARD_ID, EXPECTED_METACARD_SOURCE_ID);
        resourceRequest = mock(ResourceRequest.class);
        ResourceRetriever retriever = mock(ResourceRetriever.class);
        when(retriever.retrieveResource()).thenThrow(new ResourceNotFoundException());

        downloadMgr.download(resourceRequest, metacard, retriever);
    }

    @Test(expected = DownloadException.class)
    public void testDownloadResourceNotSupported() throws Exception {
        Metacard metacard = getMockMetacard(EXPECTED_METACARD_ID, EXPECTED_METACARD_SOURCE_ID);
        resourceRequest = mock(ResourceRequest.class);
        ResourceRetriever retriever = mock(ResourceRetriever.class);
        when(retriever.retrieveResource()).thenThrow(new ResourceNotSupportedException());

        downloadMgr.download(resourceRequest, metacard, retriever);
    }

    @Test(expected = DownloadException.class)
    public void testDownloadIOException() throws Exception {
        Metacard metacard = getMockMetacard(EXPECTED_METACARD_ID, EXPECTED_METACARD_SOURCE_ID);
        resourceRequest = mock(ResourceRequest.class);
        ResourceRetriever retriever = mock(ResourceRetriever.class);
        when(retriever.retrieveResource()).thenThrow(new IOException());

        downloadMgr.download(resourceRequest, metacard, retriever);
    }

    @Test
    //@Ignore
    public void testDownloadWithoutCaching() throws Exception {
        mis = new MockInputStream(productInputFilename);
        Metacard metacard = getMockMetacard(EXPECTED_METACARD_ID, EXPECTED_METACARD_SOURCE_ID);
        resourceResponse = getMockResourceResponse();

        ResourceRetriever retriever = mock(ResourceRetriever.class);
        when(retriever.retrieveResource()).thenReturn(resourceResponse);

        int chunkSize = 50;
        downloadMgr.setChunkSize(chunkSize);

        ResourceResponse newResourceResponse = downloadMgr.download(resourceRequest, metacard, retriever);
        assertThat(newResourceResponse, is(notNullValue()));
        productInputStream = newResourceResponse.getResource().getInputStream();
        assertThat(productInputStream, is(instanceOf(ReliableResourceInputStream.class)));

        ByteArrayOutputStream clientBytesRead = clientRead(chunkSize, productInputStream);

        verifyClientBytesRead(clientBytesRead);

        cleanup();
    }

    @Test
    //@Ignore
    public void testDownloadWithCaching() throws Exception {
        mis = new MockInputStream(productInputFilename);
        Metacard metacard = getMockMetacard(EXPECTED_METACARD_ID, EXPECTED_METACARD_SOURCE_ID);
        resourceResponse = getMockResourceResponse();

        ResourceRetriever retriever = mock(ResourceRetriever.class);
        when(retriever.retrieveResource()).thenReturn(resourceResponse);

        CacheKey cacheKey = new CacheKey(metacard, resourceResponse.getRequest());
        String key = cacheKey.generateKey();
        when(resourceCache.isPending(key)).thenReturn(false);

        int chunkSize = 50;
        startDownload(true, chunkSize, false, metacard, retriever);

        ByteArrayOutputStream clientBytesRead = clientRead(chunkSize, productInputStream);

        // Captures the ReliableResource object that should have been put in the ResourceCache's map
        ArgumentCaptor<ReliableResource> argument = ArgumentCaptor.forClass(ReliableResource.class);
        verify(resourceCache).put(argument.capture());

        verifyCaching(argument.getValue(), EXPECTED_CACHE_KEY);

        verifyClientBytesRead(clientBytesRead);

        cleanup();
    }

    /**
     * Verifies that if client is reading from @ReliableResourceInputStream slower than
     * {@link ReliableResourceCallable} is reading from product InputStream and writing to FileBackedOutputStream,
     * that complete product is still successfully downloaded by the client.
     * (This will be the case with CXF and @ReliableResourceCallable)
     *
     * @throws Exception
     */
    @Test
    //@Ignore
    public void testDownloadWithCachingDifferentChunkSizes() throws Exception {
        mis = new MockInputStream(productInputFilename);
        Metacard metacard = getMockMetacard(EXPECTED_METACARD_ID, EXPECTED_METACARD_SOURCE_ID);
        resourceResponse = getMockResourceResponse();

        ResourceRetriever retriever = mock(ResourceRetriever.class);
        when(retriever.retrieveResource()).thenReturn(resourceResponse);

        CacheKey cacheKey = new CacheKey(metacard, resourceResponse.getRequest());
        String key = cacheKey.generateKey();
        when(resourceCache.isPending(key)).thenReturn(false);

        int chunkSize = 50;
        startDownload(true, chunkSize, false, metacard, retriever);

        int clientChunkSize = 2;
        ByteArrayOutputStream clientBytesRead = clientRead(clientChunkSize, productInputStream);

        // Captures the ReliableResource object that should have been put in the ResourceCache's map
        ArgumentCaptor<ReliableResource> argument = ArgumentCaptor.forClass(ReliableResource.class);
        verify(resourceCache).put(argument.capture());

        verifyCaching(argument.getValue(), EXPECTED_CACHE_KEY);

        verifyClientBytesRead(clientBytesRead);

        cleanup();
    }

    /**
     * Test that if an Exception is thrown while reading the product's InputStream that
     * download is interrupted, retried and successfully completes on the second attempt.
     *
     * @throws Exception
     */
    @Test
    @Ignore
    public void testStoreWithInputStreamRecoverableErrorCachingDisabled() throws Exception {

        mis = new MockInputStream(productInputFilename);

        Metacard metacard = getMockMetacard(EXPECTED_METACARD_ID, EXPECTED_METACARD_SOURCE_ID);
        resourceResponse = getMockResourceResponse();
        ResourceRetriever retriever = getMockResourceRetrieverWithRetryCapability(
                RetryType.INPUT_STREAM_IO_EXCEPTION);

        int chunkSize = 50;
        startDownload(false, chunkSize, false, metacard, retriever);

        ByteArrayOutputStream clientBytesRead = clientRead(chunkSize, productInputStream);

        // Verifies client read same contents as product input file
        verifyClientBytesRead(clientBytesRead);

        cleanup();
    }

    /**
     * Test that if an Exception is thrown while reading the product's InputStream that
     * download and caching is interrupted, retried and both successfully complete on the second attempt.
     *
     * @throws Exception
     */
    @Test
    @Ignore
    public void testStoreWithInputStreamRecoverableErrorCachingEnabled() throws Exception {

        mis = new MockInputStream(productInputFilename);

        Metacard metacard = getMockMetacard(EXPECTED_METACARD_ID, EXPECTED_METACARD_SOURCE_ID);
        resourceResponse = getMockResourceResponse();
        ResourceRetriever retriever = getMockResourceRetrieverWithRetryCapability(
                RetryType.INPUT_STREAM_IO_EXCEPTION);

        int chunkSize = 50;
        startDownload(true, chunkSize, false, metacard, retriever);

        ByteArrayOutputStream clientBytesRead = clientRead(chunkSize, productInputStream);

        // Captures the ReliableResource object that should have been put in the ResourceCache's map
        ArgumentCaptor<ReliableResource> argument = ArgumentCaptor.forClass(ReliableResource.class);
        verify(resourceCache).put(argument.capture());

        verifyCaching(argument.getValue(), EXPECTED_CACHE_KEY);

        // Verifies client read same contents as product input file
        verifyClientBytesRead(clientBytesRead);

        cleanup();
    }

    /**
     * Test storing product in cache and one of the chunks being stored takes too long, triggering
     * the CacheMonitor to interrupt the caching. Verify that caching is retried and successfully
     * completes on the second attempt.
     *
     * @throws Exception
     */
    @Test
    //@Ignore
    public void testStoreWithTimeoutExceptionCachingEnabled() throws Exception {

        mis = new MockInputStream(productInputFilename);
        Metacard metacard = getMockMetacard(EXPECTED_METACARD_ID, EXPECTED_METACARD_SOURCE_ID);
        resourceResponse = getMockResourceResponse();
        ResourceRetriever retriever = getMockResourceRetrieverWithRetryCapability(RetryType.TIMEOUT_EXCEPTION);

        int chunkSize = 50;
        startDownload(true, chunkSize, false, metacard, retriever);

        ByteArrayOutputStream clientBytesRead = clientRead(chunkSize, productInputStream);

        // Captures the ReliableResource object that should have been put in the ResourceCache's map
        ArgumentCaptor<ReliableResource> argument = ArgumentCaptor.forClass(ReliableResource.class);
        verify(resourceCache).put(argument.capture());

        verifyCaching(argument.getValue(), EXPECTED_CACHE_KEY);

        verifyClientBytesRead(clientBytesRead);

        cleanup();
    }

    /**
     * Tests that if user/client cancels a product retrieval that is in progress and
     * actively being cached, and the admin has configured caching to continue,
     * that the caching continues and cached file is placed in the cache map.
     *
     * @throws Exception
     */
    @Test
    @Ignore
    public void testClientCancelProductDownloadCachingContinues() throws Exception {

        mis = new MockInputStream(productInputFilename);
        Metacard metacard = getMockMetacard(EXPECTED_METACARD_ID, EXPECTED_METACARD_SOURCE_ID);
        resourceResponse = getMockResourceResponse();

        ResourceRetriever retriever = getMockResourceRetrieverWithRetryCapability(
                RetryType.CLIENT_CANCELS_DOWNLOAD);

        int chunkSize = 50;
        startDownload(true, chunkSize, true, metacard, retriever);

        // On second read of ReliableResourceInputStream, client will close the stream simulating a cancel
        // of the product download
        clientRead(chunkSize, productInputStream, 2);

        // Captures the ReliableResource object that should have been put in the ResourceCache's map
        ArgumentCaptor<ReliableResource> argument = ArgumentCaptor.forClass(ReliableResource.class);
        verify(resourceCache, timeout(3000)).put(argument.capture());

        verifyCaching(argument.getValue(), EXPECTED_CACHE_KEY);

        cleanup();
    }

    /**
     * Tests that if user/client cancels a product retrieval that is in progress and
     * actively being cached, and the admin has not configured caching to continue,
     * that the (partially) cached file is deleted and not placed in the cache map.
     *
     * @throws Exception
     */
    @Test
    @Ignore
    public void testClientCancelProductDownloadCachingStops() throws Exception {

        mis = new MockInputStream(productInputFilename, true);
        mis.setReadDelay(MONITOR_PERIOD - 2, TimeUnit.MILLISECONDS);

        Metacard metacard = getMockMetacard(EXPECTED_METACARD_ID, EXPECTED_METACARD_SOURCE_ID);
        resourceResponse = getMockResourceResponse();

        // Need the product InputStream (MockInputStream) to read slower so that client has time to
        // start reading from the ReliableResourceInputStream and close it, simulating a cancel of
        // the product download
        ResourceRetriever retriever = getMockResourceRetrieverWithRetryCapability(RetryType.CLIENT_CANCELS_DOWNLOAD,
                true);

        int chunkSize = 50;
        startDownload(true, chunkSize, false, metacard, retriever);

        // On second read of ReliableResourceInputStream, client will close the stream simulating a cancel
        // of the product download
        clientRead(chunkSize, productInputStream, 2);

        // Verify product was not cached, i.e., its pending caching entry was removed
        String cacheKey = new CacheKey(metacard, resourceResponse.getRequest()).generateKey();
        verify(resourceCache, timeout(3000)).removePendingCacheEntry(cacheKey);

        cleanup();
    }

    /**
     * Tests that if network connection drops repeatedly during a product retrieval such that all
     * of the retry attempts are used up before entire product is downloaded, then the (partially) cached file is deleted and not
     * placed in the cache map, and product download fails.
     *
     * @throws Exception
     */
    @Test
    @Ignore
    public void testRetryAttemptsExhaustedDuringProductDownload() throws Exception {

        mis = new MockInputStream(productInputFilename);
        Metacard metacard = getMockMetacard(EXPECTED_METACARD_ID, EXPECTED_METACARD_SOURCE_ID);
        resourceResponse = getMockResourceResponse();

        ResourceRetriever retriever = getMockResourceRetrieverWithRetryCapability(
                RetryType.NETWORK_CONNECTION_UP_AND_DOWN);

        int chunkSize = 50;
        startDownload(true, chunkSize, false, metacard, retriever);

        ByteArrayOutputStream clientBytesRead = clientRead(chunkSize, productInputStream);

        // Verify client did not receive entire product download
        assertTrue(clientBytesRead.size() < expectedFileSize);

        // Verify product was not cached, i.e., its pending caching entry was removed
        String cacheKey = new CacheKey(metacard, resourceResponse.getRequest()).generateKey();
        verify(resourceCache, timeout(3000)).removePendingCacheEntry(cacheKey);

        cleanup();
    }

    /**
     * Tests that if network connection dropped during a product retrieval that is in progress and
     * actively being cached, that the (partially) cached file is deleted and not
     * placed in the cache map.
     *
     * @throws Exception
     */
    @Test
    @Ignore
    public void testNetworkConnectionDroppedDuringProductDownload() throws Exception {

        mis = new MockInputStream(productInputFilename);
        Metacard metacard = getMockMetacard(EXPECTED_METACARD_ID, EXPECTED_METACARD_SOURCE_ID);
        resourceResponse = getMockResourceResponse();

        ResourceRetriever retriever = getMockResourceRetrieverWithRetryCapability(
                RetryType.NETWORK_CONNECTION_DROPPED);

        int chunkSize = 50;
        startDownload(true, chunkSize, false, metacard, retriever);

        ByteArrayOutputStream clientBytesRead = clientRead(chunkSize, productInputStream);

        // Verify client did not receive entire product download
        assertTrue(clientBytesRead.size() < expectedFileSize);

        // Verify product was not cached, i.e., its pending caching entry was removed
        String cacheKey = new CacheKey(metacard, resourceResponse.getRequest()).generateKey();
        verify(resourceCache, timeout(3000)).removePendingCacheEntry(cacheKey);

        cleanup();
    }

    /**
     * Tests that if exception with file being cached to occurs during a product retrieval, then the (partially) cached
     * file is deleted and it is not placed in the cache map, but the product continues to be streamed to the client until
     * the EOF is detected.
     *
     * @throws Exception
     */
    @Test
    @Ignore
    public void testCacheFileExceptionDuringProductDownload() throws Exception {

        // Need the product InputStream (MockInputStream) to read slower so that client has time to
        // start reading from the ReliableResourceInputStream and close the FileOutputStream the
        // download manager is writing to, simulating a cache file exception during
        // the product download
        mis = new MockInputStream(productInputFilename, true);
        mis.setReadDelay(MONITOR_PERIOD - 2, TimeUnit.MILLISECONDS);

        Metacard metacard = getMockMetacard(EXPECTED_METACARD_ID, EXPECTED_METACARD_SOURCE_ID);
        resourceResponse = getMockResourceResponse();

        ResourceRetriever retriever = mock(ResourceRetriever.class);
        when(retriever.retrieveResource()).thenReturn(resourceResponse);

        int chunkSize = 2;
        startDownload(true, chunkSize, false, metacard, retriever);

        // On second chunk read by client it will close the download manager's cache file output stream
        // to simulate a cache file exception that should be detected by the ReliableResourceCallable
        executor = Executors.newCachedThreadPool();
        ProductDownloadClient productDownloadClient = new ProductDownloadClient(productInputStream, chunkSize);
        productDownloadClient.setSimulateCacheFileException(2, downloadMgr);
        future = executor.submit(productDownloadClient);
        ByteArrayOutputStream clientBytesRead = future.get();

        verifyClientBytesRead(clientBytesRead);

        // Verify product was not cached, i.e., its pending caching entry was removed
        String cacheKey = new CacheKey(metacard, resourceResponse.getRequest()).generateKey();
        verify(resourceCache, timeout(3000)).removePendingCacheEntry(cacheKey);

        cleanup();
    }

    /**
     * Tests that if exception with the FileBackedOutputStream being written to and concurrently read by the client occurs
     * during a product retrieval, then the product download to the client is stopped, but the caching of the
     * file continues.
     *
     * @throws Exception
     */
    @Test
    @Ignore
    // Currently Ignored because cannot figure out how to get FileBackedOutputStream (FBOS) to throw exception
    // during product download - this test successfully closes the FBOS, but the ReliableResourceCallable
    // does not seem to detect this and continues to stream successfully to the client.
    public void testStreamToClientExceptionDuringProductDownloadCachingEnabled() throws Exception {

        mis = new MockInputStream(productInputFilename);

        Metacard metacard = getMockMetacard(EXPECTED_METACARD_ID, EXPECTED_METACARD_SOURCE_ID);
        resourceResponse = getMockResourceResponse();

        downloadMgr = new ReliableResourceDownloadManager(resourceCache, eventPublisher, eventListener,
                downloadStatusInfo, Executors.newSingleThreadExecutor());

        // Use small chunk size so download takes long enough for client
        // to have time to simulate FileBackedOutputStream exception
        int chunkSize = 2;
        downloadMgr.setChunkSize(chunkSize);

        ResourceRetriever retriever = mock(ResourceRetriever.class);
        when(retriever.retrieveResource()).thenReturn(resourceResponse);

        ArgumentCaptor<ReliableResource> argument = ArgumentCaptor.forClass(ReliableResource.class);

        ResourceResponse newResourceResponse = downloadMgr.download(resourceRequest, metacard, retriever);
        assertThat(newResourceResponse, is(notNullValue()));
        productInputStream = newResourceResponse.getResource().getInputStream();
        assertThat(productInputStream, is(instanceOf(ReliableResourceInputStream.class)));

        // On second chunk read by client it will close the download manager's cache file output stream
        // to simulate a cache file exception that should be detected by the ReliableResourceCallable
        executor = Executors.newCachedThreadPool();
        ProductDownloadClient productDownloadClient = new ProductDownloadClient(productInputStream, chunkSize);
        productDownloadClient.setSimulateFbosException(chunkSize, downloadMgr);
        future = executor.submit(productDownloadClient);
        ByteArrayOutputStream clientBytesRead = future.get();

        // Verify client did not receive entire product download
        assertTrue(clientBytesRead.size() < expectedFileSize);

        // Captures the ReliableResource object that should have been put in the ResourceCache's map
        verify(resourceCache, timeout(3000)).put(argument.capture());

        verifyCaching(argument.getValue(), EXPECTED_CACHE_KEY);

        cleanup();
    }

    private void startDownload(boolean cacheEnabled, int chunkSize, boolean cacheWhenCanceled, Metacard metacard,
            ResourceRetriever retriever) throws Exception {
        downloadMgr = new ReliableResourceDownloadManager(resourceCache, eventPublisher, eventListener,
                downloadStatusInfo, Executors.newSingleThreadExecutor());
        downloadMgr.setCacheEnabled(cacheEnabled);
        downloadMgr.setChunkSize(chunkSize);
        downloadMgr.setCacheWhenCanceled(cacheWhenCanceled);

        ResourceResponse newResourceResponse = downloadMgr.download(resourceRequest, metacard, retriever);
        assertThat(newResourceResponse, is(notNullValue()));
        productInputStream = newResourceResponse.getResource().getInputStream();
        assertThat(productInputStream, is(instanceOf(ReliableResourceInputStream.class)));
    }

    /////////////////////////////////////////////////////////////////////////////////////////////////////

    private void verifyCaching(ReliableResource reliableResource, String expectedCacheKey) throws IOException {
        assertEquals(expectedCacheKey, reliableResource.getKey());
        byte[] cachedData = reliableResource.getByteArray();
        assertNotNull(cachedData);

        // Verifies cached data read in was same contents as product input file
        assertTrue(cachedData.length == expectedFileSize);
        assertEquals(expectedFileContents, new String(cachedData));

        // Verifies cached file on disk has same contents as product input file
        assertEquals(expectedFileContents, IOUtils.toString(reliableResource.getInputStream()));
    }

    private void verifyClientBytesRead(ByteArrayOutputStream clientBytesRead) {

        // Verifies client read same contents as product input file
        if (clientBytesRead != null) {
            assertThat(clientBytesRead.size(), is(Long.valueOf(expectedFileSize).intValue()));
            assertEquals(expectedFileContents, new String(clientBytesRead.toByteArray()));
        }
    }

    private Metacard getMockMetacard(String id, String source) {

        Metacard metacard = mock(Metacard.class);

        when(metacard.getId()).thenReturn(id);

        when(metacard.getSourceId()).thenReturn(source);

        when(metacard.getMetacardType()).thenReturn(BasicTypes.BASIC_METACARD);

        return metacard;
    }

    private ResourceResponse getMockResourceResponse() throws Exception {
        resourceRequest = mock(ResourceRequest.class);
        Map<String, Serializable> requestProperties = new HashMap<String, Serializable>();
        when(resourceRequest.getPropertyNames()).thenReturn(requestProperties.keySet());

        resource = mock(Resource.class);
        when(resource.getInputStream()).thenReturn(mis);
        when(resource.getName()).thenReturn("test-resource");
        when(resource.getMimeType()).thenReturn(new MimeType("text/plain"));

        resourceResponse = mock(ResourceResponse.class);
        when(resourceResponse.getRequest()).thenReturn(resourceRequest);
        when(resourceResponse.getResource()).thenReturn(resource);
        Map<String, Serializable> responseProperties = new HashMap<String, Serializable>();
        when(resourceResponse.getProperties()).thenReturn(responseProperties);

        return resourceResponse;
    }

    private ResourceRetriever getMockResourceRetrieverWithRetryCapability(final RetryType retryType)
            throws Exception {
        return getMockResourceRetrieverWithRetryCapability(retryType, false);
    }

    private ResourceRetriever getMockResourceRetrieverWithRetryCapability(final RetryType retryType,
            final boolean readSlow) throws Exception {

        // Mocking to support re-retrieval of product when error encountered
        // during caching.
        ResourceRetriever retriever = mock(ResourceRetriever.class);
        when(retriever.retrieveResource()).thenAnswer(new Answer<Object>() {
            int invocationCount = 0;

            public Object answer(InvocationOnMock invocation) throws ResourceNotFoundException {
                // Create new InputStream for retrieving the same product. This
                // simulates re-retrieving the product from the remote source.
                invocationCount++;
                if (readSlow) {
                    mis = new MockInputStream(productInputFilename, true);
                    mis.setReadDelay(MONITOR_PERIOD - 2, TimeUnit.MILLISECONDS);
                } else {
                    mis = new MockInputStream(productInputFilename);
                }

                if (retryType == RetryType.INPUT_STREAM_IO_EXCEPTION) {
                    if (invocationCount == 1) {
                        mis.setInvocationCountToThrowIOException(5);
                    } else {
                        mis.setInvocationCountToThrowIOException(-1);
                    }
                } else if (retryType == RetryType.TIMEOUT_EXCEPTION) {
                    if (invocationCount == 1) {
                        mis.setInvocationCountToTimeout(3);
                        mis.setReadDelay(MONITOR_PERIOD * 2, TimeUnit.SECONDS);
                    } else {
                        mis.setInvocationCountToTimeout(-1);
                        mis.setReadDelay(0, TimeUnit.SECONDS);
                    }
                } else if (retryType == RetryType.NETWORK_CONNECTION_UP_AND_DOWN) {
                    mis.setInvocationCountToThrowIOException(2);
                } else if (retryType == RetryType.NETWORK_CONNECTION_DROPPED) {
                    if (invocationCount == 1) {
                        mis.setInvocationCountToThrowIOException(2);
                    } else {
                        throw new ResourceNotFoundException();
                    }
                }

                // Reset the mock Resource so that it can be reconfigured to return
                // the new InputStream
                reset(resource);
                when(resource.getInputStream()).thenReturn(mis);
                when(resource.getName()).thenReturn("test-resource");
                try {
                    when(resource.getMimeType()).thenReturn(new MimeType("text/plain"));
                } catch (MimeTypeParseException e) {
                }

                // Reset the mock ResourceResponse so that it can be reconfigured to return
                // the new Resource
                reset(resourceResponse);
                when(resourceResponse.getRequest()).thenReturn(resourceRequest);
                when(resourceResponse.getResource()).thenReturn(resource);
                when(resourceResponse.getProperties()).thenReturn(new HashMap<String, Serializable>());

                return resourceResponse;
            }
        });

        ArgumentCaptor<Long> bytesReadArg = ArgumentCaptor.forClass(Long.class);

        // Mocking to support re-retrieval of product when error encountered
        // during caching. This resource retriever supports skipping.
        when(retriever.retrieveResource(anyLong())).thenAnswer(new Answer<Object>() {
            int invocationCount = 0;

            public Object answer(InvocationOnMock invocation) throws ResourceNotFoundException, IOException {
                // Create new InputStream for retrieving the same product. This
                // simulates re-retrieving the product from the remote source.
                invocationCount++;
                if (readSlow) {
                    mis = new MockInputStream(productInputFilename, true);
                    mis.setReadDelay(MONITOR_PERIOD - 2, TimeUnit.MILLISECONDS);
                } else {
                    mis = new MockInputStream(productInputFilename);
                }

                // Skip the number of bytes that have already been read
                Object[] args = invocation.getArguments();
                long bytesToSkip = (Long) args[0];

                mis.skip(bytesToSkip);

                if (retryType == RetryType.INPUT_STREAM_IO_EXCEPTION) {
                    if (invocationCount == 1) {
                        mis.setInvocationCountToThrowIOException(5);
                    } else {
                        mis.setInvocationCountToThrowIOException(-1);
                    }
                } else if (retryType == RetryType.TIMEOUT_EXCEPTION) {
                    if (invocationCount == 1) {
                        mis.setInvocationCountToTimeout(3);
                        mis.setReadDelay(MONITOR_PERIOD * 2, TimeUnit.SECONDS);
                    } else {
                        mis.setInvocationCountToTimeout(-1);
                        mis.setReadDelay(0, TimeUnit.MILLISECONDS);
                    }
                } else if (retryType == RetryType.NETWORK_CONNECTION_UP_AND_DOWN) {
                    mis.setInvocationCountToThrowIOException(2);
                } else if (retryType == RetryType.NETWORK_CONNECTION_DROPPED) {
                    if (invocationCount == 1) {
                        mis.setInvocationCountToThrowIOException(2);
                    } else {
                        throw new ResourceNotFoundException();
                    }
                }

                // Reset the mock Resource so that it can be reconfigured to return
                // the new InputStream
                reset(resource);
                when(resource.getInputStream()).thenReturn(mis);
                when(resource.getName()).thenReturn("test-resource");
                try {
                    when(resource.getMimeType()).thenReturn(new MimeType("text/plain"));
                } catch (MimeTypeParseException e) {
                }

                // Reset the mock ResourceResponse so that it can be reconfigured to return
                // the new Resource
                reset(resourceResponse);
                when(resourceResponse.getRequest()).thenReturn(resourceRequest);
                when(resourceResponse.getResource()).thenReturn(resource);
                Map<String, Serializable> responseProperties = new HashMap<>();
                responseProperties.put("BytesSkipped", true);
                when(resourceResponse.getProperties()).thenReturn(responseProperties);
                when(resourceResponse.containsPropertyName("BytesSkipped")).thenReturn(true);
                when(resourceResponse.getPropertyValue("BytesSkipped")).thenReturn(true);

                return resourceResponse;
            }
        });

        return retriever;
    }

    public ByteArrayOutputStream clientRead(int chunkSize, InputStream is) throws Exception {
        return clientRead(chunkSize, is, -1);
    }

    public ByteArrayOutputStream clientRead(int chunkSize, InputStream is, int simulatedCancelChunkCount)
            throws Exception {
        executor = Executors.newCachedThreadPool();
        ProductDownloadClient productDownloadClient = new ProductDownloadClient(is, chunkSize,
                simulatedCancelChunkCount);
        future = executor.submit(productDownloadClient);
        ByteArrayOutputStream clientBytesRead = future.get();

        return clientBytesRead;
    }

    private void cleanup() {
        try {
            IOUtils.closeQuietly(productInputStream);
            FileUtils.deleteDirectory(new File(productCacheDirectory));
        } catch (IOException e) {
        }
        future.cancel(true);
        executor.shutdownNow();
    }

    private enum RetryType {
        INPUT_STREAM_IO_EXCEPTION, TIMEOUT_EXCEPTION, NETWORK_CONNECTION_UP_AND_DOWN, NETWORK_CONNECTION_DROPPED, CLIENT_CANCELS_DOWNLOAD, CACHE_FILE_EXCEPTION
    }
}