com.spectralogic.ds3client.integration.GetJobManagement_Test.java Source code

Java tutorial

Introduction

Here is the source code for com.spectralogic.ds3client.integration.GetJobManagement_Test.java

Source

/*
 * ******************************************************************************
 *   Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
 *   Licensed under the Apache License, Version 2.0 (the "License"). You may not use
 *   this file except in compliance with the License. A copy of the License is located at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 *   or in the "license" file accompanying this file.
 *   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
 *   CONDITIONS OF ANY KIND, either express or implied. See the License for the
 *   specific language governing permissions and limitations under the License.
 * ****************************************************************************
 */

package com.spectralogic.ds3client.integration;

import com.google.common.collect.Lists;
import com.spectralogic.ds3client.Ds3Client;
import com.spectralogic.ds3client.Ds3ClientImpl;
import com.spectralogic.ds3client.commands.GetObjectRequest;
import com.spectralogic.ds3client.commands.GetObjectResponse;
import com.spectralogic.ds3client.commands.PutObjectRequest;
import com.spectralogic.ds3client.commands.spectrads3.GetBulkJobSpectraS3Request;
import com.spectralogic.ds3client.commands.spectrads3.GetBulkJobSpectraS3Response;
import com.spectralogic.ds3client.commands.spectrads3.GetJobSpectraS3Request;
import com.spectralogic.ds3client.commands.spectrads3.GetJobSpectraS3Response;
import com.spectralogic.ds3client.helpers.ChecksumListener;
import com.spectralogic.ds3client.helpers.DataTransferredListener;
import com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
import com.spectralogic.ds3client.helpers.FailureEventListener;
import com.spectralogic.ds3client.helpers.FileObjectGetter;
import com.spectralogic.ds3client.helpers.FileObjectPutter;

import com.spectralogic.ds3client.helpers.JobPart;
import com.spectralogic.ds3client.helpers.MetadataReceivedListener;
import com.spectralogic.ds3client.helpers.ObjectCompletedListener;

import com.spectralogic.ds3client.helpers.UnrecoverableIOException;
import com.spectralogic.ds3client.helpers.WaitingForChunksListener;
import com.spectralogic.ds3client.helpers.events.FailureEvent;
import com.spectralogic.ds3client.helpers.events.SameThreadEventRunner;
import com.spectralogic.ds3client.helpers.options.ReadJobOptions;
import com.spectralogic.ds3client.helpers.strategy.blobstrategy.BlobStrategy;
import com.spectralogic.ds3client.helpers.strategy.blobstrategy.ChunkAttemptRetryBehavior;
import com.spectralogic.ds3client.helpers.strategy.blobstrategy.ChunkAttemptRetryDelayBehavior;
import com.spectralogic.ds3client.helpers.strategy.blobstrategy.ClientDefinedChunkAttemptRetryDelayBehavior;
import com.spectralogic.ds3client.helpers.strategy.blobstrategy.GetSequentialBlobStrategy;
import com.spectralogic.ds3client.helpers.strategy.blobstrategy.MaxChunkAttemptsRetryBehavior;
import com.spectralogic.ds3client.helpers.strategy.channelstrategy.ChannelStrategy;
import com.spectralogic.ds3client.helpers.strategy.channelstrategy.SequentialFileWriterChannelStrategy;
import com.spectralogic.ds3client.helpers.strategy.transferstrategy.EventDispatcher;
import com.spectralogic.ds3client.helpers.strategy.transferstrategy.EventDispatcherImpl;
import com.spectralogic.ds3client.helpers.strategy.transferstrategy.MaxNumObjectTransferAttemptsDecorator;
import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferMethod;
import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferRetryDecorator;
import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferStrategy;
import com.spectralogic.ds3client.helpers.strategy.transferstrategy.TransferStrategyBuilder;
import com.spectralogic.ds3client.helpers.util.PartialObjectHelpers;
import com.spectralogic.ds3client.integration.test.helpers.ABMTestHelper;
import com.spectralogic.ds3client.integration.test.helpers.Ds3ClientShim;
import com.spectralogic.ds3client.integration.test.helpers.Ds3ClientShimFactory;
import com.spectralogic.ds3client.integration.test.helpers.TempStorageIds;
import com.spectralogic.ds3client.integration.test.helpers.TempStorageUtil;
import com.spectralogic.ds3client.models.BulkObject;
import com.spectralogic.ds3client.models.ChecksumType;
import com.spectralogic.ds3client.models.MasterObjectList;
import com.spectralogic.ds3client.models.Priority;
import com.spectralogic.ds3client.models.bulk.Ds3Object;
import com.spectralogic.ds3client.models.bulk.PartialDs3Object;
import com.spectralogic.ds3client.models.common.Range;
import com.spectralogic.ds3client.networking.Metadata;
import com.spectralogic.ds3client.utils.Platform;
import com.spectralogic.ds3client.utils.ResourceUtils;
import org.apache.commons.io.FileUtils;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assume;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.lang.reflect.InvocationTargetException;
import java.net.URISyntaxException;
import java.nio.ByteBuffer;
import java.nio.channels.SeekableByteChannel;
import java.nio.channels.WritableByteChannel;
import java.nio.file.AccessDeniedException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.security.AccessControlException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;

import static com.spectralogic.ds3client.integration.Util.RESOURCE_BASE_NAME;
import static com.spectralogic.ds3client.integration.Util.deleteAllContents;
import static com.spectralogic.ds3client.integration.Util.deleteBucketContents;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.core.IsNull.notNullValue;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;

public class GetJobManagement_Test {

    private static final Logger LOG = LoggerFactory.getLogger(GetJobManagement_Test.class);

    private static final Ds3Client client = Util.fromEnv();
    private static final Ds3ClientHelpers HELPERS = Ds3ClientHelpers.wrap(client);
    private static final String BUCKET_NAME = "Get_Job_Management_Test";
    private static final String TEST_ENV_NAME = "GetJobManagement_Test";
    private static final String DISK_FULL_MESSAGE = "There is not enough space on the disk";
    private static TempStorageIds envStorageIds;
    private static UUID dataPolicyId;

    @BeforeClass
    public static void startup() throws Exception {
        dataPolicyId = TempStorageUtil.setupDataPolicy(TEST_ENV_NAME, false, ChecksumType.Type.MD5, client);
        envStorageIds = TempStorageUtil.setup(TEST_ENV_NAME, dataPolicyId, client);
        setupBucket(dataPolicyId);
    }

    @AfterClass
    public static void teardown() throws IOException {
        try {
            deleteAllContents(client, BUCKET_NAME);
        } finally {
            TempStorageUtil.teardown(TEST_ENV_NAME, envStorageIds, client);
            client.close();
        }
    }

    /**
     * Creates the test bucket with the specified data policy to prevent cascading test failure
     * when there are multiple data policies
     */
    private static void setupBucket(final UUID dataPolicy) {
        try {
            HELPERS.ensureBucketExists(BUCKET_NAME, dataPolicy);
        } catch (final Exception e) {
            LOG.error("Setting up test environment failed: " + e.getMessage());
        }
    }

    @Before
    public void setupForEachTest() throws Exception {
        LOG.info("Setting up before test.");
        putBigFiles();
        putBeowulf();
    }

    @After
    public void teardownForEachtest() throws IOException {
        LOG.info("Tearing down after test.");
        deleteBucketContents(client, BUCKET_NAME);
    }

    private static void putBeowulf() throws Exception {
        final String book1 = "beowulf.txt";
        final Path objPath1 = ResourceUtils.loadFileResource(RESOURCE_BASE_NAME + book1);
        final Ds3Object obj = new Ds3Object(book1, Files.size(objPath1));
        final Ds3ClientHelpers.Job job = HELPERS.startWriteJob(BUCKET_NAME, Lists.newArrayList(obj));
        final UUID jobId = job.getJobId();
        final SeekableByteChannel book1Channel = new ResourceObjectPutter(RESOURCE_BASE_NAME).buildChannel(book1);
        client.putObject(new PutObjectRequest(BUCKET_NAME, book1, book1Channel, jobId, 0, Files.size(objPath1)));
        ABMTestHelper.waitForJobCachedSizeToBeMoreThanZero(jobId, client, 20);
    }

    @SuppressWarnings("deprecation")
    @Test
    public void nakedS3Get() throws IOException, URISyntaxException, InterruptedException {

        final WritableByteChannel writeChannel = new NullChannel();

        final GetObjectResponse getObjectResponse = client
                .getObject(new GetObjectRequest(BUCKET_NAME, "beowulf.txt", writeChannel));

        assertThat(getObjectResponse, is(notNullValue()));
        assertThat(getObjectResponse.getObjectSize(), is(notNullValue()));
    }

    @Test
    public void createReadJob() throws IOException, InterruptedException, URISyntaxException {

        final Ds3ClientHelpers.Job readJob = HELPERS.startReadJob(BUCKET_NAME,
                Lists.newArrayList(new Ds3Object("beowulf.txt", 10)));

        final GetJobSpectraS3Response jobSpectraS3Response = client
                .getJobSpectraS3(new GetJobSpectraS3Request(readJob.getJobId()));

        assertThat(jobSpectraS3Response.getMasterObjectListResult(), is(notNullValue()));
    }

    @Test
    public void createReadJobWithBigFile() throws IOException, URISyntaxException, NoSuchMethodException,
            IllegalAccessException, InvocationTargetException {
        final String tempPathPrefix = null;
        final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix);

        try {
            final String DIR_NAME = "largeFiles/";
            final String FILE_NAME = "lesmis-copies.txt";

            final Path objPath = ResourceUtils.loadFileResource(DIR_NAME + FILE_NAME);
            final long bookSize = Files.size(objPath);
            final Ds3Object obj = new Ds3Object(FILE_NAME, bookSize);

            final Ds3ClientShim ds3ClientShim = new Ds3ClientShim((Ds3ClientImpl) client);

            final int maxNumBlockAllocationRetries = 1;
            final int maxNumObjectTransferAttempts = 3;
            final Ds3ClientHelpers ds3ClientHelpers = Ds3ClientHelpers.wrap(ds3ClientShim,
                    maxNumBlockAllocationRetries, maxNumObjectTransferAttempts);

            final Ds3ClientHelpers.Job readJob = ds3ClientHelpers.startReadJob(BUCKET_NAME, Arrays.asList(obj));

            final AtomicBoolean dataTransferredEventReceived = new AtomicBoolean(false);
            final AtomicBoolean objectCompletedEventReceived = new AtomicBoolean(false);
            final AtomicBoolean checksumEventReceived = new AtomicBoolean(false);
            final AtomicBoolean metadataEventReceived = new AtomicBoolean(false);
            final AtomicBoolean waitingForChunksEventReceived = new AtomicBoolean(false);
            final AtomicBoolean failureEventReceived = new AtomicBoolean(false);

            readJob.attachDataTransferredListener(new DataTransferredListener() {
                @Override
                public void dataTransferred(final long size) {
                    dataTransferredEventReceived.set(true);
                    assertEquals(bookSize, size);
                }
            });
            readJob.attachObjectCompletedListener(new ObjectCompletedListener() {
                @Override
                public void objectCompleted(final String name) {
                    objectCompletedEventReceived.set(true);
                }
            });
            readJob.attachChecksumListener(new ChecksumListener() {
                @Override
                public void value(final BulkObject obj, final ChecksumType.Type type, final String checksum) {
                    checksumEventReceived.set(true);
                    assertEquals("0feqCQBgdtmmgGs9pB/Huw==", checksum);
                }
            });
            readJob.attachMetadataReceivedListener(new MetadataReceivedListener() {
                @Override
                public void metadataReceived(final String filename, final Metadata metadata) {
                    metadataEventReceived.set(true);
                }
            });
            readJob.attachWaitingForChunksListener(new WaitingForChunksListener() {
                @Override
                public void waiting(final int secondsToWait) {
                    waitingForChunksEventReceived.set(true);
                }
            });
            readJob.attachFailureEventListener(new FailureEventListener() {
                @Override
                public void onFailure(final FailureEvent failureEvent) {
                    failureEventReceived.set(true);
                }
            });

            final GetJobSpectraS3Response jobSpectraS3Response = ds3ClientShim
                    .getJobSpectraS3(new GetJobSpectraS3Request(readJob.getJobId()));

            assertThat(jobSpectraS3Response.getMasterObjectListResult(), is(notNullValue()));

            readJob.transfer(new FileObjectGetter(tempDirectory));

            final File originalFile = ResourceUtils.loadFileResource(DIR_NAME + FILE_NAME).toFile();
            final File fileCopiedFromBP = Paths.get(tempDirectory.toString(), FILE_NAME).toFile();
            assertTrue(FileUtils.contentEquals(originalFile, fileCopiedFromBP));

            assertTrue(dataTransferredEventReceived.get());
            assertTrue(objectCompletedEventReceived.get());
            assertTrue(checksumEventReceived.get());
            assertTrue(metadataEventReceived.get());
            assertFalse(waitingForChunksEventReceived.get());
            assertFalse(failureEventReceived.get());
        } finally {
            FileUtils.deleteDirectory(tempDirectory.toFile());
        }
    }

    private static void putBigFiles() throws IOException, URISyntaxException {
        final String DIR_NAME = "largeFiles/";
        final String[] FILE_NAMES = new String[] { "lesmis.txt", "lesmis-copies.txt", "GreatExpectations.txt" };

        final Path dirPath = ResourceUtils.loadFileResource(DIR_NAME);

        final List<String> bookTitles = new ArrayList<>();
        final List<Ds3Object> objects = new ArrayList<>();
        for (final String book : FILE_NAMES) {
            final Path objPath = ResourceUtils.loadFileResource(DIR_NAME + book);
            final long bookSize = Files.size(objPath);
            final Ds3Object obj = new Ds3Object(book, bookSize);

            bookTitles.add(book);
            objects.add(obj);
        }

        final int maxNumBlockAllocationRetries = 1;
        final int maxNumObjectTransferAttempts = 3;
        final Ds3ClientHelpers ds3ClientHelpers = Ds3ClientHelpers.wrap(client, maxNumBlockAllocationRetries,
                maxNumObjectTransferAttempts);

        final Ds3ClientHelpers.Job writeJob = ds3ClientHelpers.startWriteJob(BUCKET_NAME, objects);
        writeJob.transfer(new FileObjectPutter(dirPath));
    }

    @Test(expected = AccessDeniedException.class)
    public void testReadRetryBugFixWithUnwritableDirectory() throws IOException, URISyntaxException,
            NoSuchMethodException, IllegalAccessException, InvocationTargetException, InterruptedException {
        Assume.assumeFalse(iAmRoot());

        final String tempPathPrefix = null;
        final Path tempDirectoryPath = Files.createTempDirectory(Paths.get("."), tempPathPrefix);

        final String tempDirectoryName = tempDirectoryPath.toString();

        if (org.apache.commons.lang3.SystemUtils.IS_OS_WINDOWS) {
            // Deny write data access to everyone, making the directory unwritable.
            Runtime.getRuntime().exec("icacls " + tempDirectoryName + " /deny Everyone:(WD)").waitFor();
        } else {
            Runtime.getRuntime().exec("chmod -w " + tempDirectoryName).waitFor();
        }

        try {
            disableWritePermissionForRoot(tempDirectoryName);

            final String DIR_NAME = "largeFiles/";
            final String FILE_NAME = "lesmis-copies.txt";

            final Path objPath = ResourceUtils.loadFileResource(DIR_NAME + FILE_NAME);
            final long bookSize = Files.size(objPath);
            final Ds3Object obj = new Ds3Object(FILE_NAME, bookSize);

            final Ds3ClientShim ds3ClientShim = new Ds3ClientShim((Ds3ClientImpl) client);

            final int maxNumBlockAllocationRetries = 1;
            final int maxNumObjectTransferAttempts = 3;
            final Ds3ClientHelpers ds3ClientHelpers = Ds3ClientHelpers.wrap(ds3ClientShim,
                    maxNumBlockAllocationRetries, maxNumObjectTransferAttempts);

            final Ds3ClientHelpers.Job readJob = ds3ClientHelpers.startReadJob(BUCKET_NAME, Arrays.asList(obj));

            final GetJobSpectraS3Response jobSpectraS3Response = ds3ClientShim
                    .getJobSpectraS3(new GetJobSpectraS3Request(readJob.getJobId()));

            assertThat(jobSpectraS3Response.getMasterObjectListResult(), is(notNullValue()));

            readJob.transfer(new FileObjectGetter(tempDirectoryPath));

            final File originalFile = ResourceUtils.loadFileResource(DIR_NAME + FILE_NAME).toFile();
            final File fileCopiedFromBP = Paths.get(tempDirectoryPath.toString(), FILE_NAME).toFile();
            assertTrue(FileUtils.contentEquals(originalFile, fileCopiedFromBP));
        } finally {
            enableWritePermissionForRoot(tempDirectoryName);

            if (org.apache.commons.lang3.SystemUtils.IS_OS_WINDOWS) {
                // Grant write data access to everyone, so we can delete the directory
                Runtime.getRuntime().exec("icacls " + tempDirectoryName + " /grant Everyone:(WD)").waitFor();
            } else {
                Runtime.getRuntime().exec("chmod +w " + tempDirectoryName).waitFor();
            }
            FileUtils.deleteDirectory(tempDirectoryPath.toFile());
        }
    }

    private void disableWritePermissionForRoot(final String fileOrDirName) {
        try {
            if (iAmRoot()) {
                Runtime.getRuntime().exec("chattr +i " + fileOrDirName).waitFor();
            }
        } catch (final IOException | InterruptedException e) {
            LOG.error("Error setting file immutable: " + fileOrDirName, e);
        }
    }

    private boolean iAmRoot() {
        if (!Platform.isLinux()) {
            return false;
        }

        BufferedReader idProcessStdOut = null;

        try {
            final Process idProcess = Runtime.getRuntime().exec("id");
            idProcessStdOut = new BufferedReader(new InputStreamReader(idProcess.getInputStream()));
            final String idProcessResult = idProcessStdOut.readLine();
            idProcess.waitFor();

            final String[] idFields = idProcessResult.split("=");

            final String[] uidString = idFields[1].split("\\(");

            return uidString[0].equals("0");
        } catch (final IOException | InterruptedException e) {
            LOG.error("Error getting user id.", e);
        } finally {
            try {
                if (idProcessStdOut != null) {
                    idProcessStdOut.close();
                }
            } catch (final IOException e) {
                LOG.error("Failure closing stdout for process that gets user id.");
            }
        }

        return false;
    }

    private void enableWritePermissionForRoot(final String fileOrDirName) {
        try {
            if (iAmRoot()) {
                Runtime.getRuntime().exec("chattr -i " + fileOrDirName).waitFor();
            }
        } catch (final IOException | InterruptedException e) {
            LOG.error("Error setting file immutable: " + fileOrDirName, e);
        }
    }

    @Test
    public void testReadRetrybugWhenChannelThrowsAccessException() throws IOException, URISyntaxException,
            NoSuchMethodException, IllegalAccessException, InvocationTargetException {
        final String tempPathPrefix = null;
        final Path tempDirectoryPath = Files.createTempDirectory(Paths.get("."), tempPathPrefix);

        final AtomicBoolean caughtException = new AtomicBoolean(false);

        try {
            final String DIR_NAME = "largeFiles/";
            final String FILE_NAME = "lesmis-copies.txt";

            final Path objPath = ResourceUtils.loadFileResource(DIR_NAME + FILE_NAME);
            final long bookSize = Files.size(objPath);
            final Ds3Object obj = new Ds3Object(FILE_NAME, bookSize);

            final Ds3ClientShim ds3ClientShim = new Ds3ClientShim((Ds3ClientImpl) client);

            final int maxNumBlockAllocationRetries = 1;
            final int maxNumObjectTransferAttempts = 3;
            final Ds3ClientHelpers ds3ClientHelpers = Ds3ClientHelpers.wrap(ds3ClientShim,
                    maxNumBlockAllocationRetries, maxNumObjectTransferAttempts);

            final Ds3ClientHelpers.Job readJob = ds3ClientHelpers.startReadJob(BUCKET_NAME, Arrays.asList(obj));

            final GetJobSpectraS3Response jobSpectraS3Response = ds3ClientShim
                    .getJobSpectraS3(new GetJobSpectraS3Request(readJob.getJobId()));

            assertThat(jobSpectraS3Response.getMasterObjectListResult(), is(notNullValue()));

            readJob.transfer(new Ds3ClientHelpers.ObjectChannelBuilder() {
                @Override
                public SeekableByteChannel buildChannel(final String key) throws IOException {
                    throw new AccessControlException(key);
                }
            });
        } catch (final IOException e) {
            caughtException.set(true);
            assertTrue(e.getCause() instanceof AccessControlException);
        } finally {
            FileUtils.deleteDirectory(tempDirectoryPath.toFile());
        }

        assertTrue(caughtException.get());
    }

    @Test
    public void testReadRetryBugWhenDiskIsFull() throws IOException, URISyntaxException, NoSuchMethodException,
            IllegalAccessException, InvocationTargetException {
        final String tempPathPrefix = null;
        final Path tempDirectoryPath = Files.createTempDirectory(Paths.get("."), tempPathPrefix);

        try {
            final String DIR_NAME = "largeFiles/";
            final String FILE_NAME = "lesmis-copies.txt";

            final Path objPath = ResourceUtils.loadFileResource(DIR_NAME + FILE_NAME);
            final long bookSize = Files.size(objPath);
            final Ds3Object obj = new Ds3Object(FILE_NAME, bookSize);

            final Ds3ClientShim ds3ClientShim = new Ds3ClientShim((Ds3ClientImpl) client);

            final int maxNumBlockAllocationRetries = 1;
            final int maxNumObjectTransferAttempts = 3;
            final Ds3ClientHelpers ds3ClientHelpers = Ds3ClientHelpers.wrap(ds3ClientShim,
                    maxNumBlockAllocationRetries, maxNumObjectTransferAttempts);

            final Ds3ClientHelpers.Job readJob = ds3ClientHelpers.startReadJob(BUCKET_NAME, Arrays.asList(obj));

            final GetJobSpectraS3Response jobSpectraS3Response = ds3ClientShim
                    .getJobSpectraS3(new GetJobSpectraS3Request(readJob.getJobId()));

            assertThat(jobSpectraS3Response.getMasterObjectListResult(), is(notNullValue()));

            try {
                readJob.transfer(new FailingChannelBuilder());
            } catch (final UnrecoverableIOException e) {
                assertEquals(DISK_FULL_MESSAGE, e.getCause().getMessage());
            }
        } finally {
            FileUtils.deleteDirectory(tempDirectoryPath.toFile());
        }
    }

    private static class FailingChannelBuilder implements Ds3ClientHelpers.ObjectChannelBuilder {
        @Override
        public SeekableByteChannel buildChannel(final String key) throws IOException {
            final String filePath = key;
            final SeekableByteChannel seekableByteChannel = Files.newByteChannel(Paths.get(filePath),
                    StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE,
                    StandardOpenOption.DELETE_ON_CLOSE);
            return new SeekableByteChannelWrapper(seekableByteChannel);
        }
    }

    private static class SeekableByteChannelWrapper implements SeekableByteChannel {
        private final SeekableByteChannel seekableByteChannel;

        private SeekableByteChannelWrapper(final SeekableByteChannel seekableByteChannel) {
            this.seekableByteChannel = seekableByteChannel;
        }

        @Override
        public int read(final ByteBuffer dst) throws IOException {
            return seekableByteChannel.read(dst);
        }

        @Override
        public int write(final ByteBuffer src) throws IOException {
            throw new IOException(DISK_FULL_MESSAGE);
        }

        @Override
        public long position() throws IOException {
            return seekableByteChannel.position();
        }

        @Override
        public SeekableByteChannel position(final long newPosition) throws IOException {
            return seekableByteChannel.position(newPosition);
        }

        @Override
        public long size() throws IOException {
            return seekableByteChannel.size();
        }

        @Override
        public SeekableByteChannel truncate(final long size) throws IOException {
            return seekableByteChannel.truncate(size);
        }

        @Override
        public boolean isOpen() {
            return seekableByteChannel.isOpen();
        }

        @Override
        public void close() throws IOException {
            seekableByteChannel.close();
        }
    }

    @Test
    public void createReadJobWithPriorityOption() throws IOException, InterruptedException, URISyntaxException {

        final Ds3ClientHelpers.Job readJob = HELPERS.startReadJob(BUCKET_NAME,
                Lists.newArrayList(new Ds3Object("beowulf.txt", 10)),
                ReadJobOptions.create().withPriority(Priority.LOW));
        final GetJobSpectraS3Response jobSpectraS3Response = client
                .getJobSpectraS3(new GetJobSpectraS3Request(readJob.getJobId()));

        assertThat(jobSpectraS3Response.getMasterObjectListResult().getPriority(), is(Priority.LOW));
    }

    @Test
    public void testCreatingStreamedReadJobWithPriorityOption()
            throws IOException, InterruptedException, URISyntaxException {

        final Ds3ClientHelpers.Job readJob = HELPERS.startReadJobUsingStreamedBehavior(BUCKET_NAME,
                Lists.newArrayList(new Ds3Object("beowulf.txt", 10)),
                ReadJobOptions.create().withPriority(Priority.LOW));
        final GetJobSpectraS3Response jobSpectraS3Response = client
                .getJobSpectraS3(new GetJobSpectraS3Request(readJob.getJobId()));

        assertThat(jobSpectraS3Response.getMasterObjectListResult().getPriority(), is(Priority.LOW));
    }

    @Test
    public void createReadJobWithNameOption() throws IOException, URISyntaxException, InterruptedException {

        final Ds3ClientHelpers.Job readJob = HELPERS.startReadJob(BUCKET_NAME,
                Lists.newArrayList(new Ds3Object("beowulf.txt", 10)), ReadJobOptions.create().withName("test_job"));
        final GetJobSpectraS3Response jobSpectraS3Response = client
                .getJobSpectraS3(new GetJobSpectraS3Request(readJob.getJobId()));

        assertThat(jobSpectraS3Response.getMasterObjectListResult().getName(), is("test_job"));
    }

    @Test
    public void testCreatingStreamedReadJobWithNameOption()
            throws IOException, URISyntaxException, InterruptedException {

        final Ds3ClientHelpers.Job readJob = HELPERS.startReadJobUsingStreamedBehavior(BUCKET_NAME,
                Lists.newArrayList(new Ds3Object("beowulf.txt", 10)), ReadJobOptions.create().withName("test_job"));
        final GetJobSpectraS3Response jobSpectraS3Response = client
                .getJobSpectraS3(new GetJobSpectraS3Request(readJob.getJobId()));

        assertThat(jobSpectraS3Response.getMasterObjectListResult().getName(), is("test_job"));
    }

    @Test
    public void createReadJobWithNameAndPriorityOptions()
            throws IOException, URISyntaxException, InterruptedException {

        final Ds3ClientHelpers.Job readJob = HELPERS.startReadJob(BUCKET_NAME,
                Lists.newArrayList(new Ds3Object("beowulf.txt", 10)),
                ReadJobOptions.create().withName("test_job").withPriority(Priority.LOW));
        final GetJobSpectraS3Response jobSpectraS3Response = client
                .getJobSpectraS3(new GetJobSpectraS3Request(readJob.getJobId()));

        assertThat(jobSpectraS3Response.getMasterObjectListResult().getName(), is("test_job"));
        assertThat(jobSpectraS3Response.getMasterObjectListResult().getPriority(), is(Priority.LOW));
    }

    @Test
    public void testCreatingStreamedReadJobWithNameAndPriorityOption()
            throws IOException, URISyntaxException, InterruptedException {

        final Ds3ClientHelpers.Job readJob = HELPERS.startReadJobUsingStreamedBehavior(BUCKET_NAME,
                Lists.newArrayList(new Ds3Object("beowulf.txt", 10)),
                ReadJobOptions.create().withName("test_job").withPriority(Priority.LOW));
        final GetJobSpectraS3Response jobSpectraS3Response = client
                .getJobSpectraS3(new GetJobSpectraS3Request(readJob.getJobId()));

        assertThat(jobSpectraS3Response.getMasterObjectListResult().getName(), is("test_job"));
        assertThat(jobSpectraS3Response.getMasterObjectListResult().getPriority(), is(Priority.LOW));
    }

    @Test
    public void testPartialRetriesWithInjectedFailures() throws NoSuchMethodException, IllegalAccessException,
            InvocationTargetException, IOException, URISyntaxException {
        final String tempPathPrefix = null;
        final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix);

        try {
            final List<Ds3Object> filesToGet = new ArrayList<>();

            final String DIR_NAME = "largeFiles/";
            final String FILE_NAME = "GreatExpectations.txt";

            final int offsetIntoFirstRange = 10;

            filesToGet.add(new PartialDs3Object(FILE_NAME, Range.byLength(200000, 100000)));

            filesToGet.add(new PartialDs3Object(FILE_NAME, Range.byLength(100000, 100000)));

            filesToGet.add(new PartialDs3Object(FILE_NAME, Range.byLength(offsetIntoFirstRange, 100000)));

            final Ds3ClientShim ds3ClientShim = new Ds3ClientShim((Ds3ClientImpl) client);

            final int maxNumBlockAllocationRetries = 1;
            final int maxNumObjectTransferAttempts = 4;
            final Ds3ClientHelpers ds3ClientHelpers = Ds3ClientHelpers.wrap(ds3ClientShim,
                    maxNumBlockAllocationRetries, maxNumObjectTransferAttempts);

            final Ds3ClientHelpers.Job job = ds3ClientHelpers.startReadJob(BUCKET_NAME, filesToGet);
            final AtomicInteger intValue = new AtomicInteger();

            job.attachObjectCompletedListener(new ObjectCompletedListener() {
                int numPartsCompleted = 0;

                @Override
                public void objectCompleted(final String name) {
                    assertEquals(1, ++numPartsCompleted);
                    intValue.incrementAndGet();
                }
            });

            job.attachDataTransferredListener(new DataTransferredListener() {
                @Override
                public void dataTransferred(final long size) {
                    LOG.info("Data transferred size: {}", size);
                }
            });

            job.transfer(new FileObjectGetter(tempDirectory));

            assertEquals(1, intValue.get());

            try (final InputStream originalFileStream = Thread.currentThread().getContextClassLoader()
                    .getResourceAsStream(DIR_NAME + FILE_NAME)) {
                final byte[] first300000Bytes = new byte[300000 - offsetIntoFirstRange];
                originalFileStream.skip(offsetIntoFirstRange);
                int numBytesRead = originalFileStream.read(first300000Bytes, 0, 300000 - offsetIntoFirstRange);

                assertThat(numBytesRead, is(300000 - offsetIntoFirstRange));

                try (final InputStream fileReadFromBP = Files
                        .newInputStream(Paths.get(tempDirectory.toString(), FILE_NAME))) {
                    final byte[] first300000BytesFromBP = new byte[300000 - offsetIntoFirstRange];

                    numBytesRead = fileReadFromBP.read(first300000BytesFromBP, 0, 300000 - offsetIntoFirstRange);
                    assertThat(numBytesRead, is(300000 - offsetIntoFirstRange));

                    assertTrue(Arrays.equals(first300000Bytes, first300000BytesFromBP));
                }
            }
        } finally {
            FileUtils.deleteDirectory(tempDirectory.toFile());
        }
    }

    @Test
    public void testFiringFailureHandlerWhenGettingChunks() throws URISyntaxException, NoSuchMethodException,
            InvocationTargetException, IllegalAccessException, IOException {
        final String tempPathPrefix = null;
        final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix);

        try {
            final AtomicInteger numFailuresRecorded = new AtomicInteger();

            final FailureEventListener failureEventListener = new FailureEventListener() {
                @Override
                public void onFailure(final FailureEvent failureEvent) {
                    numFailuresRecorded.incrementAndGet();
                    assertEquals(FailureEvent.FailureActivity.GettingObject, failureEvent.doingWhat());
                }
            };

            final Ds3ClientHelpers.Job readJob = createReadJobWithObjectsReadyToTransfer(
                    Ds3ClientShimFactory.ClientFailureType.ChunkAllocation);

            readJob.attachFailureEventListener(failureEventListener);

            try {
                readJob.transfer(new FileObjectGetter(tempDirectory));
            } catch (final IOException e) {
                assertEquals(1, numFailuresRecorded.get());
            }
        } finally {
            FileUtils.deleteDirectory(tempDirectory.toFile());
        }
    }

    private Ds3ClientHelpers.Job createReadJobWithObjectsReadyToTransfer(
            final Ds3ClientShimFactory.ClientFailureType clientFailureType) throws IOException, URISyntaxException,
            NoSuchMethodException, IllegalAccessException, InvocationTargetException {
        final String DIR_NAME = "largeFiles/";
        final String FILE_NAME = "lesmis-copies.txt";

        final Path objPath = ResourceUtils.loadFileResource(DIR_NAME + FILE_NAME);
        final long bookSize = Files.size(objPath);
        final Ds3Object obj = new Ds3Object(FILE_NAME, bookSize);

        final Ds3Client ds3Client = Ds3ClientShimFactory.makeWrappedDs3Client(clientFailureType, client);

        final int maxNumBlockAllocationRetries = 3;
        final int maxNumObjectTransferAttempts = 3;
        final Ds3ClientHelpers ds3ClientHelpers = Ds3ClientHelpers.wrap(ds3Client, maxNumBlockAllocationRetries,
                maxNumObjectTransferAttempts);

        final Ds3ClientHelpers.Job readJob = ds3ClientHelpers.startReadJob(BUCKET_NAME, Arrays.asList(obj));

        return readJob;
    }

    @Test
    public void testFiringFailureHandlerWhenGettingObject() throws URISyntaxException, NoSuchMethodException,
            InvocationTargetException, IllegalAccessException, IOException {
        final String tempPathPrefix = null;
        final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix);

        try {
            final AtomicInteger numFailuresRecorded = new AtomicInteger();

            final FailureEventListener failureEventListener = new FailureEventListener() {
                @Override
                public void onFailure(final FailureEvent failureEvent) {
                    numFailuresRecorded.incrementAndGet();
                    assertEquals(FailureEvent.FailureActivity.GettingObject, failureEvent.doingWhat());
                }
            };

            final Ds3ClientHelpers.Job readJob = createReadJobWithObjectsReadyToTransfer(
                    Ds3ClientShimFactory.ClientFailureType.GetObject);

            readJob.attachFailureEventListener(failureEventListener);

            try {
                readJob.transfer(new FileObjectGetter(tempDirectory));
            } catch (final IOException e) {
                assertEquals(1, numFailuresRecorded.get());
            }
        } finally {
            FileUtils.deleteDirectory(tempDirectory.toFile());
        }
    }

    @Test
    public void testCreatingReadJobWithStreamedBehavior() throws IOException, URISyntaxException,
            NoSuchMethodException, IllegalAccessException, InvocationTargetException {
        doReadJobWithJobStarter(new ReadJobStarter() {
            @Override
            public Ds3ClientHelpers.Job startReadJob(final Ds3ClientHelpers ds3ClientHelpers,
                    final String bucketName, final Iterable<Ds3Object> objectsToread) throws IOException {
                return ds3ClientHelpers.startReadJobUsingStreamedBehavior(BUCKET_NAME, objectsToread);
            }
        });
    }

    private interface ReadJobStarter {
        Ds3ClientHelpers.Job startReadJob(final Ds3ClientHelpers ds3ClientHelpers, final String bucketName,
                Iterable<Ds3Object> objectsToread) throws IOException;
    }

    private void doReadJobWithJobStarter(final ReadJobStarter readJobStarter) throws IOException,
            URISyntaxException, NoSuchMethodException, IllegalAccessException, InvocationTargetException {
        final String tempPathPrefix = null;
        final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix);

        try {
            final String DIR_NAME = "largeFiles/";
            final String FILE_NAME = "lesmis.txt";

            final Path objPath = ResourceUtils.loadFileResource(DIR_NAME + FILE_NAME);
            final long bookSize = Files.size(objPath);
            final Ds3Object obj = new Ds3Object(FILE_NAME, bookSize);

            final Ds3ClientShim ds3ClientShim = new Ds3ClientShim((Ds3ClientImpl) client);

            final int maxNumBlockAllocationRetries = 1;
            final int maxNumObjectTransferAttempts = 3;
            final Ds3ClientHelpers ds3ClientHelpers = Ds3ClientHelpers.wrap(ds3ClientShim,
                    maxNumBlockAllocationRetries, maxNumObjectTransferAttempts);

            final Ds3ClientHelpers.Job readJob = readJobStarter.startReadJob(ds3ClientHelpers, BUCKET_NAME,
                    Arrays.asList(obj));

            final AtomicBoolean dataTransferredEventReceived = new AtomicBoolean(false);
            final AtomicBoolean objectCompletedEventReceived = new AtomicBoolean(false);
            final AtomicBoolean checksumEventReceived = new AtomicBoolean(false);
            final AtomicBoolean metadataEventReceived = new AtomicBoolean(false);
            final AtomicBoolean waitingForChunksEventReceived = new AtomicBoolean(false);
            final AtomicBoolean failureEventReceived = new AtomicBoolean(false);

            readJob.attachDataTransferredListener(new DataTransferredListener() {
                @Override
                public void dataTransferred(final long size) {
                    dataTransferredEventReceived.set(true);
                    assertEquals(bookSize, size);
                }
            });
            readJob.attachObjectCompletedListener(new ObjectCompletedListener() {
                @Override
                public void objectCompleted(final String name) {
                    objectCompletedEventReceived.set(true);
                }
            });
            readJob.attachChecksumListener(new ChecksumListener() {
                @Override
                public void value(final BulkObject obj, final ChecksumType.Type type, final String checksum) {
                    checksumEventReceived.set(true);
                    assertEquals("69+JXWeZuzl2HFTM6Lbo8A==", checksum);
                }
            });
            readJob.attachMetadataReceivedListener(new MetadataReceivedListener() {
                @Override
                public void metadataReceived(final String filename, final Metadata metadata) {
                    metadataEventReceived.set(true);
                }
            });
            readJob.attachWaitingForChunksListener(new WaitingForChunksListener() {
                @Override
                public void waiting(final int secondsToWait) {
                    waitingForChunksEventReceived.set(true);
                }
            });
            readJob.attachFailureEventListener(new FailureEventListener() {
                @Override
                public void onFailure(final FailureEvent failureEvent) {
                    failureEventReceived.set(true);
                }
            });

            readJob.transfer(new FileObjectGetter(tempDirectory));

            final File originalFile = ResourceUtils.loadFileResource(DIR_NAME + FILE_NAME).toFile();
            final File fileCopiedFromBP = Paths.get(tempDirectory.toString(), FILE_NAME).toFile();
            assertTrue(FileUtils.contentEquals(originalFile, fileCopiedFromBP));

            assertTrue(dataTransferredEventReceived.get());
            assertTrue(objectCompletedEventReceived.get());
            assertTrue(checksumEventReceived.get());
            assertTrue(metadataEventReceived.get());
            assertFalse(waitingForChunksEventReceived.get());
            assertFalse(failureEventReceived.get());
        } finally {
            FileUtils.deleteDirectory(tempDirectory.toFile());
        }
    }

    @Test
    public void testCreatingReadJobWithRandomAccessBehavior() throws IOException, URISyntaxException,
            NoSuchMethodException, IllegalAccessException, InvocationTargetException {
        doReadJobWithJobStarter(new ReadJobStarter() {
            @Override
            public Ds3ClientHelpers.Job startReadJob(final Ds3ClientHelpers ds3ClientHelpers,
                    final String bucketName, final Iterable<Ds3Object> objectsToread) throws IOException {
                return ds3ClientHelpers.startReadJobUsingRandomAccessBehavior(BUCKET_NAME, objectsToread);
            }
        });
    }

    @Test
    public void testStartReadAllJobUsingStreamedBehavior() throws IOException {
        final String tempPathPrefix = null;
        final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix);

        try {
            final AtomicInteger numFailuresRecorded = new AtomicInteger(0);

            final FailureEventListener failureEventListener = new FailureEventListener() {
                @Override
                public void onFailure(final FailureEvent failureEvent) {
                    numFailuresRecorded.incrementAndGet();
                    assertEquals(FailureEvent.FailureActivity.GettingObject, failureEvent.doingWhat());
                }
            };

            final Ds3ClientHelpers.Job readJob = HELPERS.startReadAllJobUsingStreamedBehavior(BUCKET_NAME);
            readJob.attachFailureEventListener(failureEventListener);
            readJob.transfer(new FileObjectGetter(tempDirectory));

            final Collection<File> filesInTempDirectory = FileUtils.listFiles(tempDirectory.toFile(), null, false);

            final List<String> filesWeExpectToBeInTempDirectory = Arrays.asList("beowulf.txt", "lesmis.txt",
                    "lesmis-copies.txt", "GreatExpectations.txt");

            for (final File fileInTempDirectory : filesInTempDirectory) {
                assertTrue(filesWeExpectToBeInTempDirectory.contains(fileInTempDirectory.getName()));
            }

            assertEquals(0, numFailuresRecorded.get());
        } finally {
            FileUtils.deleteDirectory(tempDirectory.toFile());
        }
    }

    @Test
    public void testStartReadAllJobUsingRandomAccessBehavior() throws IOException {
        final String tempPathPrefix = null;
        final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix);

        try {
            final AtomicInteger numFailuresRecorded = new AtomicInteger(0);

            final FailureEventListener failureEventListener = new FailureEventListener() {
                @Override
                public void onFailure(final FailureEvent failureEvent) {
                    numFailuresRecorded.incrementAndGet();
                    assertEquals(FailureEvent.FailureActivity.GettingObject, failureEvent.doingWhat());
                }
            };

            final Ds3ClientHelpers.Job readJob = HELPERS.startReadAllJobUsingRandomAccessBehavior(BUCKET_NAME);
            readJob.attachFailureEventListener(failureEventListener);
            readJob.transfer(new FileObjectGetter(tempDirectory));

            final Collection<File> filesInTempDirectory = FileUtils.listFiles(tempDirectory.toFile(), null, false);

            final List<String> filesWeExpectToBeInTempDirectory = Arrays.asList("beowulf.txt", "lesmis.txt",
                    "lesmis-copies.txt", "GreatExpectations.txt");

            for (final File fileInTempDirectory : filesInTempDirectory) {
                assertTrue(filesWeExpectToBeInTempDirectory.contains(fileInTempDirectory.getName()));
            }

            assertEquals(0, numFailuresRecorded.get());
        } finally {
            FileUtils.deleteDirectory(tempDirectory.toFile());
        }
    }

    @Test
    public void testGetJobUsingTransferStrategy() throws IOException, InterruptedException {
        final String tempPathPrefix = null;
        final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix);
        final String fileName = "beowulf.txt";

        try {
            final List<Ds3Object> objects = Lists.newArrayList(new Ds3Object(fileName));

            final GetBulkJobSpectraS3Request getBulkJobSpectraS3Request = new GetBulkJobSpectraS3Request(
                    BUCKET_NAME, objects);

            final GetBulkJobSpectraS3Response getBulkJobSpectraS3Response = client
                    .getBulkJobSpectraS3(getBulkJobSpectraS3Request);

            final MasterObjectList masterObjectList = getBulkJobSpectraS3Response.getMasterObjectList();

            final TransferStrategyBuilder transferStrategyBuilder = new TransferStrategyBuilder()
                    .withDs3Client(client).withMasterObjectList(masterObjectList)
                    .withChannelBuilder(new FileObjectGetter(tempDirectory))
                    .withRangesForBlobs(PartialObjectHelpers.mapRangesToBlob(masterObjectList.getObjects(),
                            PartialObjectHelpers.getPartialObjectsRanges(objects)));

            final TransferStrategy transferStrategy = transferStrategyBuilder.makeGetTransferStrategy();

            transferStrategy.transfer();

            final Collection<File> filesInTempDirectory = FileUtils.listFiles(tempDirectory.toFile(), null, false);

            for (final File file : filesInTempDirectory) {
                assertEquals(fileName, file.getName());
            }
        } finally {
            FileUtils.deleteDirectory(tempDirectory.toFile());
        }
    }

    @Test
    public void testReadJobUsingTransferStrategy() throws IOException, InterruptedException {
        final String tempPathPrefix = null;
        final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix);
        final String fileName = "beowulf.txt";

        try {
            final List<Ds3Object> objects = Lists.newArrayList(new Ds3Object(fileName));

            final GetBulkJobSpectraS3Request getBulkJobSpectraS3Request = new GetBulkJobSpectraS3Request(
                    BUCKET_NAME, objects);

            final GetBulkJobSpectraS3Response getBulkJobSpectraS3Response = client
                    .getBulkJobSpectraS3(getBulkJobSpectraS3Request);

            final MasterObjectList masterObjectList = getBulkJobSpectraS3Response.getMasterObjectList();

            final TransferStrategyBuilder transferStrategyBuilder = new TransferStrategyBuilder()
                    .withDs3Client(client).withMasterObjectList(masterObjectList)
                    .withChannelBuilder(new FileObjectGetter(tempDirectory))
                    .withRangesForBlobs(PartialObjectHelpers.mapRangesToBlob(masterObjectList.getObjects(),
                            PartialObjectHelpers.getPartialObjectsRanges(objects)));

            final TransferStrategy transferStrategy = transferStrategyBuilder.makeGetTransferStrategy();

            Ds3ClientHelpers.wrap(client).startReadJob(transferStrategy).transfer();

            final Collection<File> filesInTempDirectory = FileUtils.listFiles(tempDirectory.toFile(), null, false);

            for (final File file : filesInTempDirectory) {
                assertEquals(fileName, file.getName());
            }
        } finally {
            FileUtils.deleteDirectory(tempDirectory.toFile());
        }
    }

    @Test
    public void testGetJobWithUserSuppliedBlobStrategy() throws IOException, InterruptedException {
        final String tempPathPrefix = null;
        final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix);
        final String fileName = "beowulf.txt";

        try {
            final List<Ds3Object> objects = Lists.newArrayList(new Ds3Object(fileName));

            final GetBulkJobSpectraS3Request getBulkJobSpectraS3Request = new GetBulkJobSpectraS3Request(
                    BUCKET_NAME, objects);

            final GetBulkJobSpectraS3Response getBulkJobSpectraS3Response = client
                    .getBulkJobSpectraS3(getBulkJobSpectraS3Request);

            final MasterObjectList masterObjectList = getBulkJobSpectraS3Response.getMasterObjectList();

            final EventDispatcher eventDispatcher = new EventDispatcherImpl(new SameThreadEventRunner());

            final AtomicInteger numChunkAllocationAttempts = new AtomicInteger(0);

            final TransferStrategyBuilder transferStrategyBuilder = new TransferStrategyBuilder()
                    .withDs3Client(client).withMasterObjectList(masterObjectList)
                    .withChannelBuilder(new FileObjectGetter(tempDirectory))
                    .withRangesForBlobs(PartialObjectHelpers.mapRangesToBlob(masterObjectList.getObjects(),
                            PartialObjectHelpers.getPartialObjectsRanges(objects)))
                    .withBlobStrategy(new UserSuppliedPutBlobStrategy(client, masterObjectList, eventDispatcher,
                            new MaxChunkAttemptsRetryBehavior(5),
                            new ClientDefinedChunkAttemptRetryDelayBehavior(1, eventDispatcher), new Monitorable() {
                                @Override
                                public void monitor() {
                                    numChunkAllocationAttempts.getAndIncrement();
                                }
                            }));

            final TransferStrategy transferStrategy = transferStrategyBuilder.makeGetTransferStrategy();

            transferStrategy.transfer();

            final Collection<File> filesInTempDirectory = FileUtils.listFiles(tempDirectory.toFile(), null, false);

            for (final File file : filesInTempDirectory) {
                assertEquals(fileName, file.getName());
            }

            assertEquals(1, numChunkAllocationAttempts.get());
        } finally {
            FileUtils.deleteDirectory(tempDirectory.toFile());
        }
    }

    private interface Monitorable {
        void monitor();
    }

    private class UserSuppliedPutBlobStrategy implements BlobStrategy {
        private final BlobStrategy wrappedBlobStrategy;
        private final Monitorable monitorable;

        private UserSuppliedPutBlobStrategy(final Ds3Client client, final MasterObjectList masterObjectList,
                final EventDispatcher eventDispatcher, final ChunkAttemptRetryBehavior retryBehavior,
                final ChunkAttemptRetryDelayBehavior chunkAttemptRetryDelayBehavior,
                final Monitorable monitorable) {
            this.monitorable = monitorable;

            wrappedBlobStrategy = new GetSequentialBlobStrategy(client, masterObjectList, eventDispatcher,
                    retryBehavior, chunkAttemptRetryDelayBehavior);
        }

        @Override
        public Iterable<JobPart> getWork() throws IOException, InterruptedException {
            monitorable.monitor();

            return wrappedBlobStrategy.getWork();
        }
    }

    @Test
    public void testGetJobWithUserSuppliedChannelStrategy() throws IOException, InterruptedException {
        testGetJobWithUserSuppliedChannelStrategy(new TransferStrategyBuilderModifiable() {
            @Override
            public TransferStrategyBuilder modify(final TransferStrategyBuilder transferStrategyBuilder) {
                return transferStrategyBuilder;
            }
        });
    }

    private void testGetJobWithUserSuppliedChannelStrategy(
            final TransferStrategyBuilderModifiable transferStrategyBuilderModifiable)
            throws IOException, InterruptedException {
        final String tempPathPrefix = null;
        final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix);
        final String fileName = "beowulf.txt";

        try {
            final List<Ds3Object> objects = Lists.newArrayList(new Ds3Object(fileName));

            final GetBulkJobSpectraS3Request getBulkJobSpectraS3Request = new GetBulkJobSpectraS3Request(
                    BUCKET_NAME, objects);

            final GetBulkJobSpectraS3Response getBulkJobSpectraS3Response = client
                    .getBulkJobSpectraS3(getBulkJobSpectraS3Request);

            final MasterObjectList masterObjectList = getBulkJobSpectraS3Response.getMasterObjectList();

            final AtomicInteger numTimesChannelOpened = new AtomicInteger(0);
            final AtomicInteger numTimesChannelClosed = new AtomicInteger(0);

            TransferStrategyBuilder transferStrategyBuilder = new TransferStrategyBuilder().withDs3Client(client)
                    .withMasterObjectList(masterObjectList).withChannelBuilder(new FileObjectGetter(tempDirectory))
                    .withRangesForBlobs(PartialObjectHelpers.mapRangesToBlob(masterObjectList.getObjects(),
                            PartialObjectHelpers.getPartialObjectsRanges(objects)))
                    .withChannelStrategy(new UserSuppliedPutChannelStrategy(new FileObjectGetter(tempDirectory),
                            new ChannelMonitorable() {
                                @Override
                                public void acquired() {
                                    numTimesChannelOpened.getAndIncrement();
                                }

                                @Override
                                public void released() {
                                    numTimesChannelClosed.getAndIncrement();
                                }
                            }));

            transferStrategyBuilder = transferStrategyBuilderModifiable.modify(transferStrategyBuilder);

            final TransferStrategy transferStrategy = transferStrategyBuilder.makeGetTransferStrategy();

            transferStrategy.transfer();

            final Collection<File> filesInTempDirectory = FileUtils.listFiles(tempDirectory.toFile(), null, false);

            for (final File file : filesInTempDirectory) {
                assertEquals(fileName, file.getName());
            }

            assertEquals(1, numTimesChannelOpened.get());
            assertEquals(1, numTimesChannelClosed.get());
        } finally {
            FileUtils.deleteDirectory(tempDirectory.toFile());
        }
    }

    private interface TransferStrategyBuilderModifiable {
        TransferStrategyBuilder modify(final TransferStrategyBuilder transferStrategyBuilder);
    }

    private interface ChannelMonitorable {
        void acquired();

        void released();
    }

    private class UserSuppliedPutChannelStrategy implements ChannelStrategy {
        private final ChannelMonitorable channelMonitorable;
        private final ChannelStrategy wrappedPutStrategy;

        private UserSuppliedPutChannelStrategy(final Ds3ClientHelpers.ObjectChannelBuilder objectChannelBuilder,
                final ChannelMonitorable channelMonitorable) {
            this.channelMonitorable = channelMonitorable;
            wrappedPutStrategy = new SequentialFileWriterChannelStrategy(objectChannelBuilder);
        }

        @Override
        public SeekableByteChannel acquireChannelForBlob(final BulkObject blob) throws IOException {
            channelMonitorable.acquired();
            return wrappedPutStrategy.acquireChannelForBlob(blob);
        }

        @Override
        public void releaseChannelForBlob(final SeekableByteChannel seekableByteChannel, final BulkObject blob)
                throws IOException {
            channelMonitorable.released();
            wrappedPutStrategy.releaseChannelForBlob(seekableByteChannel, blob);
        }
    }

    @Test
    public void testStreamedGetJobWithUserSuppliedChannelStrategy() throws IOException, InterruptedException {
        testGetJobWithUserSuppliedChannelStrategy(new TransferStrategyBuilderModifiable() {
            @Override
            public TransferStrategyBuilder modify(final TransferStrategyBuilder transferStrategyBuilder) {
                return transferStrategyBuilder.usingStreamedTransferBehavior();
            }
        });
    }

    @Test
    public void testRandomAccessGetJobWithUserSuppliedChannelStrategy() throws IOException, InterruptedException {
        testGetJobWithUserSuppliedChannelStrategy(new TransferStrategyBuilderModifiable() {
            @Override
            public TransferStrategyBuilder modify(final TransferStrategyBuilder transferStrategyBuilder) {
                return transferStrategyBuilder.usingRandomAccessTransferBehavior();
            }
        });
    }

    @Test
    public void testGetJobUserSuppliedTransferRetryDecorator() throws IOException, InterruptedException {
        final String tempPathPrefix = null;
        final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix);
        final String fileName = "beowulf.txt";

        try {
            final List<Ds3Object> objects = Lists.newArrayList(new Ds3Object(fileName));

            final GetBulkJobSpectraS3Request getBulkJobSpectraS3Request = new GetBulkJobSpectraS3Request(
                    BUCKET_NAME, objects);

            final GetBulkJobSpectraS3Response getBulkJobSpectraS3Response = client
                    .getBulkJobSpectraS3(getBulkJobSpectraS3Request);

            final MasterObjectList masterObjectList = getBulkJobSpectraS3Response.getMasterObjectList();

            final AtomicInteger numTimesTransferCalled = new AtomicInteger(0);

            final TransferStrategyBuilder transferStrategyBuilder = new TransferStrategyBuilder()
                    .withDs3Client(client).withMasterObjectList(masterObjectList)
                    .withChannelBuilder(new FileObjectGetter(tempDirectory))
                    .withRangesForBlobs(PartialObjectHelpers.mapRangesToBlob(masterObjectList.getObjects(),
                            PartialObjectHelpers.getPartialObjectsRanges(objects)))
                    .withTransferRetryDecorator(new UserSuppliedTransferRetryDecorator(new Monitorable() {
                        @Override
                        public void monitor() {
                            numTimesTransferCalled.getAndIncrement();
                        }
                    }));

            final TransferStrategy transferStrategy = transferStrategyBuilder.makeGetTransferStrategy();

            transferStrategy.transfer();

            final Collection<File> filesInTempDirectory = FileUtils.listFiles(tempDirectory.toFile(), null, false);

            for (final File file : filesInTempDirectory) {
                assertEquals(fileName, file.getName());
            }

            assertEquals(1, numTimesTransferCalled.get());
        } finally {
            FileUtils.deleteDirectory(tempDirectory.toFile());
        }
    }

    private class UserSuppliedTransferRetryDecorator implements TransferRetryDecorator {
        private final TransferRetryDecorator transferRetryDecorator;
        private final Monitorable monitorable;

        private UserSuppliedTransferRetryDecorator(final Monitorable monitorable) {
            this.transferRetryDecorator = new MaxNumObjectTransferAttemptsDecorator(5);
            this.monitorable = monitorable;
        }

        @Override
        public TransferMethod wrap(final TransferMethod transferMethod) {
            transferRetryDecorator.wrap(transferMethod);
            return this;
        }

        @Override
        public void transferJobPart(final JobPart jobPart) throws IOException {
            monitorable.monitor();
            transferRetryDecorator.transferJobPart(jobPart);
        }
    }

    @Test
    public void testGetJobUserSuppliedChunkAttemptRetryBehavior() throws IOException, InterruptedException {
        final String tempPathPrefix = null;
        final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix);
        final String fileName = "beowulf.txt";

        try {
            final List<Ds3Object> objects = Lists.newArrayList(new Ds3Object(fileName));

            final GetBulkJobSpectraS3Request getBulkJobSpectraS3Request = new GetBulkJobSpectraS3Request(
                    BUCKET_NAME, objects);

            final GetBulkJobSpectraS3Response getBulkJobSpectraS3Response = client
                    .getBulkJobSpectraS3(getBulkJobSpectraS3Request);

            final MasterObjectList masterObjectList = getBulkJobSpectraS3Response.getMasterObjectList();

            final AtomicInteger numTimesInvokeCalled = new AtomicInteger(0);
            final AtomicInteger numTimesResetCalled = new AtomicInteger(0);

            final TransferStrategyBuilder transferStrategyBuilder = new TransferStrategyBuilder()
                    .withDs3Client(client).withMasterObjectList(masterObjectList)
                    .withChannelBuilder(new FileObjectGetter(tempDirectory))
                    .withRangesForBlobs(PartialObjectHelpers.mapRangesToBlob(masterObjectList.getObjects(),
                            PartialObjectHelpers.getPartialObjectsRanges(objects)))
                    .withChunkAttemptRetryBehavior(
                            new UserSuppliedChunkAttemptRetryBehavior(new ChunkAttemptRetryBehaviorMonitorable() {
                                @Override
                                public void invoke() {
                                    numTimesInvokeCalled.getAndIncrement();
                                }

                                @Override
                                public void reset() {
                                    numTimesResetCalled.getAndIncrement();
                                }
                            }));

            final TransferStrategy transferStrategy = transferStrategyBuilder.makeGetTransferStrategy();

            transferStrategy.transfer();

            final Collection<File> filesInTempDirectory = FileUtils.listFiles(tempDirectory.toFile(), null, false);

            for (final File file : filesInTempDirectory) {
                assertEquals(fileName, file.getName());
            }

            assertEquals(0, numTimesInvokeCalled.get());
            assertEquals(1, numTimesResetCalled.get());
        } finally {
            FileUtils.deleteDirectory(tempDirectory.toFile());
        }
    }

    private class UserSuppliedChunkAttemptRetryBehavior implements ChunkAttemptRetryBehavior {
        private final ChunkAttemptRetryBehaviorMonitorable chunkAttemptRetryBehaviorMonitorable;
        private final ChunkAttemptRetryBehavior wrappedChunkAttemptRetryBehavior;

        private UserSuppliedChunkAttemptRetryBehavior(
                final ChunkAttemptRetryBehaviorMonitorable chunkAttemptRetryBehaviorMonitorable) {
            this.chunkAttemptRetryBehaviorMonitorable = chunkAttemptRetryBehaviorMonitorable;
            wrappedChunkAttemptRetryBehavior = new MaxChunkAttemptsRetryBehavior(5);
        }

        @Override
        public void invoke() throws IOException {
            chunkAttemptRetryBehaviorMonitorable.invoke();
            wrappedChunkAttemptRetryBehavior.invoke();
        }

        @Override
        public void reset() {
            chunkAttemptRetryBehaviorMonitorable.reset();
            wrappedChunkAttemptRetryBehavior.reset();
        }
    }

    private interface ChunkAttemptRetryBehaviorMonitorable {
        void invoke();

        void reset();
    }

    @Test
    public void testThatFifoIsNotProcessed() throws IOException, InterruptedException {
        Assume.assumeFalse(Platform.isWindows());

        final String tempPathPrefix = null;
        final Path tempDirectory = Files.createTempDirectory(Paths.get("."), tempPathPrefix);

        final String BEOWULF_FILE_NAME = "beowulf.txt";

        final AtomicBoolean caughtException = new AtomicBoolean(false);

        try {
            Runtime.getRuntime().exec("mkfifo " + Paths.get(tempDirectory.toString(), BEOWULF_FILE_NAME)).waitFor();

            final List<Ds3Object> ds3Objects = Arrays.asList(new Ds3Object(BEOWULF_FILE_NAME));

            final Ds3ClientHelpers.Job readJob = HELPERS.startReadJob(BUCKET_NAME, ds3Objects);
            readJob.transfer(new FileObjectPutter(tempDirectory));
        } catch (final UnrecoverableIOException e) {
            assertTrue(e.getMessage().contains(BEOWULF_FILE_NAME));
            caughtException.set(true);
        } finally {
            FileUtils.deleteDirectory(tempDirectory.toFile());
        }

        assertTrue(caughtException.get());
    }
}