com.amazonaws.services.kinesis.clientlibrary.lib.worker.ShardSyncerTest.java Source code

Java tutorial

Introduction

Here is the source code for com.amazonaws.services.kinesis.clientlibrary.lib.worker.ShardSyncerTest.java

Source

/*
 * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 *
 * Licensed under the Amazon Software License (the "License").
 * You may not use this file except in compliance with the License.
 * A copy of the License is located at
 *
 * http://aws.amazon.com/asl/
 *
 * or in the "license" file accompanying this file. This file is distributed
 * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
 * express or implied. See the License for the specific language governing
 * permissions and limitations under the License.
 */
package com.amazonaws.services.kinesis.clientlibrary.lib.worker;

import java.io.File;
import java.io.IOException;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import com.amazonaws.services.dynamodbv2.local.embedded.DynamoDBEmbedded;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;

import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
import com.amazonaws.services.kinesis.clientlibrary.exceptions.internal.KinesisClientLibIOException;
import com.amazonaws.services.kinesis.clientlibrary.lib.worker.ExceptionThrowingLeaseManager.ExceptionThrowingLeaseManagerMethods;
import com.amazonaws.services.kinesis.clientlibrary.proxies.IKinesisProxy;
import com.amazonaws.services.kinesis.clientlibrary.proxies.KinesisLocalFileProxy;
import com.amazonaws.services.kinesis.clientlibrary.proxies.util.KinesisLocalFileDataCreator;
import com.amazonaws.services.kinesis.clientlibrary.types.ExtendedSequenceNumber;
import com.amazonaws.services.kinesis.leases.exceptions.DependencyException;
import com.amazonaws.services.kinesis.leases.exceptions.InvalidStateException;
import com.amazonaws.services.kinesis.leases.exceptions.LeasingException;
import com.amazonaws.services.kinesis.leases.exceptions.ProvisionedThroughputException;
import com.amazonaws.services.kinesis.leases.impl.KinesisClientLease;
import com.amazonaws.services.kinesis.leases.impl.KinesisClientLeaseManager;
import com.amazonaws.services.kinesis.leases.impl.LeaseManager;
import com.amazonaws.services.kinesis.model.HashKeyRange;
import com.amazonaws.services.kinesis.model.SequenceNumberRange;
import com.amazonaws.services.kinesis.model.Shard;

import junit.framework.Assert;

/**
 *
 */
// CHECKSTYLE:IGNORE JavaNCSS FOR NEXT 800 LINES
public class ShardSyncerTest {
    private static final Log LOG = LogFactory.getLog(ShardSyncer.class);
    private static final InitialPositionInStreamExtended INITIAL_POSITION_LATEST = InitialPositionInStreamExtended
            .newInitialPosition(InitialPositionInStream.LATEST);
    private static final InitialPositionInStreamExtended INITIAL_POSITION_TRIM_HORIZON = InitialPositionInStreamExtended
            .newInitialPosition(InitialPositionInStream.TRIM_HORIZON);
    private static final InitialPositionInStreamExtended INITIAL_POSITION_AT_TIMESTAMP = InitialPositionInStreamExtended
            .newInitialPositionAtTimestamp(new Date(1000L));
    private final boolean cleanupLeasesOfCompletedShards = true;
    AmazonDynamoDB ddbClient = DynamoDBEmbedded.create().amazonDynamoDB();
    LeaseManager<KinesisClientLease> leaseManager = new KinesisClientLeaseManager("tempTestTable", ddbClient);
    private static final int EXPONENT = 128;
    /**
     * Old/Obsolete max value of a sequence number (2^128 -1).
     */
    public static final BigInteger MAX_SEQUENCE_NUMBER = new BigInteger("2").pow(EXPONENT).subtract(BigInteger.ONE);

    /**
     * @throws java.lang.Exception
     */
    @BeforeClass
    public static void setUpBeforeClass() throws Exception {
    }

    /**
     * @throws java.lang.Exception
     */
    @AfterClass
    public static void tearDownAfterClass() throws Exception {
    }

    /**
     * @throws java.lang.Exception
     */
    @Before
    public void setUp() throws Exception {
        boolean created = leaseManager.createLeaseTableIfNotExists(1L, 1L);
        if (created) {
            LOG.info("New table created.");
        }
        leaseManager.deleteAll();
    }

    /**
     * @throws java.lang.Exception
     */
    @After
    public void tearDown() throws Exception {
        leaseManager.deleteAll();
    }

    /**
     * Test determineNewLeasesToCreate() where there are no shards
     */
    @Test
    public final void testDetermineNewLeasesToCreateNoShards() {
        List<Shard> shards = new ArrayList<Shard>();
        List<KinesisClientLease> leases = new ArrayList<KinesisClientLease>();

        Assert.assertTrue(
                ShardSyncer.determineNewLeasesToCreate(shards, leases, INITIAL_POSITION_LATEST).isEmpty());
    }

    /**
     * Test determineNewLeasesToCreate() where there are no leases and no resharding operations have been performed
     */
    @Test
    public final void testDetermineNewLeasesToCreate0Leases0Reshards() {
        List<Shard> shards = new ArrayList<Shard>();
        List<KinesisClientLease> currentLeases = new ArrayList<KinesisClientLease>();
        SequenceNumberRange sequenceRange = ShardObjectHelper.newSequenceNumberRange("342980", null);

        String shardId0 = "shardId-0";
        shards.add(ShardObjectHelper.newShard(shardId0, null, null, sequenceRange));

        String shardId1 = "shardId-1";
        shards.add(ShardObjectHelper.newShard(shardId1, null, null, sequenceRange));

        List<KinesisClientLease> newLeases = ShardSyncer.determineNewLeasesToCreate(shards, currentLeases,
                INITIAL_POSITION_LATEST);
        Assert.assertEquals(2, newLeases.size());
        Set<String> expectedLeaseShardIds = new HashSet<String>();
        expectedLeaseShardIds.add(shardId0);
        expectedLeaseShardIds.add(shardId1);
        for (KinesisClientLease lease : newLeases) {
            Assert.assertTrue(expectedLeaseShardIds.contains(lease.getLeaseKey()));
        }
    }

    /**
     * Test bootstrapShardLeases() starting at TRIM_HORIZON ("beginning" of stream)
     * 
     * @throws ProvisionedThroughputException
     * @throws InvalidStateException
     * @throws DependencyException
     * @throws IOException
     * @throws KinesisClientLibIOException
     */
    @Test
    public final void testBootstrapShardLeasesAtTrimHorizon() throws DependencyException, InvalidStateException,
            ProvisionedThroughputException, IOException, KinesisClientLibIOException {
        testBootstrapShardLeasesAtStartingPosition(INITIAL_POSITION_TRIM_HORIZON);
    }

    /**
     * Test bootstrapShardLeases() starting at LATEST (tip of stream)
     * 
     * @throws ProvisionedThroughputException
     * @throws InvalidStateException
     * @throws DependencyException
     * @throws IOException
     * @throws KinesisClientLibIOException
     */
    @Test
    public final void testBootstrapShardLeasesAtLatest() throws DependencyException, InvalidStateException,
            ProvisionedThroughputException, IOException, KinesisClientLibIOException {
        testBootstrapShardLeasesAtStartingPosition(INITIAL_POSITION_LATEST);
    }

    /**
     * @throws KinesisClientLibIOException
     * @throws DependencyException
     * @throws InvalidStateException
     * @throws ProvisionedThroughputException
     * @throws IOException
     */
    @Test
    public final void testCheckAndCreateLeasesForNewShardsAtLatest() throws KinesisClientLibIOException,
            DependencyException, InvalidStateException, ProvisionedThroughputException, IOException {
        List<Shard> shards = constructShardListForGraphA();
        File dataFile = KinesisLocalFileDataCreator.generateTempDataFile(shards, 2, "testBootstrap1");
        dataFile.deleteOnExit();
        IKinesisProxy kinesisProxy = new KinesisLocalFileProxy(dataFile.getAbsolutePath());

        ShardSyncer.checkAndCreateLeasesForNewShards(kinesisProxy, leaseManager, INITIAL_POSITION_LATEST,
                cleanupLeasesOfCompletedShards);
        List<KinesisClientLease> newLeases = leaseManager.listLeases();
        Set<String> expectedLeaseShardIds = new HashSet<String>();
        expectedLeaseShardIds.add("shardId-4");
        expectedLeaseShardIds.add("shardId-8");
        expectedLeaseShardIds.add("shardId-9");
        expectedLeaseShardIds.add("shardId-10");
        Assert.assertEquals(expectedLeaseShardIds.size(), newLeases.size());
        for (KinesisClientLease lease1 : newLeases) {
            Assert.assertTrue(expectedLeaseShardIds.contains(lease1.getLeaseKey()));
            Assert.assertEquals(ExtendedSequenceNumber.LATEST, lease1.getCheckpoint());
        }
        dataFile.delete();
    }

    /**
     * @throws KinesisClientLibIOException
     * @throws DependencyException
     * @throws InvalidStateException
     * @throws ProvisionedThroughputException
     * @throws IOException
     */
    @Test
    public final void testCheckAndCreateLeasesForNewShardsAtTrimHorizon() throws KinesisClientLibIOException,
            DependencyException, InvalidStateException, ProvisionedThroughputException, IOException {
        List<Shard> shards = constructShardListForGraphA();
        File dataFile = KinesisLocalFileDataCreator.generateTempDataFile(shards, 2, "testBootstrap1");
        dataFile.deleteOnExit();
        IKinesisProxy kinesisProxy = new KinesisLocalFileProxy(dataFile.getAbsolutePath());

        ShardSyncer.checkAndCreateLeasesForNewShards(kinesisProxy, leaseManager, INITIAL_POSITION_TRIM_HORIZON,
                cleanupLeasesOfCompletedShards);
        List<KinesisClientLease> newLeases = leaseManager.listLeases();
        Set<String> expectedLeaseShardIds = new HashSet<String>();
        for (int i = 0; i < 11; i++) {
            expectedLeaseShardIds.add("shardId-" + i);
        }
        Assert.assertEquals(expectedLeaseShardIds.size(), newLeases.size());
        for (KinesisClientLease lease1 : newLeases) {
            Assert.assertTrue(expectedLeaseShardIds.contains(lease1.getLeaseKey()));
            Assert.assertEquals(ExtendedSequenceNumber.TRIM_HORIZON, lease1.getCheckpoint());
        }
        dataFile.delete();
    }

    /**
     * @throws KinesisClientLibIOException
     * @throws DependencyException
     * @throws InvalidStateException
     * @throws ProvisionedThroughputException
     * @throws IOException
     */
    @Test
    public final void testCheckAndCreateLeasesForNewShardsAtTimestamp() throws KinesisClientLibIOException,
            DependencyException, InvalidStateException, ProvisionedThroughputException, IOException {
        List<Shard> shards = constructShardListForGraphA();
        File dataFile = KinesisLocalFileDataCreator.generateTempDataFile(shards, 1, "testBootstrap1");
        dataFile.deleteOnExit();
        IKinesisProxy kinesisProxy = new KinesisLocalFileProxy(dataFile.getAbsolutePath());

        ShardSyncer.checkAndCreateLeasesForNewShards(kinesisProxy, leaseManager, INITIAL_POSITION_AT_TIMESTAMP,
                cleanupLeasesOfCompletedShards);
        List<KinesisClientLease> newLeases = leaseManager.listLeases();
        Set<String> expectedLeaseShardIds = new HashSet<String>();
        for (int i = 0; i < 11; i++) {
            expectedLeaseShardIds.add("shardId-" + i);
        }
        Assert.assertEquals(expectedLeaseShardIds.size(), newLeases.size());
        for (KinesisClientLease lease1 : newLeases) {
            Assert.assertTrue(expectedLeaseShardIds.contains(lease1.getLeaseKey()));
            Assert.assertEquals(ExtendedSequenceNumber.AT_TIMESTAMP, lease1.getCheckpoint());
        }
        dataFile.delete();
    }

    /**
     * @throws KinesisClientLibIOException
     * @throws DependencyException
     * @throws InvalidStateException
     * @throws ProvisionedThroughputException
     * @throws IOException
     */
    @Test(expected = KinesisClientLibIOException.class)
    public final void testCheckAndCreateLeasesForNewShardsWhenParentIsOpen() throws KinesisClientLibIOException,
            DependencyException, InvalidStateException, ProvisionedThroughputException, IOException {
        List<Shard> shards = constructShardListForGraphA();
        SequenceNumberRange range = shards.get(0).getSequenceNumberRange();
        range.setEndingSequenceNumber(null);
        shards.get(3).setSequenceNumberRange(range);
        File dataFile = KinesisLocalFileDataCreator.generateTempDataFile(shards, 2, "testBootstrap1");
        dataFile.deleteOnExit();
        IKinesisProxy kinesisProxy = new KinesisLocalFileProxy(dataFile.getAbsolutePath());

        ShardSyncer.checkAndCreateLeasesForNewShards(kinesisProxy, leaseManager, INITIAL_POSITION_TRIM_HORIZON,
                cleanupLeasesOfCompletedShards);
        dataFile.delete();
    }

    /**
     * @throws KinesisClientLibIOException
     * @throws DependencyException
     * @throws InvalidStateException
     * @throws ProvisionedThroughputException
     * @throws IOException
     */
    @Test
    public final void testCheckAndCreateLeasesForNewShardsAtTrimHorizonAndClosedShard()
            throws KinesisClientLibIOException, DependencyException, InvalidStateException,
            ProvisionedThroughputException, IOException {
        testCheckAndCreateLeasesForNewShardsAtSpecifiedPositionAndClosedShardImpl(null, Integer.MAX_VALUE,
                INITIAL_POSITION_TRIM_HORIZON);
    }

    /**
     * @throws KinesisClientLibIOException
     * @throws DependencyException
     * @throws InvalidStateException
     * @throws ProvisionedThroughputException
     * @throws IOException
     */
    @Test
    public final void testCheckAndCreateLeasesForNewShardsAtTrimHorizonAndClosedShardWithDeleteLeaseExceptions()
            throws KinesisClientLibIOException, DependencyException, InvalidStateException,
            ProvisionedThroughputException, IOException {
        // Define the max calling count for lease manager methods.
        // From the Shard Graph, the max count of calling could be 10
        int maxCallingCount = 10;
        for (int c = 1; c <= maxCallingCount; c = c + 2) {
            testCheckAndCreateLeasesForNewShardsAtSpecifiedPositionAndClosedShardImpl(
                    ExceptionThrowingLeaseManagerMethods.DELETELEASE, c, INITIAL_POSITION_TRIM_HORIZON);
            // Need to clean up lease manager every time after calling ShardSyncer
            leaseManager.deleteAll();
        }
    }

    /**
     * @throws KinesisClientLibIOException
     * @throws DependencyException
     * @throws InvalidStateException
     * @throws ProvisionedThroughputException
     * @throws IOException
     */
    @Test
    public final void testCheckAndCreateLeasesForNewShardsAtTrimHorizonAndClosedShardWithListLeasesExceptions()
            throws KinesisClientLibIOException, DependencyException, InvalidStateException,
            ProvisionedThroughputException, IOException {
        // Define the max calling count for lease manager methods.
        // From the Shard Graph, the max count of calling could be 10
        int maxCallingCount = 10;
        for (int c = 1; c <= maxCallingCount; c = c + 2) {
            testCheckAndCreateLeasesForNewShardsAtSpecifiedPositionAndClosedShardImpl(
                    ExceptionThrowingLeaseManagerMethods.LISTLEASES, c, INITIAL_POSITION_TRIM_HORIZON);
            // Need to clean up lease manager every time after calling ShardSyncer
            leaseManager.deleteAll();
        }
    }

    /**
     * @throws KinesisClientLibIOException
     * @throws DependencyException
     * @throws InvalidStateException
     * @throws ProvisionedThroughputException
     * @throws IOException
     */
    @Test
    public final void testCheckAndCreateLeasesForNewShardsAtTrimHorizonAndClosedShardWithCreateLeaseExceptions()
            throws KinesisClientLibIOException, DependencyException, InvalidStateException,
            ProvisionedThroughputException, IOException {
        // Define the max calling count for lease manager methods.
        // From the Shard Graph, the max count of calling could be 10
        int maxCallingCount = 5;
        for (int c = 1; c <= maxCallingCount; c = c + 2) {
            testCheckAndCreateLeasesForNewShardsAtSpecifiedPositionAndClosedShardImpl(
                    ExceptionThrowingLeaseManagerMethods.CREATELEASEIFNOTEXISTS, c, INITIAL_POSITION_TRIM_HORIZON);
            // Need to clean up lease manager every time after calling ShardSyncer
            leaseManager.deleteAll();
        }
    }

    // Try catch leaseException for different lease manager methods and eventually let it succeed.
    // This would not throw any exceptions if:
    // 1). exceptionMethod equals to null or NONE.
    // 2). exceptionTime is a very big or negative value.
    private void retryCheckAndCreateLeaseForNewShards(IKinesisProxy kinesisProxy,
            ExceptionThrowingLeaseManagerMethods exceptionMethod, int exceptionTime,
            InitialPositionInStreamExtended position) throws KinesisClientLibIOException, DependencyException,
            InvalidStateException, ProvisionedThroughputException {
        if (exceptionMethod != null) {
            ExceptionThrowingLeaseManager exceptionThrowingLeaseManager = new ExceptionThrowingLeaseManager(
                    leaseManager);
            // Set exception and throwing time for exceptionThrowingManager.
            exceptionThrowingLeaseManager.setLeaseLeaseManagerThrowingExceptionScenario(exceptionMethod,
                    exceptionTime);
            // Only need to try two times.
            for (int i = 1; i <= 2; i++) {
                try {
                    ShardSyncer.checkAndCreateLeasesForNewShards(kinesisProxy, exceptionThrowingLeaseManager,
                            position, cleanupLeasesOfCompletedShards);
                    return;
                } catch (LeasingException e) {
                    LOG.debug("Catch leasing exception", e);
                }
                // Clear throwing exception scenario every time after calling ShardSyncer
                exceptionThrowingLeaseManager.clearLeaseManagerThrowingExceptionScenario();
            }
        } else {
            ShardSyncer.checkAndCreateLeasesForNewShards(kinesisProxy, leaseManager, position,
                    cleanupLeasesOfCompletedShards);
        }
    }

    /**
     * @throws KinesisClientLibIOException
     * @throws DependencyException
     * @throws InvalidStateException
     * @throws ProvisionedThroughputException
     * @throws IOException
     */
    @Test
    public final void testCheckAndCreateLeasesForNewShardsAtTimestampAndClosedShard()
            throws KinesisClientLibIOException, DependencyException, InvalidStateException,
            ProvisionedThroughputException, IOException {
        testCheckAndCreateLeasesForNewShardsAtSpecifiedPositionAndClosedShardImpl(null, Integer.MAX_VALUE,
                INITIAL_POSITION_AT_TIMESTAMP);
    }

    /**
     * @throws KinesisClientLibIOException
     * @throws DependencyException
     * @throws InvalidStateException
     * @throws ProvisionedThroughputException
     * @throws IOException
     */
    @Test
    public final void testCheckAndCreateLeasesForNewShardsAtTimestampAndClosedShardWithDeleteLeaseExceptions()
            throws KinesisClientLibIOException, DependencyException, InvalidStateException,
            ProvisionedThroughputException, IOException {
        // Define the max calling count for lease manager methods.
        // From the Shard Graph, the max count of calling could be 10
        int maxCallingCount = 10;
        for (int c = 1; c <= maxCallingCount; c = c + 2) {
            testCheckAndCreateLeasesForNewShardsAtSpecifiedPositionAndClosedShardImpl(
                    ExceptionThrowingLeaseManagerMethods.DELETELEASE, c, INITIAL_POSITION_AT_TIMESTAMP);
            // Need to clean up lease manager every time after calling ShardSyncer
            leaseManager.deleteAll();
        }
    }

    /**
     * @throws KinesisClientLibIOException
     * @throws DependencyException
     * @throws InvalidStateException
     * @throws ProvisionedThroughputException
     * @throws IOException
     */
    @Test
    public final void testCheckAndCreateLeasesForNewShardsAtTimestampAndClosedShardWithListLeasesExceptions()
            throws KinesisClientLibIOException, DependencyException, InvalidStateException,
            ProvisionedThroughputException, IOException {
        // Define the max calling count for lease manager methods.
        // From the Shard Graph, the max count of calling could be 10
        int maxCallingCount = 10;
        for (int c = 1; c <= maxCallingCount; c = c + 2) {
            testCheckAndCreateLeasesForNewShardsAtSpecifiedPositionAndClosedShardImpl(
                    ExceptionThrowingLeaseManagerMethods.LISTLEASES, c, INITIAL_POSITION_AT_TIMESTAMP);
            // Need to clean up lease manager every time after calling ShardSyncer
            leaseManager.deleteAll();
        }
    }

    /**
     * @throws KinesisClientLibIOException
     * @throws DependencyException
     * @throws InvalidStateException
     * @throws ProvisionedThroughputException
     * @throws IOException
     */
    @Test
    public final void testCheckAndCreateLeasesForNewShardsAtTimestampAndClosedShardWithCreateLeaseExceptions()
            throws KinesisClientLibIOException, DependencyException, InvalidStateException,
            ProvisionedThroughputException, IOException {
        // Define the max calling count for lease manager methods.
        // From the Shard Graph, the max count of calling could be 10
        int maxCallingCount = 5;
        for (int c = 1; c <= maxCallingCount; c = c + 2) {
            testCheckAndCreateLeasesForNewShardsAtSpecifiedPositionAndClosedShardImpl(
                    ExceptionThrowingLeaseManagerMethods.CREATELEASEIFNOTEXISTS, c, INITIAL_POSITION_AT_TIMESTAMP);
            // Need to clean up lease manager every time after calling ShardSyncer
            leaseManager.deleteAll();
        }
    }

    // Real implementation of testing CheckAndCreateLeasesForNewShards with different leaseManager types.
    private void testCheckAndCreateLeasesForNewShardsAtSpecifiedPositionAndClosedShardImpl(
            ExceptionThrowingLeaseManagerMethods exceptionMethod, int exceptionTime,
            InitialPositionInStreamExtended position) throws KinesisClientLibIOException, DependencyException,
            InvalidStateException, ProvisionedThroughputException, IOException {
        ExtendedSequenceNumber extendedSequenceNumber = new ExtendedSequenceNumber(
                position.getInitialPositionInStream().toString());
        List<Shard> shards = constructShardListForGraphA();
        File dataFile = KinesisLocalFileDataCreator.generateTempDataFile(shards, 2, "testBootstrap1");
        dataFile.deleteOnExit();
        IKinesisProxy kinesisProxy = new KinesisLocalFileProxy(dataFile.getAbsolutePath());

        retryCheckAndCreateLeaseForNewShards(kinesisProxy, exceptionMethod, exceptionTime, position);

        List<KinesisClientLease> newLeases = leaseManager.listLeases();
        Map<String, ExtendedSequenceNumber> expectedShardIdToCheckpointMap = new HashMap<String, ExtendedSequenceNumber>();
        for (int i = 0; i < 11; i++) {
            expectedShardIdToCheckpointMap.put("shardId-" + i, extendedSequenceNumber);
        }
        Assert.assertEquals(expectedShardIdToCheckpointMap.size(), newLeases.size());
        for (KinesisClientLease lease1 : newLeases) {
            ExtendedSequenceNumber expectedCheckpoint = expectedShardIdToCheckpointMap.get(lease1.getLeaseKey());
            Assert.assertNotNull(expectedCheckpoint);
            Assert.assertEquals(expectedCheckpoint, lease1.getCheckpoint());
        }

        KinesisClientLease closedShardLease = leaseManager.getLease("shardId-0");
        closedShardLease.setCheckpoint(ExtendedSequenceNumber.SHARD_END);
        leaseManager.updateLease(closedShardLease);
        expectedShardIdToCheckpointMap.remove(closedShardLease.getLeaseKey());
        KinesisClientLease childShardLease = leaseManager.getLease("shardId-6");
        childShardLease.setCheckpoint(new ExtendedSequenceNumber("34290"));
        leaseManager.updateLease(childShardLease);
        expectedShardIdToCheckpointMap.put(childShardLease.getLeaseKey(), new ExtendedSequenceNumber("34290"));

        retryCheckAndCreateLeaseForNewShards(kinesisProxy, exceptionMethod, exceptionTime, position);

        newLeases = leaseManager.listLeases();
        Assert.assertEquals(expectedShardIdToCheckpointMap.size(), newLeases.size());
        for (KinesisClientLease lease1 : newLeases) {
            ExtendedSequenceNumber expectedCheckpoint = expectedShardIdToCheckpointMap.get(lease1.getLeaseKey());
            Assert.assertNotNull(expectedCheckpoint);
            Assert.assertEquals(expectedCheckpoint, lease1.getCheckpoint());
        }

        dataFile.delete();
    }

    /**
     * Test bootstrapShardLeases() - cleanup garbage leases.
     * 
     * @throws ProvisionedThroughputException
     * @throws InvalidStateException
     * @throws DependencyException
     * @throws IOException
     * @throws KinesisClientLibIOException
     */
    @Test
    public final void testBootstrapShardLeasesCleanupGarbage() throws DependencyException, InvalidStateException,
            ProvisionedThroughputException, IOException, KinesisClientLibIOException {
        String garbageShardId = "shardId-garbage-001";
        KinesisClientLease garbageLease = ShardSyncer.newKCLLease(ShardObjectHelper.newShard(garbageShardId, null,
                null, ShardObjectHelper.newSequenceNumberRange("101", null)));
        garbageLease.setCheckpoint(new ExtendedSequenceNumber("999"));
        leaseManager.createLeaseIfNotExists(garbageLease);
        Assert.assertEquals(garbageShardId, leaseManager.getLease(garbageShardId).getLeaseKey());
        testBootstrapShardLeasesAtStartingPosition(INITIAL_POSITION_LATEST);
        Assert.assertNull(leaseManager.getLease(garbageShardId));
    }

    private void testBootstrapShardLeasesAtStartingPosition(InitialPositionInStreamExtended initialPosition)
            throws DependencyException, InvalidStateException, ProvisionedThroughputException, IOException,
            KinesisClientLibIOException {
        List<Shard> shards = new ArrayList<Shard>();
        SequenceNumberRange sequenceRange = ShardObjectHelper.newSequenceNumberRange("342980", null);

        String shardId0 = "shardId-0";
        shards.add(ShardObjectHelper.newShard(shardId0, null, null, sequenceRange));
        String shardId1 = "shardId-1";
        shards.add(ShardObjectHelper.newShard(shardId1, null, null, sequenceRange));
        File dataFile = KinesisLocalFileDataCreator.generateTempDataFile(shards, 2, "testBootstrap1");
        dataFile.deleteOnExit();
        IKinesisProxy kinesisProxy = new KinesisLocalFileProxy(dataFile.getAbsolutePath());

        ShardSyncer.bootstrapShardLeases(kinesisProxy, leaseManager, initialPosition,
                cleanupLeasesOfCompletedShards);
        List<KinesisClientLease> newLeases = leaseManager.listLeases();
        Assert.assertEquals(2, newLeases.size());
        Set<String> expectedLeaseShardIds = new HashSet<String>();
        expectedLeaseShardIds.add(shardId0);
        expectedLeaseShardIds.add(shardId1);
        for (KinesisClientLease lease1 : newLeases) {
            Assert.assertTrue(expectedLeaseShardIds.contains(lease1.getLeaseKey()));
            Assert.assertEquals(new ExtendedSequenceNumber(initialPosition.getInitialPositionInStream().toString()),
                    lease1.getCheckpoint());
        }
        dataFile.delete();
    }

    /**
     * Test determineNewLeasesToCreate() starting at latest and at trim horizon ("beginning" of shard)
     */
    @Test
    public final void testDetermineNewLeasesToCreateStartingPosition() {
        List<Shard> shards = new ArrayList<Shard>();
        List<KinesisClientLease> currentLeases = new ArrayList<KinesisClientLease>();
        SequenceNumberRange sequenceRange = ShardObjectHelper.newSequenceNumberRange("342980", null);

        String shardId0 = "shardId-0";
        shards.add(ShardObjectHelper.newShard(shardId0, null, null, sequenceRange));

        String shardId1 = "shardId-1";
        shards.add(ShardObjectHelper.newShard(shardId1, null, null, sequenceRange));

        Set<InitialPositionInStreamExtended> initialPositions = new HashSet<InitialPositionInStreamExtended>();
        initialPositions.add(INITIAL_POSITION_LATEST);
        initialPositions.add(INITIAL_POSITION_TRIM_HORIZON);

        for (InitialPositionInStreamExtended initialPosition : initialPositions) {
            List<KinesisClientLease> newLeases = ShardSyncer.determineNewLeasesToCreate(shards, currentLeases,
                    initialPosition);
            Assert.assertEquals(2, newLeases.size());
            Set<String> expectedLeaseShardIds = new HashSet<String>();
            expectedLeaseShardIds.add(shardId0);
            expectedLeaseShardIds.add(shardId1);
            for (KinesisClientLease lease : newLeases) {
                Assert.assertTrue(expectedLeaseShardIds.contains(lease.getLeaseKey()));
                Assert.assertEquals(
                        new ExtendedSequenceNumber(initialPosition.getInitialPositionInStream().toString()),
                        lease.getCheckpoint());
            }
        }
    }

    /**
     * Test determineNewLeasesToCreate() - 1 closed and 1 open shard (ignore closed shard)
     */
    @Test
    public final void testDetermineNewLeasesToCreateIgnoreClosedShard() {
        List<Shard> shards = new ArrayList<Shard>();
        List<KinesisClientLease> currentLeases = new ArrayList<KinesisClientLease>();

        shards.add(ShardObjectHelper.newShard("shardId-0", null, null,
                ShardObjectHelper.newSequenceNumberRange("303", "404")));
        String lastShardId = "shardId-1";
        shards.add(ShardObjectHelper.newShard(lastShardId, null, null,
                ShardObjectHelper.newSequenceNumberRange("405", null)));

        List<KinesisClientLease> newLeases = ShardSyncer.determineNewLeasesToCreate(shards, currentLeases,
                INITIAL_POSITION_LATEST);
        Assert.assertEquals(1, newLeases.size());
        Assert.assertEquals(lastShardId, newLeases.get(0).getLeaseKey());
    }

    /**
     * Test CheckIfDescendantAndAddNewLeasesForAncestors (initial position Latest)
     * Shard structure (each level depicts a stream segment):
     * 0 1 2 3 4 5- shards till epoch 102
     * \ / \ / | |
     * 6 7 4 5- shards from epoch 103 - 205
     * \ / | /\
     * 8 4 9 10 - shards from epoch 206 (open - no ending sequenceNumber)
     * Current leases: (3, 4, 5)
     */
    @Test
    public final void testDetermineNewLeasesToCreateSplitMergeLatest1() {
        List<Shard> shards = constructShardListForGraphA();
        List<KinesisClientLease> currentLeases = new ArrayList<KinesisClientLease>();

        currentLeases.add(newLease("shardId-3"));
        currentLeases.add(newLease("shardId-4"));
        currentLeases.add(newLease("shardId-5"));

        List<KinesisClientLease> newLeases = ShardSyncer.determineNewLeasesToCreate(shards, currentLeases,
                INITIAL_POSITION_LATEST);
        Map<String, ExtendedSequenceNumber> expectedShardIdCheckpointMap = new HashMap<String, ExtendedSequenceNumber>();
        expectedShardIdCheckpointMap.put("shardId-8", ExtendedSequenceNumber.TRIM_HORIZON);
        expectedShardIdCheckpointMap.put("shardId-9", ExtendedSequenceNumber.TRIM_HORIZON);
        expectedShardIdCheckpointMap.put("shardId-10", ExtendedSequenceNumber.TRIM_HORIZON);
        expectedShardIdCheckpointMap.put("shardId-6", ExtendedSequenceNumber.LATEST);
        expectedShardIdCheckpointMap.put("shardId-2", ExtendedSequenceNumber.LATEST);
        expectedShardIdCheckpointMap.put("shardId-7", ExtendedSequenceNumber.TRIM_HORIZON);

        Assert.assertEquals(expectedShardIdCheckpointMap.size(), newLeases.size());
        for (KinesisClientLease lease : newLeases) {
            Assert.assertTrue("Unexpected lease: " + lease,
                    expectedShardIdCheckpointMap.containsKey(lease.getLeaseKey()));
            Assert.assertEquals(expectedShardIdCheckpointMap.get(lease.getLeaseKey()), lease.getCheckpoint());
        }
    }

    /**
     * Test CheckIfDescendantAndAddNewLeasesForAncestors (initial position Latest)
     * Shard structure (each level depicts a stream segment):
     * 0 1 2 3 4 5- shards till epoch 102
     * \ / \ / | |
     * 6 7 4 5- shards from epoch 103 - 205
     * \ / | /\
     * 8 4 9 10 - shards from epoch 206 (open - no ending sequenceNumber)
     * Current leases: (4, 5, 7)
     */
    @Test
    public final void testDetermineNewLeasesToCreateSplitMergeLatest2() {
        List<Shard> shards = constructShardListForGraphA();
        List<KinesisClientLease> currentLeases = new ArrayList<KinesisClientLease>();

        currentLeases.add(newLease("shardId-4"));
        currentLeases.add(newLease("shardId-5"));
        currentLeases.add(newLease("shardId-7"));

        List<KinesisClientLease> newLeases = ShardSyncer.determineNewLeasesToCreate(shards, currentLeases,
                INITIAL_POSITION_LATEST);
        Map<String, ExtendedSequenceNumber> expectedShardIdCheckpointMap = new HashMap<String, ExtendedSequenceNumber>();
        expectedShardIdCheckpointMap.put("shardId-8", ExtendedSequenceNumber.TRIM_HORIZON);
        expectedShardIdCheckpointMap.put("shardId-9", ExtendedSequenceNumber.TRIM_HORIZON);
        expectedShardIdCheckpointMap.put("shardId-10", ExtendedSequenceNumber.TRIM_HORIZON);
        expectedShardIdCheckpointMap.put("shardId-6", ExtendedSequenceNumber.LATEST);

        Assert.assertEquals(expectedShardIdCheckpointMap.size(), newLeases.size());
        for (KinesisClientLease lease : newLeases) {
            Assert.assertTrue("Unexpected lease: " + lease,
                    expectedShardIdCheckpointMap.containsKey(lease.getLeaseKey()));
            Assert.assertEquals(expectedShardIdCheckpointMap.get(lease.getLeaseKey()), lease.getCheckpoint());
        }
    }

    /**
     * Test CheckIfDescendantAndAddNewLeasesForAncestors (initial position TrimHorizon)
     * Shard structure (each level depicts a stream segment):
     * 0 1 2 3 4 5- shards till epoch 102
     * \ / \ / | |
     * 6 7 4 5- shards from epoch 103 - 205
     * \ / | /\
     * 8 4 9 10 - shards from epoch 206 (open - no ending sequenceNumber)
     * Current leases: (3, 4, 5)
     */
    @Test
    public final void testDetermineNewLeasesToCreateSplitMergeHorizon1() {
        List<Shard> shards = constructShardListForGraphA();
        List<KinesisClientLease> currentLeases = new ArrayList<KinesisClientLease>();

        currentLeases.add(newLease("shardId-3"));
        currentLeases.add(newLease("shardId-4"));
        currentLeases.add(newLease("shardId-5"));

        List<KinesisClientLease> newLeases = ShardSyncer.determineNewLeasesToCreate(shards, currentLeases,
                INITIAL_POSITION_TRIM_HORIZON);
        Map<String, ExtendedSequenceNumber> expectedShardIdCheckpointMap = new HashMap<String, ExtendedSequenceNumber>();
        expectedShardIdCheckpointMap.put("shardId-8", ExtendedSequenceNumber.TRIM_HORIZON);
        expectedShardIdCheckpointMap.put("shardId-9", ExtendedSequenceNumber.TRIM_HORIZON);
        expectedShardIdCheckpointMap.put("shardId-10", ExtendedSequenceNumber.TRIM_HORIZON);
        expectedShardIdCheckpointMap.put("shardId-6", ExtendedSequenceNumber.TRIM_HORIZON);
        expectedShardIdCheckpointMap.put("shardId-2", ExtendedSequenceNumber.TRIM_HORIZON);
        expectedShardIdCheckpointMap.put("shardId-7", ExtendedSequenceNumber.TRIM_HORIZON);
        expectedShardIdCheckpointMap.put("shardId-0", ExtendedSequenceNumber.TRIM_HORIZON);
        expectedShardIdCheckpointMap.put("shardId-1", ExtendedSequenceNumber.TRIM_HORIZON);

        Assert.assertEquals(expectedShardIdCheckpointMap.size(), newLeases.size());
        for (KinesisClientLease lease : newLeases) {
            Assert.assertTrue("Unexpected lease: " + lease,
                    expectedShardIdCheckpointMap.containsKey(lease.getLeaseKey()));
            Assert.assertEquals(expectedShardIdCheckpointMap.get(lease.getLeaseKey()), lease.getCheckpoint());
        }
    }

    /**
     * Test CheckIfDescendantAndAddNewLeasesForAncestors (initial position TrimHorizon)
     * Shard structure (each level depicts a stream segment):
     * 0 1 2 3 4 5- shards till epoch 102
     * \ / \ / | |
     * 6 7 4 5- shards from epoch 103 - 205
     * \ / | /\
     * 8 4 9 10 - shards from epoch 206 (open - no ending sequenceNumber)
     * Current leases: (4, 5, 7)
     */
    @Test
    public final void testDetermineNewLeasesToCreateSplitMergeHorizon2() {
        List<Shard> shards = constructShardListForGraphA();
        List<KinesisClientLease> currentLeases = new ArrayList<KinesisClientLease>();

        currentLeases.add(newLease("shardId-4"));
        currentLeases.add(newLease("shardId-5"));
        currentLeases.add(newLease("shardId-7"));

        List<KinesisClientLease> newLeases = ShardSyncer.determineNewLeasesToCreate(shards, currentLeases,
                INITIAL_POSITION_TRIM_HORIZON);
        Map<String, ExtendedSequenceNumber> expectedShardIdCheckpointMap = new HashMap<String, ExtendedSequenceNumber>();
        expectedShardIdCheckpointMap.put("shardId-8", ExtendedSequenceNumber.TRIM_HORIZON);
        expectedShardIdCheckpointMap.put("shardId-9", ExtendedSequenceNumber.TRIM_HORIZON);
        expectedShardIdCheckpointMap.put("shardId-10", ExtendedSequenceNumber.TRIM_HORIZON);
        expectedShardIdCheckpointMap.put("shardId-6", ExtendedSequenceNumber.TRIM_HORIZON);
        expectedShardIdCheckpointMap.put("shardId-0", ExtendedSequenceNumber.TRIM_HORIZON);
        expectedShardIdCheckpointMap.put("shardId-1", ExtendedSequenceNumber.TRIM_HORIZON);

        Assert.assertEquals(expectedShardIdCheckpointMap.size(), newLeases.size());
        for (KinesisClientLease lease : newLeases) {
            Assert.assertTrue("Unexpected lease: " + lease,
                    expectedShardIdCheckpointMap.containsKey(lease.getLeaseKey()));
            Assert.assertEquals(expectedShardIdCheckpointMap.get(lease.getLeaseKey()), lease.getCheckpoint());
        }
    }

    /**
     * Test CheckIfDescendantAndAddNewLeasesForAncestors (initial position TrimHorizon)
     * For shard graph B (see the construct method doc for structure).
     *
     * Current leases: empty set
     */
    @Test
    public final void testDetermineNewLeasesToCreateGraphBNoInitialLeasesTrim() {
        List<Shard> shards = constructShardListForGraphB();
        List<KinesisClientLease> currentLeases = new ArrayList<KinesisClientLease>();
        List<KinesisClientLease> newLeases = ShardSyncer.determineNewLeasesToCreate(shards, currentLeases,
                INITIAL_POSITION_TRIM_HORIZON);
        Map<String, ExtendedSequenceNumber> expectedShardIdCheckpointMap = new HashMap<String, ExtendedSequenceNumber>();
        for (int i = 0; i < 11; i++) {
            String expectedShardId = "shardId-" + i;
            expectedShardIdCheckpointMap.put(expectedShardId, ExtendedSequenceNumber.TRIM_HORIZON);
        }

        Assert.assertEquals(expectedShardIdCheckpointMap.size(), newLeases.size());
        for (KinesisClientLease lease : newLeases) {
            Assert.assertTrue("Unexpected lease: " + lease,
                    expectedShardIdCheckpointMap.containsKey(lease.getLeaseKey()));
            Assert.assertEquals(expectedShardIdCheckpointMap.get(lease.getLeaseKey()), lease.getCheckpoint());
        }
    }

    /**
     * Test CheckIfDescendantAndAddNewLeasesForAncestors (initial position AT_TIMESTAMP)
     * Shard structure (each level depicts a stream segment):
     * 0 1 2 3 4   5- shards till epoch 102
     * \ / \ / |   |
     *  6   7  4   5- shards from epoch 103 - 205
     *   \ /   |  /\
     *    8    4 9  10 - shards from epoch 206 (open - no ending sequenceNumber)
     * Current leases: (3, 4, 5)
     */
    @Test
    public final void testDetermineNewLeasesToCreateSplitMergeAtTimestamp1() {
        List<Shard> shards = constructShardListForGraphA();
        List<KinesisClientLease> currentLeases = new ArrayList<KinesisClientLease>();

        currentLeases.add(newLease("shardId-3"));
        currentLeases.add(newLease("shardId-4"));
        currentLeases.add(newLease("shardId-5"));

        List<KinesisClientLease> newLeases = ShardSyncer.determineNewLeasesToCreate(shards, currentLeases,
                INITIAL_POSITION_AT_TIMESTAMP);
        Map<String, ExtendedSequenceNumber> expectedShardIdCheckpointMap = new HashMap<String, ExtendedSequenceNumber>();
        expectedShardIdCheckpointMap.put("shardId-8", ExtendedSequenceNumber.AT_TIMESTAMP);
        expectedShardIdCheckpointMap.put("shardId-9", ExtendedSequenceNumber.AT_TIMESTAMP);
        expectedShardIdCheckpointMap.put("shardId-10", ExtendedSequenceNumber.AT_TIMESTAMP);
        expectedShardIdCheckpointMap.put("shardId-6", ExtendedSequenceNumber.AT_TIMESTAMP);
        expectedShardIdCheckpointMap.put("shardId-2", ExtendedSequenceNumber.AT_TIMESTAMP);
        expectedShardIdCheckpointMap.put("shardId-7", ExtendedSequenceNumber.AT_TIMESTAMP);
        expectedShardIdCheckpointMap.put("shardId-0", ExtendedSequenceNumber.AT_TIMESTAMP);
        expectedShardIdCheckpointMap.put("shardId-1", ExtendedSequenceNumber.AT_TIMESTAMP);

        Assert.assertEquals(expectedShardIdCheckpointMap.size(), newLeases.size());
        for (KinesisClientLease lease : newLeases) {
            Assert.assertTrue("Unexpected lease: " + lease,
                    expectedShardIdCheckpointMap.containsKey(lease.getLeaseKey()));
            Assert.assertEquals(expectedShardIdCheckpointMap.get(lease.getLeaseKey()), lease.getCheckpoint());
        }
    }

    /**
     * Test CheckIfDescendantAndAddNewLeasesForAncestors (initial position AT_TIMESTAMP)
     * Shard structure (each level depicts a stream segment):
     * 0 1 2 3 4   5- shards till epoch 102
     * \ / \ / |   |
     *  6   7  4   5- shards from epoch 103 - 205
     *   \ /   |  /\
     *    8    4 9  10 - shards from epoch 206 (open - no ending sequenceNumber)
     * Current leases: (4, 5, 7)
     */
    @Test
    public final void testDetermineNewLeasesToCreateSplitMergeAtTimestamp2() {
        List<Shard> shards = constructShardListForGraphA();
        List<KinesisClientLease> currentLeases = new ArrayList<KinesisClientLease>();

        currentLeases.add(newLease("shardId-4"));
        currentLeases.add(newLease("shardId-5"));
        currentLeases.add(newLease("shardId-7"));

        List<KinesisClientLease> newLeases = ShardSyncer.determineNewLeasesToCreate(shards, currentLeases,
                INITIAL_POSITION_AT_TIMESTAMP);
        Map<String, ExtendedSequenceNumber> expectedShardIdCheckpointMap = new HashMap<String, ExtendedSequenceNumber>();
        expectedShardIdCheckpointMap.put("shardId-8", ExtendedSequenceNumber.AT_TIMESTAMP);
        expectedShardIdCheckpointMap.put("shardId-9", ExtendedSequenceNumber.AT_TIMESTAMP);
        expectedShardIdCheckpointMap.put("shardId-10", ExtendedSequenceNumber.AT_TIMESTAMP);
        expectedShardIdCheckpointMap.put("shardId-6", ExtendedSequenceNumber.AT_TIMESTAMP);
        expectedShardIdCheckpointMap.put("shardId-0", ExtendedSequenceNumber.AT_TIMESTAMP);
        expectedShardIdCheckpointMap.put("shardId-1", ExtendedSequenceNumber.AT_TIMESTAMP);

        Assert.assertEquals(expectedShardIdCheckpointMap.size(), newLeases.size());
        for (KinesisClientLease lease : newLeases) {
            Assert.assertTrue("Unexpected lease: " + lease,
                    expectedShardIdCheckpointMap.containsKey(lease.getLeaseKey()));
            Assert.assertEquals(expectedShardIdCheckpointMap.get(lease.getLeaseKey()), lease.getCheckpoint());
        }
    }

    /**
     * Test CheckIfDescendantAndAddNewLeasesForAncestors (initial position AT_TIMESTAMP)
     * For shard graph B (see the construct method doc for structure).
     * Current leases: empty set
     */
    @Test
    public final void testDetermineNewLeasesToCreateGraphBNoInitialLeasesAtTimestamp() {
        List<Shard> shards = constructShardListForGraphB();
        List<KinesisClientLease> currentLeases = new ArrayList<KinesisClientLease>();
        List<KinesisClientLease> newLeases = ShardSyncer.determineNewLeasesToCreate(shards, currentLeases,
                INITIAL_POSITION_AT_TIMESTAMP);
        Map<String, ExtendedSequenceNumber> expectedShardIdCheckpointMap = new HashMap<String, ExtendedSequenceNumber>();
        for (int i = 0; i < shards.size(); i++) {
            String expectedShardId = "shardId-" + i;
            expectedShardIdCheckpointMap.put(expectedShardId, ExtendedSequenceNumber.AT_TIMESTAMP);
        }

        Assert.assertEquals(expectedShardIdCheckpointMap.size(), newLeases.size());
        for (KinesisClientLease lease : newLeases) {
            Assert.assertTrue("Unexpected lease: " + lease,
                    expectedShardIdCheckpointMap.containsKey(lease.getLeaseKey()));
            Assert.assertEquals(expectedShardIdCheckpointMap.get(lease.getLeaseKey()), lease.getCheckpoint());
        }
    }

    /*
     * Helper method to construct a shard list for graph A. Graph A is defined below.
     * Shard structure (y-axis is epochs):
     * 0 1 2 3 4 5- shards till epoch 102
     * \ / \ / | |
     * 6 7 4 5- shards from epoch 103 - 205
     * \ / | /\
     * 8 4 9 10 - shards from epoch 206 (open - no ending sequenceNumber)
     */
    List<Shard> constructShardListForGraphA() {
        List<Shard> shards = new ArrayList<Shard>();

        SequenceNumberRange range0 = ShardObjectHelper.newSequenceNumberRange("11", "102");
        SequenceNumberRange range1 = ShardObjectHelper.newSequenceNumberRange("11", null);
        SequenceNumberRange range2 = ShardObjectHelper.newSequenceNumberRange("11", "205");
        SequenceNumberRange range3 = ShardObjectHelper.newSequenceNumberRange("103", "205");
        SequenceNumberRange range4 = ShardObjectHelper.newSequenceNumberRange("206", null);

        HashKeyRange hashRange0 = ShardObjectHelper.newHashKeyRange("0", "99");
        HashKeyRange hashRange1 = ShardObjectHelper.newHashKeyRange("100", "199");
        HashKeyRange hashRange2 = ShardObjectHelper.newHashKeyRange("200", "299");
        HashKeyRange hashRange3 = ShardObjectHelper.newHashKeyRange("300", "399");
        HashKeyRange hashRange4 = ShardObjectHelper.newHashKeyRange("400", "499");
        HashKeyRange hashRange5 = ShardObjectHelper.newHashKeyRange("500", ShardObjectHelper.MAX_HASH_KEY);
        HashKeyRange hashRange6 = ShardObjectHelper.newHashKeyRange("0", "199");
        HashKeyRange hashRange7 = ShardObjectHelper.newHashKeyRange("200", "399");
        HashKeyRange hashRange8 = ShardObjectHelper.newHashKeyRange("0", "399");
        HashKeyRange hashRange9 = ShardObjectHelper.newHashKeyRange("500", "799");
        HashKeyRange hashRange10 = ShardObjectHelper.newHashKeyRange("800", ShardObjectHelper.MAX_HASH_KEY);

        shards.add(ShardObjectHelper.newShard("shardId-0", null, null, range0, hashRange0));
        shards.add(ShardObjectHelper.newShard("shardId-1", null, null, range0, hashRange1));
        shards.add(ShardObjectHelper.newShard("shardId-2", null, null, range0, hashRange2));
        shards.add(ShardObjectHelper.newShard("shardId-3", null, null, range0, hashRange3));
        shards.add(ShardObjectHelper.newShard("shardId-4", null, null, range1, hashRange4));
        shards.add(ShardObjectHelper.newShard("shardId-5", null, null, range2, hashRange5));

        shards.add(ShardObjectHelper.newShard("shardId-6", "shardId-0", "shardId-1", range3, hashRange6));
        shards.add(ShardObjectHelper.newShard("shardId-7", "shardId-2", "shardId-3", range3, hashRange7));

        shards.add(ShardObjectHelper.newShard("shardId-8", "shardId-6", "shardId-7", range4, hashRange8));
        shards.add(ShardObjectHelper.newShard("shardId-9", "shardId-5", null, range4, hashRange9));
        shards.add(ShardObjectHelper.newShard("shardId-10", null, "shardId-5", range4, hashRange10));

        return shards;
    }

    /*
     * Helper method to construct a shard list for graph B. Graph B is defined below.
     * Shard structure (x-axis is epochs):
     * 0 3 6 9
     * \ / \ / \ /
     * 2 5 8
     * / \ / \ / \
     * 1 4 7 10
     */
    List<Shard> constructShardListForGraphB() {
        List<Shard> shards = new ArrayList<Shard>();

        SequenceNumberRange range0 = ShardObjectHelper.newSequenceNumberRange("1000", "1049");
        SequenceNumberRange range1 = ShardObjectHelper.newSequenceNumberRange("1050", "1099");
        SequenceNumberRange range2 = ShardObjectHelper.newSequenceNumberRange("1100", "1149");
        SequenceNumberRange range3 = ShardObjectHelper.newSequenceNumberRange("1150", "1199");
        SequenceNumberRange range4 = ShardObjectHelper.newSequenceNumberRange("1200", "1249");
        SequenceNumberRange range5 = ShardObjectHelper.newSequenceNumberRange("1250", "1299");
        SequenceNumberRange range6 = ShardObjectHelper.newSequenceNumberRange("1300", null);

        HashKeyRange hashRange0 = ShardObjectHelper.newHashKeyRange("0", "499");
        HashKeyRange hashRange1 = ShardObjectHelper.newHashKeyRange("500", ShardObjectHelper.MAX_HASH_KEY);
        HashKeyRange hashRange2 = ShardObjectHelper.newHashKeyRange("0", ShardObjectHelper.MAX_HASH_KEY);

        shards.add(ShardObjectHelper.newShard("shardId-0", null, null, range0, hashRange0));
        shards.add(ShardObjectHelper.newShard("shardId-1", null, null, range0, hashRange1));
        shards.add(ShardObjectHelper.newShard("shardId-2", "shardId-0", "shardId-1", range1, hashRange2));
        shards.add(ShardObjectHelper.newShard("shardId-3", "shardId-2", null, range2, hashRange0));
        shards.add(ShardObjectHelper.newShard("shardId-4", "shardId-2", null, range2, hashRange1));
        shards.add(ShardObjectHelper.newShard("shardId-5", "shardId-3", "shardId-4", range3, hashRange2));
        shards.add(ShardObjectHelper.newShard("shardId-6", "shardId-5", null, range4, hashRange0));
        shards.add(ShardObjectHelper.newShard("shardId-7", "shardId-5", null, range4, hashRange1));
        shards.add(ShardObjectHelper.newShard("shardId-8", "shardId-6", "shardId-7", range5, hashRange2));
        shards.add(ShardObjectHelper.newShard("shardId-9", "shardId-8", null, range6, hashRange0));
        shards.add(ShardObjectHelper.newShard("shardId-10", null, "shardId-8", range6, hashRange1));

        return shards;
    }

    /**
     * Test CheckIfDescendantAndAddNewLeasesForAncestors when shardId is null
     */
    @Test
    public final void testCheckIfDescendantAndAddNewLeasesForAncestorsNullShardId() {
        Map<String, Boolean> memoizationContext = new HashMap<>();
        Assert.assertFalse(ShardSyncer.checkIfDescendantAndAddNewLeasesForAncestors(null, INITIAL_POSITION_LATEST,
                null, null, null, memoizationContext));
    }

    /**
     * Test CheckIfDescendantAndAddNewLeasesForAncestors when shard has been trimmed
     */
    @Test
    public final void testCheckIfDescendantAndAddNewLeasesForAncestorsTrimmedShard() {
        String shardId = "shardId-trimmed";
        Map<String, Shard> kinesisShards = new HashMap<String, Shard>();
        Map<String, Boolean> memoizationContext = new HashMap<>();
        Assert.assertFalse(ShardSyncer.checkIfDescendantAndAddNewLeasesForAncestors(shardId,
                INITIAL_POSITION_LATEST, null, kinesisShards, null, memoizationContext));
    }

    /**
     * Test CheckIfDescendantAndAddNewLeasesForAncestors when there is a current lease for the shard
     */
    @Test
    public final void testCheckIfDescendantAndAddNewLeasesForAncestorsForShardWithCurrentLease() {
        String shardId = "shardId-current";
        Map<String, Shard> kinesisShards = new HashMap<String, Shard>();
        kinesisShards.put(shardId, ShardObjectHelper.newShard(shardId, null, null, null));
        Set<String> shardIdsOfCurrentLeases = new HashSet<String>();
        shardIdsOfCurrentLeases.add(shardId);
        Map<String, KinesisClientLease> newLeaseMap = new HashMap<String, KinesisClientLease>();
        Map<String, Boolean> memoizationContext = new HashMap<>();
        Assert.assertTrue(ShardSyncer.checkIfDescendantAndAddNewLeasesForAncestors(shardId, INITIAL_POSITION_LATEST,
                shardIdsOfCurrentLeases, kinesisShards, newLeaseMap, memoizationContext));
        Assert.assertTrue(newLeaseMap.isEmpty());
    }

    /**
     * Test CheckIfDescendantAndAddNewLeasesForAncestors - two parents, two ancestors, not descendant
     */
    @Test
    public final void testCheckIfDescendantAndAddNewLeasesForAncestors2P2ANotDescendant() {
        Set<String> shardIdsOfCurrentLeases = new HashSet<String>();
        Map<String, KinesisClientLease> newLeaseMap = new HashMap<String, KinesisClientLease>();
        Map<String, Shard> kinesisShards = new HashMap<String, Shard>();

        String parentShardId = "shardId-parent";
        kinesisShards.put(parentShardId, ShardObjectHelper.newShard(parentShardId, null, null, null));

        String adjacentParentShardId = "shardId-adjacentParent";
        kinesisShards.put(adjacentParentShardId,
                ShardObjectHelper.newShard(adjacentParentShardId, null, null, null));

        String shardId = "shardId-9-1";
        kinesisShards.put(shardId, ShardObjectHelper.newShard(shardId, parentShardId, adjacentParentShardId, null));

        Map<String, Boolean> memoizationContext = new HashMap<>();
        Assert.assertFalse(ShardSyncer.checkIfDescendantAndAddNewLeasesForAncestors(shardId,
                INITIAL_POSITION_LATEST, shardIdsOfCurrentLeases, kinesisShards, newLeaseMap, memoizationContext));
        Assert.assertTrue(newLeaseMap.isEmpty());
    }

    /**
     * Test CheckIfDescendantAndAddNewLeasesForAncestors - two parents, there is a lease for one parent.
     */
    @Test
    public final void testCheckIfDescendantAndAddNewLeasesForAncestors2P2A1PDescendant() {
        Set<String> shardIdsOfCurrentLeases = new HashSet<String>();
        Map<String, KinesisClientLease> newLeaseMap = new HashMap<String, KinesisClientLease>();
        Map<String, Shard> kinesisShards = new HashMap<String, Shard>();

        String parentShardId = "shardId-parent";
        kinesisShards.put(parentShardId, ShardObjectHelper.newShard(parentShardId, null, null, null));
        shardIdsOfCurrentLeases.add(parentShardId);

        String adjacentParentShardId = "shardId-adjacentParent";
        kinesisShards.put(adjacentParentShardId,
                ShardObjectHelper.newShard(adjacentParentShardId, null, null, null));

        String shardId = "shardId-9-1";
        Shard shard = ShardObjectHelper.newShard(shardId, parentShardId, adjacentParentShardId, null);
        kinesisShards.put(shardId, shard);

        Map<String, Boolean> memoizationContext = new HashMap<>();
        Assert.assertTrue(ShardSyncer.checkIfDescendantAndAddNewLeasesForAncestors(shardId, INITIAL_POSITION_LATEST,
                shardIdsOfCurrentLeases, kinesisShards, newLeaseMap, memoizationContext));
        Assert.assertEquals(1, newLeaseMap.size());
        Assert.assertTrue(newLeaseMap.containsKey(adjacentParentShardId));
        KinesisClientLease adjacentParentLease = newLeaseMap.get(adjacentParentShardId);
        Assert.assertEquals(ExtendedSequenceNumber.LATEST, adjacentParentLease.getCheckpoint());
    }

    /**
     * Test getParentShardIds() when the shard has no parents.
     */
    @Test
    public final void testGetParentShardIdsNoParents() {
        Shard shard = new Shard();
        Assert.assertTrue(ShardSyncer.getParentShardIds(shard, null).isEmpty());
    }

    /**
     * Test getParentShardIds() when the shard has no parents.
     */
    @Test
    public final void testGetParentShardIdsTrimmedParents() {
        Map<String, Shard> shardMap = new HashMap<String, Shard>();
        Shard shard = ShardObjectHelper.newShard("shardId-test", "foo", "bar", null);
        Assert.assertTrue(ShardSyncer.getParentShardIds(shard, shardMap).isEmpty());
    }

    /**
     * Test getParentShardIds() when the shard has a single parent.
     */
    @Test
    public final void testGetParentShardIdsSingleParent() {
        Map<String, Shard> shardMap = new HashMap<String, Shard>();

        String parentShardId = "shardId-parent";
        shardMap.put(parentShardId, ShardObjectHelper.newShard(parentShardId, null, null, null));

        Shard shard = ShardObjectHelper.newShard("shardId-test", parentShardId, null, null);
        Set<String> parentShardIds = ShardSyncer.getParentShardIds(shard, shardMap);
        Assert.assertEquals(1, parentShardIds.size());
        Assert.assertTrue(parentShardIds.contains(parentShardId));

        shard.setParentShardId(null);
        parentShardIds = ShardSyncer.getParentShardIds(shard, shardMap);
        Assert.assertTrue(parentShardIds.isEmpty());

        shard.setAdjacentParentShardId(parentShardId);
        parentShardIds = ShardSyncer.getParentShardIds(shard, shardMap);
        Assert.assertEquals(1, parentShardIds.size());
        Assert.assertTrue(parentShardIds.contains(parentShardId));
    }

    /**
     * Test getParentShardIds() when the shard has two parents, one is trimmed.
     */
    @Test
    public final void testGetParentShardIdsOneTrimmedParent() {
        Map<String, Shard> shardMap = new HashMap<String, Shard>();

        String parentShardId = "shardId-parent";
        Shard parent = ShardObjectHelper.newShard(parentShardId, null, null, null);

        String adjacentParentShardId = "shardId-adjacentParent";
        Shard adjacentParent = ShardObjectHelper.newShard(adjacentParentShardId, null, null, null);

        Shard shard = ShardObjectHelper.newShard("shardId-test", parentShardId, adjacentParentShardId, null);

        shardMap.put(parentShardId, parent);
        Set<String> parentShardIds = ShardSyncer.getParentShardIds(shard, shardMap);
        Assert.assertEquals(1, parentShardIds.size());
        Assert.assertTrue(parentShardIds.contains(parentShardId));

        shardMap.remove(parentShardId);
        parentShardIds = ShardSyncer.getParentShardIds(shard, shardMap);
        Assert.assertTrue(parentShardIds.isEmpty());

        shardMap.put(adjacentParentShardId, adjacentParent);
        parentShardIds = ShardSyncer.getParentShardIds(shard, shardMap);
        Assert.assertEquals(1, parentShardIds.size());
        Assert.assertTrue(parentShardIds.contains(adjacentParentShardId));
    }

    /**
     * Test getParentShardIds() when the shard has two parents.
     */
    @Test
    public final void testGetParentShardIdsTwoParents() {
        Map<String, Shard> shardMap = new HashMap<String, Shard>();

        String parentShardId = "shardId-parent";
        shardMap.put(parentShardId, ShardObjectHelper.newShard(parentShardId, null, null, null));

        String adjacentParentShardId = "shardId-adjacentParent";
        shardMap.put(adjacentParentShardId, ShardObjectHelper.newShard(adjacentParentShardId, null, null, null));

        Shard shard = ShardObjectHelper.newShard("shardId-test", parentShardId, adjacentParentShardId, null);

        Set<String> parentShardIds = ShardSyncer.getParentShardIds(shard, shardMap);
        Assert.assertEquals(2, parentShardIds.size());
        Assert.assertTrue(parentShardIds.contains(parentShardId));
        Assert.assertTrue(parentShardIds.contains(adjacentParentShardId));
    }

    /**
     */
    @Test
    public final void testNewLease() {
        Shard shard = new Shard();
        String shardId = "shardId-95";
        shard.setShardId(shardId);
        String parentShardId = "shardId-parent";
        String adjacentParentShardId = "shardId-adjacentParent";
        shard.setParentShardId(parentShardId);
        shard.setAdjacentParentShardId(adjacentParentShardId);

        KinesisClientLease lease = ShardSyncer.newKCLLease(shard);
        Assert.assertEquals(shardId, lease.getLeaseKey());
        Assert.assertNull(lease.getCheckpoint());
        Set<String> parentIds = lease.getParentShardIds();
        Assert.assertEquals(2, parentIds.size());
        Assert.assertTrue(parentIds.contains(parentShardId));
        Assert.assertTrue(parentIds.contains(adjacentParentShardId));
    }

    /**
     * Test method for constructShardIdToShardMap.
     *
     * .
     */
    @Test
    public final void testConstructShardIdToShardMap() {
        List<Shard> shards = new ArrayList<Shard>(2);
        shards.add(ShardObjectHelper.newShard("shardId-0", null, null, null));
        shards.add(ShardObjectHelper.newShard("shardId-1", null, null, null));

        Map<String, Shard> shardIdToShardMap = ShardSyncer.constructShardIdToShardMap(shards);
        Assert.assertEquals(shards.size(), shardIdToShardMap.size());
        for (Shard shard : shards) {
            Assert.assertSame(shard, shardIdToShardMap.get(shard.getShardId()));
        }
    }

    /**
     * Test getOpenShards() - no shards are open.
     */
    @Test
    public final void testGetOpenShardsNoneOpen() {
        List<Shard> shards = new ArrayList<Shard>();
        shards.add(ShardObjectHelper.newShard("shardId-9384", null, null,
                ShardObjectHelper.newSequenceNumberRange("123", "345")));
        Assert.assertTrue(ShardSyncer.getOpenShards(shards).isEmpty());
    }

    /**
     * Test getOpenShards() - test null and max end sequence number.
     */
    @Test
    public final void testGetOpenShardsNullAndMaxEndSeqNum() {
        List<Shard> shards = new ArrayList<Shard>();
        String shardId = "shardId-2738";
        SequenceNumberRange sequenceNumberRange = ShardObjectHelper.newSequenceNumberRange("123", null);
        shards.add(ShardObjectHelper.newShard(shardId, null, null, sequenceNumberRange));

        // Verify shard is considered open when it has a null end sequence number
        List<Shard> openShards = ShardSyncer.getOpenShards(shards);
        Assert.assertEquals(1, openShards.size());
        Assert.assertEquals(shardId, openShards.get(0).getShardId());

        // Close shard before testing for max sequence number
        sequenceNumberRange.setEndingSequenceNumber("1000");
        openShards = ShardSyncer.getOpenShards(shards);
        Assert.assertTrue(openShards.isEmpty());

        // Verify shard is considered closed when the end sequence number is set to max allowed sequence number
        sequenceNumberRange.setEndingSequenceNumber(MAX_SEQUENCE_NUMBER.toString());
        openShards = ShardSyncer.getOpenShards(shards);
        Assert.assertEquals(0, openShards.size());
    }

    /**
     * Test isCandidateForCleanup
     * 
     * @throws KinesisClientLibIOException
     */
    @Test
    public final void testIsCandidateForCleanup() throws KinesisClientLibIOException {
        String parentShardId = "shardId-0000";
        String adjacentParentShardId = "shardId-0001";
        String shardId = "shardId-0002";
        KinesisClientLease lease = newLease(shardId);
        List<String> parentShardIds = new ArrayList<>();
        parentShardIds.add(parentShardId);
        parentShardIds.add(adjacentParentShardId);
        lease.setParentShardIds(parentShardIds);
        Set<String> currentKinesisShardIds = new HashSet<>();

        currentKinesisShardIds.add(shardId);
        Assert.assertFalse(ShardSyncer.isCandidateForCleanup(lease, currentKinesisShardIds));

        currentKinesisShardIds.clear();
        Assert.assertTrue(ShardSyncer.isCandidateForCleanup(lease, currentKinesisShardIds));

        currentKinesisShardIds.add(parentShardId);
        // Assert.assertFalse(ShardSyncer.isCandidateForCleanup(lease, currentKinesisShardIds));

        currentKinesisShardIds.clear();
        Assert.assertTrue(ShardSyncer.isCandidateForCleanup(lease, currentKinesisShardIds));

        currentKinesisShardIds.add(adjacentParentShardId);
        // Assert.assertFalse(ShardSyncer.isCandidateForCleanup(lease, currentKinesisShardIds));
        currentKinesisShardIds.add(parentShardId);
        // Assert.assertFalse(ShardSyncer.isCandidateForCleanup(lease, currentKinesisShardIds));
        currentKinesisShardIds.add(shardId);
        Assert.assertFalse(ShardSyncer.isCandidateForCleanup(lease, currentKinesisShardIds));
    }

    /**
     * Test isCandidateForCleanup
     * 
     * @throws KinesisClientLibIOException
     */
    @Test(expected = KinesisClientLibIOException.class)
    public final void testIsCandidateForCleanupParentExists() throws KinesisClientLibIOException {
        String parentShardId = "shardId-0000";
        String adjacentParentShardId = "shardId-0001";
        String shardId = "shardId-0002";
        KinesisClientLease lease = newLease(shardId);
        List<String> parentShardIds = new ArrayList<>();
        parentShardIds.add(parentShardId);
        parentShardIds.add(adjacentParentShardId);
        lease.setParentShardIds(parentShardIds);
        Set<String> currentKinesisShardIds = new HashSet<>();

        currentKinesisShardIds.add(parentShardId);
        Assert.assertFalse(ShardSyncer.isCandidateForCleanup(lease, currentKinesisShardIds));
    }

    /**
     * Test isCandidateForCleanup
     * 
     * @throws KinesisClientLibIOException
     */
    @Test(expected = KinesisClientLibIOException.class)
    public final void testIsCandidateForCleanupAdjacentParentExists() throws KinesisClientLibIOException {
        String parentShardId = "shardId-0000";
        String adjacentParentShardId = "shardId-0001";
        String shardId = "shardId-0002";
        KinesisClientLease lease = newLease(shardId);
        List<String> parentShardIds = new ArrayList<>();
        parentShardIds.add(parentShardId);
        parentShardIds.add(adjacentParentShardId);
        lease.setParentShardIds(parentShardIds);
        Set<String> currentKinesisShardIds = new HashSet<>();

        currentKinesisShardIds.add(adjacentParentShardId);
        Assert.assertFalse(ShardSyncer.isCandidateForCleanup(lease, currentKinesisShardIds));
    }

    /**
     * Test cleanup of lease for a shard that has been fully processed (and processing of child shards has begun).
     * 
     * @throws DependencyException
     * @throws InvalidStateException
     * @throws ProvisionedThroughputException
     */
    @Test
    public final void testCleanupLeaseForClosedShard()
            throws DependencyException, InvalidStateException, ProvisionedThroughputException {
        String closedShardId = "shardId-2";
        KinesisClientLease leaseForClosedShard = newLease(closedShardId);
        leaseForClosedShard.setCheckpoint(new ExtendedSequenceNumber("1234"));
        leaseManager.createLeaseIfNotExists(leaseForClosedShard);

        Set<String> childShardIds = new HashSet<>();
        List<KinesisClientLease> trackedLeases = new ArrayList<>();
        Set<String> parentShardIds = new HashSet<>();
        parentShardIds.add(closedShardId);
        String childShardId1 = "shardId-5";
        KinesisClientLease childLease1 = newLease(childShardId1);
        childLease1.setParentShardIds(parentShardIds);
        childLease1.setCheckpoint(ExtendedSequenceNumber.TRIM_HORIZON);
        String childShardId2 = "shardId-7";
        KinesisClientLease childLease2 = newLease(childShardId2);
        childLease2.setParentShardIds(parentShardIds);
        childLease2.setCheckpoint(ExtendedSequenceNumber.TRIM_HORIZON);
        Map<String, KinesisClientLease> trackedLeaseMap = ShardSyncer.constructShardIdToKCLLeaseMap(trackedLeases);

        // empty list of leases
        ShardSyncer.cleanupLeaseForClosedShard(closedShardId, childShardIds, trackedLeaseMap, leaseManager);
        Assert.assertNotNull(leaseManager.getLease(closedShardId));

        // closed shard has not been fully processed yet (checkpoint != SHARD_END)
        trackedLeases.add(leaseForClosedShard);
        trackedLeaseMap = ShardSyncer.constructShardIdToKCLLeaseMap(trackedLeases);
        ShardSyncer.cleanupLeaseForClosedShard(closedShardId, childShardIds, trackedLeaseMap, leaseManager);
        Assert.assertNotNull(leaseManager.getLease(closedShardId));

        // closed shard has been fully processed yet (checkpoint == SHARD_END)
        leaseForClosedShard.setCheckpoint(ExtendedSequenceNumber.SHARD_END);
        leaseManager.updateLease(leaseForClosedShard);
        ShardSyncer.cleanupLeaseForClosedShard(closedShardId, childShardIds, trackedLeaseMap, leaseManager);
        Assert.assertNull(leaseManager.getLease(closedShardId));

        // lease for only one child exists
        childShardIds.add(childShardId1);
        childShardIds.add(childShardId2);
        leaseManager.createLeaseIfNotExists(leaseForClosedShard);
        leaseManager.createLeaseIfNotExists(childLease1);
        trackedLeases.add(childLease1);
        trackedLeaseMap = ShardSyncer.constructShardIdToKCLLeaseMap(trackedLeases);
        ShardSyncer.cleanupLeaseForClosedShard(closedShardId, childShardIds, trackedLeaseMap, leaseManager);
        Assert.assertNotNull(leaseManager.getLease(closedShardId));

        // leases for both children exists, but they are both at TRIM_HORIZON
        leaseManager.createLeaseIfNotExists(childLease2);
        trackedLeases.add(childLease2);
        trackedLeaseMap = ShardSyncer.constructShardIdToKCLLeaseMap(trackedLeases);
        ShardSyncer.cleanupLeaseForClosedShard(closedShardId, childShardIds, trackedLeaseMap, leaseManager);
        Assert.assertNotNull(leaseManager.getLease(closedShardId));

        // leases for both children exists, one is at TRIM_HORIZON
        childLease1.setCheckpoint(new ExtendedSequenceNumber("34890"));
        leaseManager.updateLease(childLease1);
        ShardSyncer.cleanupLeaseForClosedShard(closedShardId, childShardIds, trackedLeaseMap, leaseManager);
        Assert.assertNotNull(leaseManager.getLease(closedShardId));

        // leases for both children exists, NONE of them are at TRIM_HORIZON
        childLease2.setCheckpoint(new ExtendedSequenceNumber("43789"));
        leaseManager.updateLease(childLease2);
        ShardSyncer.cleanupLeaseForClosedShard(closedShardId, childShardIds, trackedLeaseMap, leaseManager);
        Assert.assertNull(leaseManager.getLease(closedShardId));
    }

    /**
     * Test we can handle trimmed Kinesis shards (absent from the shard list), and valid closed shards.
     * 
     * @throws KinesisClientLibIOException
     */
    @Test
    public final void testAssertShardCoveredOrAbsentTestAbsentAndValid() throws KinesisClientLibIOException {
        List<Shard> shards = new ArrayList<>();
        String expectedClosedShardId = "shardId-34098";
        SequenceNumberRange sequenceNumberRange = ShardObjectHelper.newSequenceNumberRange("103", "205");
        HashKeyRange hashKeyRange = ShardObjectHelper.newHashKeyRange("10", "25");
        Shard closedShard = ShardObjectHelper.newShard(expectedClosedShardId, null, null, sequenceNumberRange,
                hashKeyRange);
        SequenceNumberRange childSequenceNumberRange = ShardObjectHelper.newSequenceNumberRange("206", "300");
        Shard child1 = ShardObjectHelper.newShard("shardId-54879", expectedClosedShardId, null,
                childSequenceNumberRange);
        Map<String, Shard> shardIdToShardMap = ShardSyncer.constructShardIdToShardMap(shards);
        Map<String, Set<String>> shardIdToChildShardIdsMap = ShardSyncer
                .constructShardIdToChildShardIdsMap(shardIdToShardMap);
        Set<String> closedShardIds = new HashSet<>();
        closedShardIds.add(expectedClosedShardId);
        ShardSyncer.assertClosedShardsAreCoveredOrAbsent(shardIdToShardMap, shardIdToChildShardIdsMap,
                closedShardIds);

        // test for case where shard has been trimmed (absent from list)
        ShardSyncer.assertClosedShardsAreCoveredOrAbsent(shardIdToShardMap, shardIdToChildShardIdsMap,
                closedShardIds);

        // Populate shards.
        shards.add(closedShard);
        shards.add(child1);
        shardIdToShardMap.put(expectedClosedShardId, closedShard);
        shardIdToShardMap.put(child1.getShardId(), child1);
        shardIdToChildShardIdsMap = ShardSyncer.constructShardIdToChildShardIdsMap(shardIdToShardMap);

        // test degenerate split/merge
        child1.setHashKeyRange(hashKeyRange);
        ShardSyncer.assertClosedShardsAreCoveredOrAbsent(shardIdToShardMap, shardIdToChildShardIdsMap,
                closedShardIds);

        // test merge
        child1.setHashKeyRange(ShardObjectHelper.newHashKeyRange("10", "2985"));
        ShardSyncer.assertClosedShardsAreCoveredOrAbsent(shardIdToShardMap, shardIdToChildShardIdsMap,
                closedShardIds);
        child1.setHashKeyRange(ShardObjectHelper.newHashKeyRange("3", "25"));
        ShardSyncer.assertClosedShardsAreCoveredOrAbsent(shardIdToShardMap, shardIdToChildShardIdsMap,
                closedShardIds);

        // test split
        HashKeyRange childHashKeyRange1 = ShardObjectHelper.newHashKeyRange("10", "15");
        HashKeyRange childHashKeyRange2 = ShardObjectHelper.newHashKeyRange("16", "25");
        child1.setHashKeyRange(childHashKeyRange1);
        Shard child2 = ShardObjectHelper.newShard("shardId-43789", null, expectedClosedShardId,
                childSequenceNumberRange, childHashKeyRange2);
        shards.add(child2);
        shardIdToShardMap.put(child2.getShardId(), child2);
        shardIdToChildShardIdsMap = ShardSyncer.constructShardIdToChildShardIdsMap(shardIdToShardMap);
        ShardSyncer.assertClosedShardsAreCoveredOrAbsent(shardIdToShardMap, shardIdToChildShardIdsMap,
                closedShardIds);
    }

    /**
     * Test we throw an exception if the shard is open
     * 
     * @throws KinesisClientLibIOException
     */
    @Test(expected = KinesisClientLibIOException.class)
    public final void testAssertShardCoveredOrAbsentTestOpen() throws KinesisClientLibIOException {
        List<Shard> shards = new ArrayList<>();
        String expectedClosedShardId = "shardId-34098";
        SequenceNumberRange sequenceNumberRange = ShardObjectHelper.newSequenceNumberRange("103", null);
        HashKeyRange hashKeyRange = ShardObjectHelper.newHashKeyRange("10", "25");
        Shard openShard = ShardObjectHelper.newShard(expectedClosedShardId, null, null, sequenceNumberRange,
                hashKeyRange);
        shards.add(openShard);
        Map<String, Shard> shardIdToShardMap = ShardSyncer.constructShardIdToShardMap(shards);
        Map<String, Set<String>> shardIdToChildShardIdsMap = ShardSyncer
                .constructShardIdToChildShardIdsMap(shardIdToShardMap);
        Set<String> closedShardIds = new HashSet<>();
        closedShardIds.add(expectedClosedShardId);
        ShardSyncer.assertClosedShardsAreCoveredOrAbsent(shardIdToShardMap, shardIdToChildShardIdsMap,
                closedShardIds);
    }

    /**
     * Test we throw an exception if there are no children
     * 
     * @throws KinesisClientLibIOException
     */
    @Test(expected = KinesisClientLibIOException.class)
    public final void testAssertShardCoveredOrAbsentTestNoChildren() throws KinesisClientLibIOException {
        List<Shard> shards = new ArrayList<>();
        String expectedClosedShardId = "shardId-34098";
        SequenceNumberRange sequenceNumberRange = ShardObjectHelper.newSequenceNumberRange("103", "205");
        HashKeyRange hashKeyRange = ShardObjectHelper.newHashKeyRange("10", "25");
        Shard closedShard = ShardObjectHelper.newShard(expectedClosedShardId, null, null, sequenceNumberRange,
                hashKeyRange);
        shards.add(closedShard);
        Map<String, Shard> shardIdToShardMap = ShardSyncer.constructShardIdToShardMap(shards);
        Map<String, Set<String>> shardIdToChildShardIdsMap = ShardSyncer
                .constructShardIdToChildShardIdsMap(shardIdToShardMap);
        Set<String> closedShardIds = new HashSet<>();
        closedShardIds.add(expectedClosedShardId);
        ShardSyncer.assertClosedShardsAreCoveredOrAbsent(shardIdToShardMap, shardIdToChildShardIdsMap,
                closedShardIds);
    }

    /**
     * Test we throw an exception if children don't cover hash key range (min of children > min of parent)
     * 
     * @throws KinesisClientLibIOException
     */
    @Test(expected = KinesisClientLibIOException.class)
    public final void testAssertShardCoveredOrAbsentTestIncompleteSplitMin() throws KinesisClientLibIOException {
        HashKeyRange hashKeyRange = ShardObjectHelper.newHashKeyRange("10", "25");
        HashKeyRange childHashKeyRange1 = ShardObjectHelper.newHashKeyRange("12", "15");
        HashKeyRange childHashKeyRange2 = ShardObjectHelper.newHashKeyRange("16", "25");
        testAssertShardCoveredOrAbsentTestIncompleteSplit(hashKeyRange, childHashKeyRange1, childHashKeyRange2);
    }

    /**
     * Test we throw an exception if children don't cover hash key range (max of children < max of parent)
     * 
     * @throws KinesisClientLibIOException
     */
    @Test(expected = KinesisClientLibIOException.class)
    public final void testAssertShardCoveredOrAbsentTestIncompleteSplitMax() throws KinesisClientLibIOException {
        HashKeyRange hashKeyRange = ShardObjectHelper.newHashKeyRange("10", "25");
        HashKeyRange childHashKeyRange1 = ShardObjectHelper.newHashKeyRange("10", "15");
        HashKeyRange childHashKeyRange2 = ShardObjectHelper.newHashKeyRange("16", "23");
        testAssertShardCoveredOrAbsentTestIncompleteSplit(hashKeyRange, childHashKeyRange1, childHashKeyRange2);
    }

    private void testAssertShardCoveredOrAbsentTestIncompleteSplit(HashKeyRange parentHashKeyRange,
            HashKeyRange child1HashKeyRange, HashKeyRange child2HashKeyRange) throws KinesisClientLibIOException {
        List<Shard> shards = new ArrayList<>();
        String expectedClosedShardId = "shardId-34098";
        SequenceNumberRange sequenceNumberRange = ShardObjectHelper.newSequenceNumberRange("103", "205");
        Shard closedShard = ShardObjectHelper.newShard(expectedClosedShardId, null, null, sequenceNumberRange,
                parentHashKeyRange);
        shards.add(closedShard);

        SequenceNumberRange childSequenceNumberRange = ShardObjectHelper.newSequenceNumberRange("206", "300");
        Shard child1 = ShardObjectHelper.newShard("shardId-43789", null, expectedClosedShardId,
                childSequenceNumberRange, child1HashKeyRange);
        shards.add(child1);
        Shard child2 = ShardObjectHelper.newShard("shardId-43789", null, expectedClosedShardId,
                childSequenceNumberRange, child2HashKeyRange);
        shards.add(child2);

        Map<String, Shard> shardIdToShardMap = ShardSyncer.constructShardIdToShardMap(shards);
        Map<String, Set<String>> shardIdToChildShardIdsMap = ShardSyncer
                .constructShardIdToChildShardIdsMap(shardIdToShardMap);
        Set<String> closedShardIds = new HashSet<>();
        closedShardIds.add(expectedClosedShardId);
        ShardSyncer.assertClosedShardsAreCoveredOrAbsent(shardIdToShardMap, shardIdToChildShardIdsMap,
                closedShardIds);
    }

    /**
     * Helper method.
     * 
     * @param shardId
     * @return
     */
    private KinesisClientLease newLease(String shardId) {
        KinesisClientLease lease = new KinesisClientLease();
        lease.setLeaseKey(shardId);

        return lease;
    }

}