jp.classmethod.aws.dynamodb.DynamoDbRepository.java Source code

Java tutorial

Introduction

Here is the source code for jp.classmethod.aws.dynamodb.DynamoDbRepository.java

Source

/*
 * Copyright 2016 Classmethod, Inc. or its affiliates. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License").
 * You may not use this file except in compliance with the License.
 * A copy of the License is located at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * or in the "license" file accompanying this file. This file is distributed
 * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
 * express or implied. See the License for the specific language governing
 * permissions and limitations under the License.
 */
package jp.classmethod.aws.dynamodb;

import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.IOException;
import java.io.Serializable;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;

import lombok.extern.slf4j.Slf4j;

import org.springframework.beans.factory.InitializingBean;
import org.springframework.dao.DataAccessException;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.dao.IncorrectResultSizeDataAccessException;
import org.springframework.dao.InvalidDataAccessResourceUsageException;
import org.springframework.dao.NonTransientDataAccessResourceException;
import org.springframework.dao.OptimisticLockingFailureException;
import org.springframework.dao.QueryTimeoutException;
import org.springframework.data.domain.Sort;
import org.springframework.data.repository.NoRepositoryBean;

import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
import com.amazonaws.services.dynamodbv2.document.Attribute;
import com.amazonaws.services.dynamodbv2.document.Item;
import com.amazonaws.services.dynamodbv2.document.ItemCollection;
import com.amazonaws.services.dynamodbv2.document.KeyAttribute;
import com.amazonaws.services.dynamodbv2.document.PrimaryKey;
import com.amazonaws.services.dynamodbv2.document.QueryOutcome;
import com.amazonaws.services.dynamodbv2.document.ScanOutcome;
import com.amazonaws.services.dynamodbv2.document.Table;
import com.amazonaws.services.dynamodbv2.document.UpdateItemOutcome;
import com.amazonaws.services.dynamodbv2.document.internal.InternalUtils;
import com.amazonaws.services.dynamodbv2.document.spec.DeleteItemSpec;
import com.amazonaws.services.dynamodbv2.document.spec.GetItemSpec;
import com.amazonaws.services.dynamodbv2.document.spec.PutItemSpec;
import com.amazonaws.services.dynamodbv2.document.spec.QuerySpec;
import com.amazonaws.services.dynamodbv2.document.spec.ScanSpec;
import com.amazonaws.services.dynamodbv2.document.spec.UpdateItemSpec;
import com.amazonaws.services.dynamodbv2.document.spec.UpdateTableSpec;
import com.amazonaws.services.dynamodbv2.model.AttributeDefinition;
import com.amazonaws.services.dynamodbv2.model.AttributeValue;
import com.amazonaws.services.dynamodbv2.model.BatchGetItemRequest;
import com.amazonaws.services.dynamodbv2.model.BatchGetItemResult;
import com.amazonaws.services.dynamodbv2.model.ConditionalCheckFailedException;
import com.amazonaws.services.dynamodbv2.model.CreateTableRequest;
import com.amazonaws.services.dynamodbv2.model.GlobalSecondaryIndex;
import com.amazonaws.services.dynamodbv2.model.GlobalSecondaryIndexUpdate;
import com.amazonaws.services.dynamodbv2.model.KeySchemaElement;
import com.amazonaws.services.dynamodbv2.model.KeyType;
import com.amazonaws.services.dynamodbv2.model.KeysAndAttributes;
import com.amazonaws.services.dynamodbv2.model.Projection;
import com.amazonaws.services.dynamodbv2.model.ProjectionType;
import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughput;
import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughputDescription;
import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughputExceededException;
import com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException;
import com.amazonaws.services.dynamodbv2.model.ReturnValue;
import com.amazonaws.services.dynamodbv2.model.ScalarAttributeType;
import com.amazonaws.services.dynamodbv2.model.TableDescription;
import com.amazonaws.services.dynamodbv2.model.UpdateGlobalSecondaryIndexAction;
import com.amazonaws.services.dynamodbv2.xspec.ExpressionSpecBuilder;
import com.amazonaws.services.dynamodbv2.xspec.PutItemExpressionSpec;
import com.amazonaws.services.dynamodbv2.xspec.UpdateItemExpressionSpec;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.github.fge.jsonpatch.JsonPatch;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;

import jp.xet.sparwings.spring.data.chunk.Chunk;
import jp.xet.sparwings.spring.data.chunk.ChunkImpl;
import jp.xet.sparwings.spring.data.chunk.Chunkable;
import jp.xet.sparwings.spring.data.repository.BatchReadableRepository;
import jp.xet.sparwings.spring.data.repository.ChunkableRepository;
import jp.xet.sparwings.spring.data.repository.ConditionalUpdatableRepository;
import jp.xet.sparwings.spring.data.repository.CreatableRepository;
import jp.xet.sparwings.spring.data.repository.DeletableRepository;
import jp.xet.sparwings.spring.data.repository.TruncatableRepository;

/**
 * @author Alexander Patrikalakis
 * @since #version#
 *
 * This class contains the basic logic and implementations of some of the methods of the
 * many BaseRepository interfaces, when the backing store is DynamoDB
 *
 * @param <E> the entity type to be persisted
 */
@Slf4j
@NoRepositoryBean
public abstract class DynamoDbRepository<E, K extends Serializable>
        implements InitializingBean, ChunkableRepository<E, K>, BatchReadableRepository<E, K>,
        CreatableRepository<E, K>, DeletableRepository<E, K>, PatchableRepository<E, K>,
        ConditionalUpdatableRepository<E, K, VersionCondition>, GetDeletableRepository<E, K>,
        TruncatableRepository<E, K> {

    public static final String UPDATE_FAILED_ENTITY_NOT_FOUND = "update failed because entity not found";

    public static final String UPDATE_FAILED_NOT_FOUND_OR_BAD_VERSION = UPDATE_FAILED_ENTITY_NOT_FOUND
            + " or the version was wrong or JSON patch conditions were not met";

    private static final String EXPRESSION_REFERS_TO_NON_EXTANT_ATTRIBUTE = "The provided expression refers to an attribute that does not exist in the item";

    private static final String VALIDATION_EXCEPTION = "ValidationException";

    private ProvisionedThroughput convert(ProvisionedThroughputDescription d) {
        return new ProvisionedThroughput(d.getReadCapacityUnits(), d.getWriteCapacityUnits());
    }

    /**
     * Creates a GSI configuration object
     *
     * @param name             name of the index object to create
     * @param hashKey          hash key of the index
     * @param rangeKey         range key of the index
     * @param nonKeyAttributes determines the projection type and top level projected attributes.
     *                         if null, ALL attributes are projected. if an empty list, KEYS_ONLY are projected.
     *                         if the list has elements, only the elements INCLUDED in the list are projected
     * @return a description of a global secondary index that can be used to create a table.
     */
    protected static GlobalSecondaryIndex createGlobalSecondaryIndex(String name, String hashKey, String rangeKey,
            List<String> nonKeyAttributes) {
        Preconditions.checkArgument(false == Strings.isNullOrEmpty(hashKey));
        final KeySchemaElement hks = new KeySchemaElement(hashKey, KeyType.HASH);

        final Projection projection;
        if (nonKeyAttributes == null) {
            projection = new Projection().withProjectionType(ProjectionType.ALL);
        } else if (nonKeyAttributes.isEmpty()) {
            projection = new Projection().withProjectionType(ProjectionType.KEYS_ONLY);
        } else {
            projection = new Projection().withProjectionType(ProjectionType.INCLUDE)
                    .withNonKeyAttributes(nonKeyAttributes);
        }

        final GlobalSecondaryIndex result = new GlobalSecondaryIndex().withIndexName(name)
                .withProjection(projection);
        if (Strings.isNullOrEmpty(rangeKey)) {
            result.withKeySchema(hks);
        } else {
            result.withKeySchema(hks, new KeySchemaElement(rangeKey, KeyType.RANGE));
        }
        return result;
    }

    /** document sdk table wrapper */
    protected final Table table;

    /**scalar key type definitions**/
    final Map<String, ScalarAttributeType> definitions;

    /**base table primary keys**/
    final List<KeySchemaElement> schemata;

    /**the key condition to use when persisting a new entity**/
    final String conditionalCreateCondition;

    /**base table hash key name**/
    final String hashKeyName;

    /**base table range key name. Null if hash only schema**/
    final String rangeKeyName;

    /**hash key names of the gsis**/
    protected final Map<String, String> gsiHashKeys;

    /**range key names of the gsis**/
    protected final Map<String, String> gsiRangeKeys;

    protected final AmazonDynamoDB dynamoDB;

    private final Map<String, ProvisionedThroughput> ptMap;

    private final Map<String, GlobalSecondaryIndex> gsis;

    private final Map<String, String> lookupKeyConditions;

    private final ObjectMapper objectMapper;

    private final String conditionalDeleteCondition;

    private final Class<E> clazz;

    private final String tableNameSuffix;

    private final String versionProperty;

    /**
     * Create instance.
     *
     * @param prefix the table prefix
     * @param tableNameSuffix the suffix for the table name
     * @param amazonDynamoDB dynamodb client
     * @param provisionedThroughputMap map of provisioned througput
     * @param objectMapper mapper to use for back/forth to json
     * @param clazz class reference of E
     * @param attributeDefinitions types of keys on base table and GSI
     * @param baseTableKeyNames names of base table keys
     * @param gsiList list of GSI definitions
     * @param versionString
     * @since #version#
     */
    protected DynamoDbRepository(String prefix, String tableNameSuffix, AmazonDynamoDB amazonDynamoDB,
            Map<String, ProvisionedThroughput> provisionedThroughputMap, ObjectMapper objectMapper, Class<E> clazz,
            Map<String, ScalarAttributeType> attributeDefinitions, List<String> baseTableKeyNames,
            Map<String, GlobalSecondaryIndex> gsiList, String versionString) {
        Preconditions.checkNotNull(amazonDynamoDB);
        Preconditions.checkArgument(false == Strings.isNullOrEmpty(tableNameSuffix));
        Preconditions.checkNotNull(provisionedThroughputMap);
        Preconditions.checkNotNull(attributeDefinitions);
        this.dynamoDB = amazonDynamoDB;
        this.tableNameSuffix = tableNameSuffix;
        final String tableName = Strings.isNullOrEmpty(prefix) ? tableNameSuffix
                : String.format(Locale.ENGLISH, "%s_%s", prefix, tableNameSuffix);
        this.table = new Table(this.dynamoDB, tableName);
        this.ptMap = provisionedThroughputMap;
        this.gsis = gsiList != null ? new HashMap<>() : null;
        this.definitions = new HashMap<>(attributeDefinitions);
        this.lookupKeyConditions = new HashMap<>();
        this.objectMapper = objectMapper;
        this.clazz = clazz;
        this.gsiHashKeys = new HashMap<>();
        this.gsiRangeKeys = new HashMap<>();
        this.versionProperty = Strings.isNullOrEmpty(versionString) ? null : versionString;
        Optional.ofNullable(gsiList).orElse(new HashMap<>()).values().forEach(gsi -> {
            final String indexName = gsi.getIndexName();
            //make a copy
            final GlobalSecondaryIndex copy = new GlobalSecondaryIndex().withIndexName(gsi.getIndexName())
                    .withKeySchema(gsi.getKeySchema()).withProjection(gsi.getProjection())
                    .withProvisionedThroughput(ptMap.get(indexName));
            final String hk = copy.getKeySchema().get(0).getAttributeName();
            final String rk = copy.getKeySchema().size() == 2 ? copy.getKeySchema().get(1).getAttributeName()
                    : null;
            this.gsis.put(indexName, copy);
            this.gsiHashKeys.put(indexName, hk);
            if (rk != null) {
                lookupKeyConditions.put(indexName,
                        String.format(Locale.ENGLISH, "%s = :%s and %s = :%s", hk, hk, rk, rk));
                this.gsiRangeKeys.put(indexName, rk);
            } else {
                lookupKeyConditions.put(indexName, String.format(Locale.ENGLISH, "%s = :%s", hk, hk));
            }
        });

        Preconditions.checkNotNull(baseTableKeyNames);
        Preconditions.checkArgument(false == baseTableKeyNames.isEmpty(), "need at least one key");
        Preconditions.checkArgument(baseTableKeyNames.size() <= 2,
                "cant have more than two keys (one partition and one sort key)");

        //add the attribute definitions for the base table key schema
        baseTableKeyNames.stream().forEach(name -> Preconditions.checkArgument(this.definitions.containsKey(name)));
        this.schemata = Lists.newArrayList(new KeySchemaElement(baseTableKeyNames.get(0), KeyType.HASH));
        if (baseTableKeyNames.size() == 2) {
            schemata.add(new KeySchemaElement(baseTableKeyNames.get(1), KeyType.RANGE));
        }
        hashKeyName = schemata.get(0).getAttributeName();
        rangeKeyName = schemata.size() == 2 ? schemata.get(1).getAttributeName() : null;
        conditionalCreateCondition = rangeKeyName != null
                ? String.format(Locale.ENGLISH, "attribute_not_exists(%s) and attribute_not_exists(%s)",
                        hashKeyName, rangeKeyName)
                : String.format(Locale.ENGLISH, "attribute_not_exists(%s)", hashKeyName);
        conditionalDeleteCondition = rangeKeyName != null
                ? String.format(Locale.ENGLISH, "attribute_exists(%s) and attribute_exists(%s)", hashKeyName,
                        rangeKeyName)
                : String.format(Locale.ENGLISH, "attribute_exists(%s)", hashKeyName);
    }

    @Override
    public void afterPropertiesSet() {
        open();
    }

    private TableDescription updateTable(TableDescription desc) {
        Preconditions.checkNotNull(desc, "table description must not be null");
        UpdateTableSpec spec = null;
        if (false == ptMap.get(tableNameSuffix).equals(convert(desc.getProvisionedThroughput()))) {
            //if the throughput of the table is not the same as the throughput in the ptMap configuration,
            //update the thruput of the table
            spec = new UpdateTableSpec().withProvisionedThroughput(ptMap.get(tableNameSuffix));
        }
        final List<GlobalSecondaryIndexUpdate> gsiUpdates = new ArrayList<>();
        if (desc.getGlobalSecondaryIndexes() != null && false == desc.getGlobalSecondaryIndexes().isEmpty()) {
            //if the table description has updates to secondary indexes
            desc.getGlobalSecondaryIndexes().forEach(gsi -> {
                //for each gsi in the table description
                final String indexName = gsi.getIndexName();
                ProvisionedThroughput pt = ptMap.get(indexName);
                if (pt != null && false == pt.equals(convert(gsi.getProvisionedThroughput()))) {
                    //if the throughput of the gsi in the description is not the same as the throughput in the pt map
                    //add an update to the gsi's thruput
                    gsiUpdates
                            .add(new GlobalSecondaryIndexUpdate().withUpdate(new UpdateGlobalSecondaryIndexAction()
                                    .withIndexName(indexName).withProvisionedThroughput(pt)));
                }
            });
        }
        if (false == gsiUpdates.isEmpty()) {
            if (spec == null) {
                spec = new UpdateTableSpec();
            }
            spec.withGlobalSecondaryIndexUpdates(gsiUpdates);
        }
        return spec == null ? null : table.updateTable(spec);
    }

    private CreateTableRequest createTableRequest() {
        return new CreateTableRequest().withTableName(table.getTableName())
                .withProvisionedThroughput(ptMap.get(tableNameSuffix)).withKeySchema(schemata)
                .withAttributeDefinitions(definitions.keySet().stream()
                        .map(name -> new AttributeDefinition(name, definitions.get(name)))
                        .collect(Collectors.toList()))
                .withGlobalSecondaryIndexes(gsis == null ? null : gsis.values());
    }

    public String tableName() {
        return table.getTableName();
    }

    @Override
    public E findOne(K keys) {
        //interface specifies throw IllegalArgumentException so use checkArgument instead
        Preconditions.checkArgument(keys != null, "keys must not be null");
        //just read the item and return it
        final Item item;
        final PrimaryKey pk = createKeys(keys);
        GetItemSpec spec = new GetItemSpec().withPrimaryKey(pk);
        try {
            //TODO add projection expression for keys
            item = table.getItem(spec);
            return item == null ? null : convertItemToDomain(item);
        } catch (AmazonClientException e) {
            throw convertDynamoDBException(e, "read",
                    null /* conditionMessage is null because GetItem doesnt take a condition */);
        }
    }

    @Override
    public boolean exists(K keys) {
        Preconditions.checkArgument(keys != null, "keys must not be null");
        return null != findOne(keys);
    }

    /**
     * Translates low level database exceptions to application level exceptions
     *
     * @param e the low level amazon client exception
     * @param action crud action name
     * @param conditionalSupplier creates condition failed exception (context dependent)
     * @return a translation of the AWS/DynamoDB exception
     */
    DataAccessException convertDynamoDBException(AmazonClientException e, String action,
            Supplier<? extends DataAccessException> conditionalSupplier) {

        final String format = "unable to %s entity due to %s.";
        if (e instanceof ConditionalCheckFailedException) {
            return conditionalSupplier.get();
        } else if (e instanceof ProvisionedThroughputExceededException) {
            return new QueryTimeoutException(String.format(Locale.ENGLISH, format, action, "throttling"), e);
        } else if (e instanceof ResourceNotFoundException) {
            return new NonTransientDataAccessResourceException(
                    String.format(Locale.ENGLISH, format, action, "table does not exist"), e);
        } else if (e instanceof AmazonServiceException) {
            AmazonServiceException ase = (AmazonServiceException) e;
            if (VALIDATION_EXCEPTION.equals(((AmazonServiceException) e).getErrorCode())) {
                return new InvalidDataAccessResourceUsageException(
                        String.format(Locale.ENGLISH, format, action, "client error"), e);
            } else {
                return new DynamoDbServiceException(
                        String.format(Locale.ENGLISH, format, action, "DynamoDB service error"), ase);
            }
        } else {
            return new InvalidDataAccessResourceUsageException(
                    String.format(Locale.ENGLISH, format, action, "client error"), e);
        }
    }

    private Chunk<Item> getItemListForGsi(String indexName, QuerySpec spec) {
        Preconditions.checkNotNull(spec, "spec must not be null");
        final ItemCollection<QueryOutcome> outcome = table.getIndex(indexName).query(spec);

        List<Item> results = new ArrayList<>();
        try {
            outcome.pages().forEach(p -> {
                p.iterator().forEachRemaining(o -> results.add(o));
            });
        } catch (AmazonServiceException e) {
            throw convertDynamoDBException(e, "getting by spec: " + spec.toString(),
                    null /*no write condition exception*/);
        }
        Map<String, AttributeValue> lastEvaluatedKey = outcome.getLastLowLevelResult().getQueryResult()
                .getLastEvaluatedKey();
        String lastEvaluatedItemJson = lastEvaluatedKey == null ? null
                : Item.fromMap(InternalUtils.toSimpleMapValue(lastEvaluatedKey)).toJSON();

        return new ChunkImpl<>(results, lastEvaluatedItemJson, null /*chunkable*/);
    }

    /**
     * gets a full item from a GSI
     * @param indexName name of GSI
     * @param spec query spec for gsi query
     * @return a full item keyed at keys in the GSI. If the GSI does not project all attributes, will read the item from
     * the base table
     *
     * TODO, ideally, return a wrapper of the last evaluated key from the query result and the list, but since we are
     * not filtering on the server side, do this later.
     */
    private Chunk<E> getFromGSI(String indexName, QuerySpec spec, boolean isUnique) {
        Preconditions.checkNotNull(spec, "query spec was null");
        Chunk<Item> chunk = getItemListForGsi(indexName, spec);
        //check if the index was not a unique index
        if (isUnique) {
            Preconditions.checkState(chunk.getContent().size() < 2,
                    "the index had more than one item at spec=" + spec.toString());
        }

        if (ProjectionType
                .fromValue(gsis.get(indexName).getProjection().getProjectionType()) == ProjectionType.ALL) {
            //the GSI had the full item so return it.
            return new ChunkImpl<>(chunk.getContent().parallelStream().map(i -> convertItemToDomain(i))
                    .collect(Collectors.toList()), chunk.getPaginationToken(), null /*chunkable*/);
        }
        //else read the item from the base table
        try {
            List<AttributeValue> pks = chunk.getContent().stream().map(i -> getPrimaryKeyFromItem(i))
                    .map(pk -> Iterables.getOnlyElement(pk.getComponents().stream()
                            .filter(c -> c.getName().equals(hashKeyName)).map(Attribute::getValue)
                            .map(InternalUtils::toAttributeValue).collect(Collectors.toList())))
                    .collect(Collectors.toList());
            return new ChunkImpl<>(findAll(pks, true), chunk.getPaginationToken(), null /*chunkable*/);
        } catch (AmazonClientException e) {
            throw convertDynamoDBException(e, "read", null /*condition failed exception provider*/);
        }
    }

    @Override
    public E getAndDelete(K key, long version) {
        Preconditions.checkNotNull(key, "keys must not be null");
        Preconditions.checkArgument(version >= -1L, "version must be greater than or equal to -1");
        final PrimaryKey pk = createKeys(key);
        final boolean conditioning = version >= 0;
        final String actualCondition;
        final Map<String, Object> valueMap;
        final Map<String, String> nameMap;

        if (conditioning) {
            Preconditions.checkState(versionProperty != null);
            actualCondition = String.format(Locale.ENGLISH, "%s and #version = :v", conditionalDeleteCondition);
            valueMap = Collections.singletonMap(":v", version);
            nameMap = Collections.singletonMap("#version", versionProperty);
        } else {
            actualCondition = conditionalDeleteCondition;
            valueMap = null;
            nameMap = null;
        }
        final DeleteItemSpec spec = new DeleteItemSpec().withPrimaryKey(pk).withNameMap(nameMap)
                .withValueMap(valueMap).withConditionExpression(actualCondition)
                .withReturnValues(ReturnValue.ALL_OLD);
        final Item item;
        try {
            item = table.deleteItem(spec).getItem();
        } catch (AmazonClientException e) {
            throw convertDynamoDBException(e, "delete",
                    () -> convertConditionalCheckFailedExceptionForDelete(e, version, key));
        }
        return convertItemToDomain(item);
    }

    private DataAccessException convertConditionalCheckFailedExceptionForDelete(AmazonClientException e,
            long version, K key) {
        if (version == -1 || false == exists(key)) {
            return getNotFoundException("didnt delete since entity didnt exist", e);
        }
        return new OptimisticLockingFailureException("did not delete entity because of version mismatch", e);
    }

    @Override
    public Iterable<E> findAll(Iterable<K> ids) {
        Preconditions.checkNotNull(ids, "ids may not be null");
        List<AttributeValue> idList = Lists.newArrayList(ids).parallelStream()
                .map(DynamoDbInternalUtils::toAttributeValue).collect(Collectors.toList());
        return findAll(idList, true /*useParallelBatches*/);
    }

    private List<E> findAll(Iterable<AttributeValue> ids, boolean useParallelBatches) {
        Preconditions.checkNotNull(ids, "ids may not be null");
        List<AttributeValue> idList = Lists.newArrayList(ids);
        if (idList.isEmpty()) {
            return new ArrayList<>();
        }
        List<Map<String, AttributeValue>> resultantItems = new ArrayList<>();

        StreamSupport.stream(Iterables.partition(idList, 25).spliterator(), useParallelBatches).forEach(inner -> {
            BatchGetItemRequest req = new BatchGetItemRequest();
            KeysAndAttributes keysAndAttributes = new KeysAndAttributes();
            keysAndAttributes.setConsistentRead(true);
            keysAndAttributes.setKeys(
                    inner.stream().map(id -> ImmutableMap.of(hashKeyName, id)).collect(Collectors.toList()));
            String tableName = tableName();
            req.withRequestItems(ImmutableMap.of(tableName, keysAndAttributes));

            BatchGetItemResult result;

            do {
                try {
                    result = dynamoDB.batchGetItem(req);
                    resultantItems.addAll(result.getResponses().get(tableName));
                    req.setRequestItems(result.getUnprocessedKeys());
                } catch (AmazonClientException e) {
                    throw this.convertDynamoDBException(e, "batch get", null /*no conditions for reads*/);
                }
            } while (false == result.getUnprocessedKeys().isEmpty());
        });

        return resultantItems.stream().map(legacyItem -> Item.fromMap(InternalUtils.toSimpleMapValue(legacyItem)))
                .map(item -> convertItemToDomain(item)).collect(Collectors.toList());
    }

    private PrimaryKey getPrimaryKeyFromItem(Item item) {
        KeyAttribute[] keyAttributes = schemata.stream().map(keySchemaElement -> {
            final String attrName = keySchemaElement.getAttributeName();
            Preconditions.checkArgument(item.hasAttribute(attrName),
                    "must provide keys with " + attrName + " field set");
            return new KeyAttribute(attrName, item.get(attrName));
        }).toArray(KeyAttribute[]::new);
        return new PrimaryKey(keyAttributes);
    }

    @Override
    public E update(K key, JsonPatch patch, boolean increment, long version) {
        final PrimaryKey pk = createKeys(key);
        Preconditions.checkNotNull(patch, "patch must not be null");
        Preconditions.checkArgument(version >= -1);

        ExpressionSpecBuilder builder = patch.get();

        //add a condition on item existence
        builder.withCondition(ExpressionSpecBuilder.attribute_exists(hashKeyName));
        //add update expression for incrementing the version
        if (increment && versionProperty != null) {
            builder.addUpdate(ExpressionSpecBuilder.N(versionProperty)
                    .set(ExpressionSpecBuilder.N(versionProperty).plus(1L)));
        }
        //add version condition
        if (version >= 0) {
            Preconditions.checkState(versionProperty != null);
            builder.withCondition(ExpressionSpecBuilder.N(versionProperty).eq(version));
        }

        UpdateItemExpressionSpec spec = builder.buildForUpdate();
        Preconditions.checkArgument(false == Strings.isNullOrEmpty(spec.getUpdateExpression()),
                "patch may not be empty"); // TODO add mechanism to JSON patch to allow iterating over list of ops
        try {
            UpdateItemOutcome updateItemOutcome = table.updateItem(new UpdateItemSpec().withExpressionSpec(spec)
                    .withPrimaryKey(pk).withReturnValues(ReturnValue.ALL_NEW));
            return convertItemToDomain(updateItemOutcome.getItem());
        } catch (AmazonClientException e) {
            throw processUpdateItemException(key, e);
        }
    }

    protected DataAccessException processUpdateItemException(K key, AmazonClientException e) {
        final String format = "unable to update entity due to %s.";
        if (e instanceof ConditionalCheckFailedException) {
            if (null == findOne(key)) {
                return getNotFoundException(UPDATE_FAILED_ENTITY_NOT_FOUND, e);
            }
            return new OptimisticLockingFailureException(UPDATE_FAILED_NOT_FOUND_OR_BAD_VERSION, e);
        } else if (e instanceof ProvisionedThroughputExceededException) {
            throw new QueryTimeoutException(String.format(Locale.ENGLISH, format, "throttling"), e);
        } else if (e instanceof AmazonServiceException) {
            AmazonServiceException ase = (AmazonServiceException) e;
            if (VALIDATION_EXCEPTION.equals(ase.getErrorCode())) {
                if (EXPRESSION_REFERS_TO_NON_EXTANT_ATTRIBUTE.equals(ase.getErrorMessage())
                        && null == findOne(key)) {
                    // if no locking and we get a specific message, then it also means the item does not exist
                    return getNotFoundException(UPDATE_FAILED_ENTITY_NOT_FOUND, e);
                }
                return new InvalidDataAccessResourceUsageException(
                        String.format(Locale.ENGLISH, format, "client error"), e);
            } else {
                return new DynamoDbServiceException(String.format(Locale.ENGLISH, format, "DynamoDB service error"),
                        ase);
            }
        } else {
            return new InvalidDataAccessResourceUsageException(
                    String.format(Locale.ENGLISH, format, "client error"), e);
        }
    }

    private <T> String convertDomainToJSON(T domain) {
        try {
            return objectMapper.writeValueAsString(domain);
        } catch (JsonProcessingException e) {
            throw new IllegalArgumentException("unable to convert domain object to JSON", e);
        }
    }

    /**
     * converts a Jackson annotated domain object to a DynamoDB document (item)
     * @param domain the object to convert
     * @return a DynamoDB Document representation of the domain object
     */
    <T> Item convertDomainToItem(T domain) {
        return Item.fromJSON(convertDomainToJSON(domain));
    }

    protected E convertItemToDomain(Item item) {
        return convertItemToDomain(item, clazz);
    }

    protected E findOneByGsi(String gsiName, QuerySpec spec) {
        Chunk<E> chunk = getFromGSI(gsiName, spec, true /*isUnique*/);
        return Optional
                .ofNullable(chunk.getContent().isEmpty() ? null : Iterables.getOnlyElement(chunk.getContent()))
                .orElseThrow(() -> new IncorrectResultSizeDataAccessException(
                        "could not find one matching record for spec:" + spec.toString(), 1 /*expected*/,
                        0 /*actual*/));
    }

    protected Chunk<E> findAllByGsi(String gsiName, QuerySpec spec) {
        return getFromGSI(gsiName, spec, false /*isUnique*/);
    }

    <S extends E> S convertItemToDomain(Item item, Class<? extends S> crass) {
        try {
            if (item == null) {
                return null;
            }
            String json = item.toJSON();
            return objectMapper.readValue(json, crass);
        } catch (IOException e) {
            throw new IllegalStateException("unable to convert orders JSON to domain object", e);
        }
    }

    @Override
    public Chunk<E> findAll(Chunkable chunkable) {
        Preconditions.checkNotNull(chunkable);
        Preconditions.checkArgument(Sort.Direction.DESC != chunkable.getDirection(),
                "DynamoDB only supports scanning forwards");
        ScanSpec spec = new ScanSpec();
        if (false == Strings.isNullOrEmpty(chunkable.getPaginationToken())) {
            spec.withExclusiveStartKey(new PrimaryKey(hashKeyName, chunkable.getPaginationToken()));
        }
        spec.withMaxPageSize(chunkable.getMaxPageSize()).withMaxResultSize(chunkable.getMaxPageSize());

        final ItemCollection<ScanOutcome> results = table.scan(spec);

        final List<Item> itemList;
        try {
            itemList = Lists.newArrayList(results.iterator());
        } catch (AmazonClientException e) {
            throw convertDynamoDBException(e, "scan", null /* conditionMessage */);
        }
        final List<E> entities = itemList.stream().map(this::convertItemToDomain) //O(n)
                .collect(Collectors.toList()); //O(n)
        final Map<String, AttributeValue> lastEvaluatedKey = results.getLastLowLevelResult() == null ? null
                : results.getLastLowLevelResult().getScanResult().getLastEvaluatedKey();
        final String paginationToken = lastEvaluatedKey == null ? null : lastEvaluatedKey.get(hashKeyName).getS();
        return new ChunkImpl<>(entities, paginationToken, chunkable);
    }

    private PrimaryKey createKeys(K key) {
        return createKeys(hashKeyName, definitions.get(hashKeyName), key);
    }

    private static <K extends Serializable> PrimaryKey createKeys(String hashKeyName,
            ScalarAttributeType hashKeyType, K key) {
        Preconditions.checkNotNull(key);
        Preconditions.checkArgument(hashKeyType == scalarAttributeType(key));

        //hash key only
        return new PrimaryKey().addComponent(hashKeyName, key);
    }

    private static ScalarAttributeType scalarAttributeType(Object o) {
        if (o instanceof String) {
            return ScalarAttributeType.S;
        }
        if (o instanceof ByteBuffer) {
            return ScalarAttributeType.B;
        }
        if (o instanceof Number) {
            return ScalarAttributeType.N;
        }
        throw new IllegalArgumentException("Unsupported scalar type");
    }

    PutItemSpec putItemSpec(Item domainItem) {
        return new PutItemSpec().withItem(domainItem).withConditionExpression(conditionalCreateCondition);
    }

    @Override
    public void delete(E entity) {
        Preconditions.checkArgument(entity != null, "cannot delete null entity");
        delete(getId(entity));
    }

    @Override
    public void delete(K keys) {
        //not throwing NPE as per DeletableRepository spec (says IAE for null)
        Preconditions.checkArgument(keys != null, "keys may not be null");
        getAndDelete(keys, -1 /* version->not conditioning */);
    }

    public void open() {
        TableDescription desc = null;
        boolean tableNotFound = false;
        try {
            desc = table.describe();
        } catch (ResourceNotFoundException rnfe) {
            tableNotFound = true;
        }
        if (tableNotFound) {
            log.info("creating {}", table.getTableName());
            CreateTableRequest ctr = createTableRequest();
            try {
                dynamoDB.createTable(ctr);
            } catch (AmazonClientException e) {
                log.error(ctr.toString());
                throw convertDynamoDBException(e, "CreateTable", null /* conditionMessage */);
            }
            try {
                table.waitForActive();
                log.info("created {}", table.getTableName());
            } catch (InterruptedException ie) {
                throw new IllegalStateException("Unable to create table " + table.getTableName(), ie);
            }
        }
        log.debug("{} exists", table.getTableName());
        if (desc != null) { //if the table is not a newly created table
            if (null != updateTable(desc)) { //if there are updates
                log.info("updating {}", table.getTableName());
                try {
                    table.waitForActive();
                    log.info("updated {}", table.getTableName());
                } catch (InterruptedException e) {
                    throw new IllegalStateException("Unable to patch table " + table.getTableName(), e);
                }
            }
        }
    }

    @Override
    public <S extends E> S update(S domain, VersionCondition condition) {
        Preconditions.checkNotNull(domain, "domain must not be null");
        final Item domainItem = convertDomainToItem(domain);
        Preconditions.checkArgument(domainItem.hasAttribute(hashKeyName),
                "hash key must be set in domain object when updating: " + hashKeyName);

        ExpressionSpecBuilder builder = new ExpressionSpecBuilder();
        builder.withCondition(ExpressionSpecBuilder.S(hashKeyName).exists());
        if (condition != null) {
            Preconditions.checkState(versionProperty != null);
            builder.withCondition(ExpressionSpecBuilder.N(versionProperty).eq(condition.getVersion()));
        }
        PutItemExpressionSpec xSpec = builder.buildForPut();
        PutItemSpec spec = new PutItemSpec().withItem(domainItem).withExpressionSpec(xSpec);
        try {
            table.putItem(spec);
        } catch (AmazonClientException e) {
            throw processUpdateItemException(getId(domain), e);
        }
        // PutItem does not accept ReturnValue.ALL_NEW
        return domain;
    }

    private IncorrectResultSizeDataAccessException getNotFoundException(String msg, Throwable e) {
        return new IncorrectResultSizeDataAccessException(msg, 1 /*expected*/, e);
    }

    @Override
    public <S extends E> S update(S domain) {
        return update(domain, null);
    }

    @Override
    public <S extends E> S create(S domain) {
        if (domain == null) {
            return null;
        }
        final Item domainItem = convertDomainToItem(domain);

        // stackoverflow.com/questions/4460580/java-generics-why-someobject-getclass-doesnt-return-class-extends-t
        @SuppressWarnings("unchecked")
        final Class<? extends S> domainClass = (Class<? extends S>) domain.getClass();
        Item itemCreated = DynamoDbInternalUtils.cloneItem(domainItem, true /*filterEmptyStrings*/);
        PutItemSpec spec = putItemSpec(itemCreated);
        try {
            table.putItem(spec);
        } catch (AmazonClientException e) {
            throw convertDynamoDBException(e, "create",
                    () -> new DuplicateKeyException("uuid " + getId(domain) + " already exists", e));
        }
        return convertItemToDomain(itemCreated, domainClass);
    }

    @Override
    public void deleteAll() {
        try {
            table.delete();
            open();
        } catch (AmazonClientException e) {
            throw convertDynamoDBException(e, "delete table", null /*no condition supplier*/);
        }
    }
}