com.cloudant.sync.datastore.DatastoreImpl.java Source code

Java tutorial

Introduction

Here is the source code for com.cloudant.sync.datastore.DatastoreImpl.java

Source

/**
 * Original iOS version by  Jens Alfke, ported to Android by Marty Schoch
 * Copyright (c) 2012 Couchbase, Inc. All rights reserved.
 *
 * Modifications for this distribution by Cloudant, Inc., Copyright (c) 2013 Cloudant, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
 * except in compliance with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the
 * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied. See the License for the specific language governing permissions
 * and limitations under the License.
 */

package com.cloudant.sync.datastore;

import com.cloudant.android.Base64InputStreamFactory;
import com.cloudant.android.ContentValues;
import com.cloudant.sync.datastore.callables.GetAllDocumentIdsCallable;
import com.cloudant.sync.datastore.callables.GetPossibleAncestorRevisionIdsCallable;
import com.cloudant.sync.datastore.callables.InsertRevisionCallable;
import com.cloudant.sync.datastore.encryption.KeyProvider;
import com.cloudant.sync.datastore.encryption.NullKeyProvider;
import com.cloudant.sync.datastore.migrations.MigrateDatabase6To100;
import com.cloudant.sync.datastore.migrations.SchemaOnlyMigration;
import com.cloudant.sync.event.EventBus;
import com.cloudant.sync.notifications.DatabaseClosed;
import com.cloudant.sync.notifications.DocumentCreated;
import com.cloudant.sync.notifications.DocumentDeleted;
import com.cloudant.sync.notifications.DocumentModified;
import com.cloudant.sync.notifications.DocumentUpdated;
import com.cloudant.sync.sqlite.Cursor;
import com.cloudant.sync.sqlite.SQLDatabase;
import com.cloudant.sync.sqlite.SQLDatabaseQueue;
import com.cloudant.sync.sqlite.SQLQueueCallable;
import com.cloudant.sync.util.CouchUtils;
import com.cloudant.sync.util.DatabaseUtils;
import com.cloudant.sync.util.JSONUtils;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;

import org.apache.commons.io.FilenameUtils;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.logging.Level;
import java.util.logging.Logger;

/**
 * @api_private
 */
public class DatastoreImpl implements Datastore {

    private static final String LOG_TAG = "BasicDatastore";
    private static final Logger logger = Logger.getLogger(DatastoreImpl.class.getCanonicalName());

    private static final String FULL_DOCUMENT_COLS = "docs.docid, docs.doc_id, revid, sequence, json, current, deleted, parent";

    private static final String GET_DOC_NUMERIC_ID = "SELECT doc_id from docs WHERE docid=?";

    // get all document columns for current ("winning") revision of a given doc id
    private static final String GET_DOCUMENT_CURRENT_REVISION = "SELECT " + FULL_DOCUMENT_COLS
            + " FROM revs, docs WHERE docs.docid=? AND revs.doc_id=docs.doc_id "
            + "AND current=1 ORDER BY revid DESC LIMIT 1";

    // get sequence number for current ("winning") revision of a given doc id
    private static final String GET_SEQUENCE_CURRENT_REVISION = "SELECT revs.sequence FROM revs, docs WHERE docs.docid=? AND revs.doc_id=docs.doc_id "
            + "AND current=1 ORDER BY revid DESC LIMIT 1";

    // get all document columns for a given revision and doc id
    private static final String GET_DOCUMENT_GIVEN_REVISION = "SELECT " + FULL_DOCUMENT_COLS
            + " FROM revs, docs WHERE docs.docid=? AND revs.doc_id=docs.doc_id " + "AND revid=? LIMIT 1";

    // get sequence number for a given revision and doc id
    private static final String GET_SEQUENCE_GIVEN_REVISION = "SELECT revs.sequence FROM revs, docs WHERE docs.docid=? AND revs.doc_id=docs.doc_id "
            + "AND revid=? LIMIT 1";

    public static final String SQL_CHANGE_IDS_SINCE_LIMIT = "SELECT doc_id, max(sequence) FROM revs "
            + "WHERE sequence > ? AND sequence <= ? GROUP BY doc_id ";

    // get all non-deleted leaf rev ids for a given doc id
    public static final String GET_NON_DELETED_LEAFS = "SELECT revs.revid FROM revs " + "WHERE revs.doc_id = ? "
            + "AND revs.deleted = 0 AND revs.sequence NOT IN "
            + "(SELECT DISTINCT parent FROM revs WHERE parent NOT NULL) ";

    // get all leaf rev ids for a given doc id
    public static final String GET_ALL_LEAFS = "SELECT revs.revid FROM revs " + "WHERE revs.doc_id = ? "
            + "AND revs.sequence NOT IN " + "(SELECT DISTINCT parent FROM revs WHERE parent NOT NULL) ";

    // Limit of parameters (placeholders) one query can have.
    // SQLite has limit on the number of placeholders on a single query, default 999.
    // http://www.sqlite.org/limits.html
    public static final int SQLITE_QUERY_PLACEHOLDERS_LIMIT = 500;

    private final String datastoreName;
    private final EventBus eventBus;

    final String datastoreDir;
    final String extensionsDir;

    private static final String DB_FILE_NAME = "db.sync";

    /**
     * Stores a reference to the encryption key provider so
     * it can be passed to extensions.
     */
    private final KeyProvider keyProvider;

    /**
     * Queue for all database tasks.
     */
    private final SQLDatabaseQueue queue;

    /** Name used to get storage folder for attachments */
    private static final String ATTACHMENTS_EXTENSION_NAME = "com.cloudant.attachments";

    /** Directory where attachments are stored for this datastore */
    private final String attachmentsDir;

    /**
     * Creates streams used for encrypting and encoding (gzip etc.) attachments when
     * reading to and from disk.
     */
    private final AttachmentStreamFactory attachmentStreamFactory;

    public DatastoreImpl(String dir, String name) throws SQLException, IOException, DatastoreException {
        this(dir, name, new NullKeyProvider());
    }

    /**
     * Constructor for single thread SQLCipher-based datastore.
     * @param dir The directory where the datastore will be created
     * @param name The user-defined name of the datastore
     * @param provider The key provider object that contains the user-defined SQLCipher key
     * @throws SQLException
     * @throws IOException
     */
    public DatastoreImpl(String dir, String name, KeyProvider provider)
            throws SQLException, IOException, DatastoreException {
        Preconditions.checkNotNull(dir);
        Preconditions.checkNotNull(name);
        Preconditions.checkNotNull(provider);

        this.keyProvider = provider;
        this.datastoreDir = dir;
        this.datastoreName = name;
        this.extensionsDir = FilenameUtils.concat(this.datastoreDir, "extensions");
        final String dbFilename = FilenameUtils.concat(this.datastoreDir, DB_FILE_NAME);
        queue = new SQLDatabaseQueue(dbFilename, provider);

        int dbVersion = queue.getVersion();
        // Increment the hundreds position if a schema change means that older
        // versions of the code will not be able to read the migrated database.
        if (dbVersion >= 200) {
            throw new DatastoreException(String.format("Database version is higher than the version supported "
                    + "by this library, current version %d , highest supported version %d", dbVersion, 99));
        }
        queue.updateSchema(new SchemaOnlyMigration(DatastoreConstants.getSchemaVersion3()), 3);
        queue.updateSchema(new SchemaOnlyMigration(DatastoreConstants.getSchemaVersion4()), 4);
        queue.updateSchema(new SchemaOnlyMigration(DatastoreConstants.getSchemaVersion5()), 5);
        queue.updateSchema(new SchemaOnlyMigration(DatastoreConstants.getSchemaVersion6()), 6);
        queue.updateSchema(new MigrateDatabase6To100(), 100);
        this.eventBus = new EventBus();

        this.attachmentsDir = this.extensionDataFolder(ATTACHMENTS_EXTENSION_NAME);
        this.attachmentStreamFactory = new AttachmentStreamFactory(this.getKeyProvider());
    }

    @Override
    public String getDatastoreName() {
        Preconditions.checkState(this.isOpen(), "Database is closed");
        return this.datastoreName;
    }

    public KeyProvider getKeyProvider() {
        return this.keyProvider;
    }

    @Override
    public long getLastSequence() {
        Preconditions.checkState(this.isOpen(), "Database is closed");

        try {

            return queue.submit(new SQLQueueCallable<Long>() {
                @Override
                public Long call(SQLDatabase db) throws Exception {
                    String sql = "SELECT MAX(sequence) FROM revs";
                    Cursor cursor = null;
                    long result = 0;
                    try {
                        cursor = db.rawQuery(sql, null);
                        if (cursor.moveToFirst()) {
                            if (cursor.columnType(0) == Cursor.FIELD_TYPE_INTEGER) {
                                result = cursor.getLong(0);
                            } else if (cursor.columnType(0) == Cursor.FIELD_TYPE_NULL) {
                                result = SEQUENCE_NUMBER_START;
                            } else {
                                throw new IllegalStateException("SQLite return an unexpected value.");
                            }
                        }
                    } catch (SQLException e) {
                        logger.log(Level.SEVERE, "Error getting last sequence", e);
                        throw new DatastoreException(e);
                    } finally {
                        DatabaseUtils.closeCursorQuietly(cursor);
                    }
                    return result;
                }
            }).get();
        } catch (InterruptedException e) {
            logger.log(Level.SEVERE, "Failed to get last Sequence", e);
            throw new RuntimeException(e);
        } catch (ExecutionException e) {
            logger.log(Level.SEVERE, "Failed to get last Sequence", e);
            if (e.getCause() != null) {
                if (e.getCause() instanceof IllegalStateException) {
                    throw (IllegalStateException) e.getCause();
                }
            }
        }
        return 0;

    }

    @Override
    public int getDocumentCount() {
        Preconditions.checkState(this.isOpen(), "Database is closed");
        try {
            return queue.submit(new SQLQueueCallable<Integer>() {
                @Override
                public Integer call(SQLDatabase db) throws Exception {
                    String sql = "SELECT COUNT(DISTINCT doc_id) FROM revs WHERE current=1 AND deleted=0";
                    Cursor cursor = null;
                    int result = 0;
                    try {
                        cursor = db.rawQuery(sql, null);
                        if (cursor.moveToFirst()) {
                            result = cursor.getInt(0);
                        }
                    } catch (SQLException e) {
                        logger.log(Level.SEVERE, "Error getting document count", e);
                        throw new DatastoreException(e);
                    } finally {
                        DatabaseUtils.closeCursorQuietly(cursor);
                    }
                    return result;
                }
            }).get();
        } catch (InterruptedException e) {
            logger.log(Level.SEVERE, "Failed to get document count", e);
        } catch (ExecutionException e) {
            logger.log(Level.SEVERE, "Failed to get document count", e);
        }
        return 0;

    }

    @Override
    public boolean containsDocument(String docId, String revId) {
        Preconditions.checkState(this.isOpen(), "Database is closed");
        try {
            return getDocument(docId, revId) != null;
        } catch (DocumentNotFoundException e) {
            return false;
        }
    }

    @Override
    public boolean containsDocument(String docId) {
        Preconditions.checkState(this.isOpen(), "Database is closed");
        try {
            return getDocument(docId) != null;
        } catch (DocumentNotFoundException e) {
            return false;
        }
    }

    @Override
    public DocumentRevision getDocument(String id) throws DocumentNotFoundException {
        Preconditions.checkState(this.isOpen(), "Database is closed");
        return getDocument(id, null);
    }

    private long getSequenceInQueue(SQLDatabase db, String id, String rev) throws DatastoreException {
        Cursor cursor = null;
        try {
            String[] args = (rev == null) ? new String[] { id } : new String[] { id, rev };
            String sql = (rev == null) ? GET_SEQUENCE_CURRENT_REVISION : GET_SEQUENCE_GIVEN_REVISION;
            cursor = db.rawQuery(sql, args);
            if (cursor.moveToFirst()) {
                long sequence = cursor.getLong(0);
                return sequence;
            } else {
                return -1;
            }
        } catch (SQLException e) {
            logger.log(Level.SEVERE, "Error sequence with id: " + id + "and rev " + rev, e);
            throw new DatastoreException(
                    String.format("Could not find sequence with id %s at revision %s", id, rev), e);
        } finally {
            DatabaseUtils.closeCursorQuietly(cursor);
        }
    }

    private long getNumericIdInQueue(SQLDatabase db, String id)
            throws AttachmentException, DocumentNotFoundException, DatastoreException {
        Cursor cursor = null;
        try {
            String sql = GET_DOC_NUMERIC_ID;
            cursor = db.rawQuery(sql, new String[] { id });
            if (cursor.moveToFirst()) {
                long sequence = cursor.getLong(0);
                return sequence;
            } else {
                return -1;
            }
        } catch (SQLException e) {
            logger.log(Level.SEVERE, "Error sequence with id: " + id);
            throw new DatastoreException(String.format("Could not find sequence with id %s", id), e);
        } finally {
            DatabaseUtils.closeCursorQuietly(cursor);
        }
    }

    /**
     * Gets a document with the specified ID at the specified revision.
     * @param db The database from which to load the document
     * @param id The id of the document to be loaded
     * @param rev The revision of the document to load
     * @return The loaded document revision.
     * @throws AttachmentException If an error occurred loading the document's attachment
     * @throws DocumentNotFoundException If the document was not found.
     */
    private DocumentRevision getDocumentInQueue(SQLDatabase db, String id, String rev)
            throws AttachmentException, DocumentNotFoundException, DatastoreException {
        Cursor cursor = null;
        try {
            String[] args = (rev == null) ? new String[] { id } : new String[] { id, rev };
            String sql = (rev == null) ? GET_DOCUMENT_CURRENT_REVISION : GET_DOCUMENT_GIVEN_REVISION;
            cursor = db.rawQuery(sql, args);
            if (cursor.moveToFirst()) {
                long sequence = cursor.getLong(3);
                List<? extends Attachment> atts = AttachmentManager.attachmentsForRevision(db, this.attachmentsDir,
                        this.attachmentStreamFactory, sequence);
                return getFullRevisionFromCurrentCursor(cursor, atts);
            } else {
                throw new DocumentNotFoundException(id, rev);
            }
        } catch (SQLException e) {
            logger.log(Level.SEVERE, "Error getting document with id: " + id + "and rev " + rev, e);
            throw new DatastoreException(
                    String.format("Could not find document with id %s at revision %s", id, rev), e);
        } finally {
            DatabaseUtils.closeCursorQuietly(cursor);
        }
    }

    @Override
    public DocumentRevision getDocument(final String id, final String rev) throws DocumentNotFoundException {
        Preconditions.checkState(this.isOpen(), "Database is closed");
        Preconditions.checkArgument(!Strings.isNullOrEmpty(id), "DocumentRevisionTree id cannot " + "be empty");

        try {
            return queue.submit(new SQLQueueCallable<DocumentRevision>() {
                @Override
                public DocumentRevision call(SQLDatabase db) throws Exception {
                    return getDocumentInQueue(db, id, rev);
                }
            }).get();
        } catch (InterruptedException e) {
            logger.log(Level.SEVERE, "Failed to get document", e);
        } catch (ExecutionException e) {
            throw new DocumentNotFoundException(e);
        }
        return null;
    }

    /**
     * <p>Returns {@code DocumentRevisionTree} of a document.</p>
     *
     * <p>The tree contains the complete revision history of the document,
     * including branches for conflicts and deleted leaves.</p>
     *
     * @param docId  id of the document
     * @return {@code DocumentRevisionTree} of the specified document
     */
    public DocumentRevisionTree getAllRevisionsOfDocument(final String docId) {

        try {
            return queue.submit(new SQLQueueCallable<DocumentRevisionTree>() {
                @Override
                public DocumentRevisionTree call(SQLDatabase db) throws Exception {

                    return getAllRevisionsOfDocumentInQueue(db, docId);
                }

            }).get();
        } catch (InterruptedException e) {
            logger.log(Level.SEVERE, "Failed to get all revisions of document", e);
        } catch (ExecutionException e) {
            logger.log(Level.SEVERE, "Failed to get all revisions of document", e);
        }
        return null;
    }

    private DocumentRevisionTree getAllRevisionsOfDocumentInQueue(SQLDatabase db, String docId)
            throws DocumentNotFoundException, AttachmentException, DatastoreException {
        String sql = "SELECT " + FULL_DOCUMENT_COLS + " FROM revs, docs "
                + "WHERE docs.docid=? AND revs.doc_id = docs.doc_id ORDER BY sequence ASC";

        String[] args = { docId };
        Cursor cursor = null;

        try {
            DocumentRevisionTree tree = new DocumentRevisionTree();
            cursor = db.rawQuery(sql, args);
            while (cursor.moveToNext()) {
                long sequence = cursor.getLong(3);
                List<? extends Attachment> atts = AttachmentManager.attachmentsForRevision(db, this.attachmentsDir,
                        this.attachmentStreamFactory, sequence);
                DocumentRevision rev = getFullRevisionFromCurrentCursor(cursor, atts);
                logger.finer("Rev: " + rev);
                tree.add(rev);
            }
            return tree;
        } catch (SQLException e) {
            logger.log(Level.SEVERE, "Error getting all revisions of document", e);
            throw new DatastoreException("DocumentRevisionTree not found with id: " + docId, e);
        } finally {
            DatabaseUtils.closeCursorQuietly(cursor);
        }

    }

    @Override
    public Changes changes(long since, final int limit) {
        Preconditions.checkState(this.isOpen(), "Database is closed");
        Preconditions.checkArgument(limit > 0, "Limit must be positive number");
        final long verifiedSince = since >= 0 ? since : 0;

        try {
            return queue.submit(new SQLQueueCallable<Changes>() {
                @Override
                public Changes call(SQLDatabase db) throws Exception {
                    String[] args = { Long.toString(verifiedSince), Long.toString(verifiedSince + limit) };
                    Cursor cursor = null;
                    try {
                        Long lastSequence = verifiedSince;
                        List<Long> ids = new ArrayList<Long>();
                        cursor = db.rawQuery(SQL_CHANGE_IDS_SINCE_LIMIT, args);
                        while (cursor.moveToNext()) {
                            ids.add(cursor.getLong(0));
                            lastSequence = Math.max(lastSequence, cursor.getLong(1));
                        }
                        List<DocumentRevision> results = getDocumentsWithInternalIdsInQueue(db, ids);
                        if (results.size() != ids.size()) {
                            throw new IllegalStateException("The number of document does not match number of ids, "
                                    + "something must be wrong here.");
                        }

                        return new Changes(lastSequence, results);
                    } catch (SQLException e) {
                        throw new IllegalStateException(
                                "Error querying all changes since: " + verifiedSince + ", limit: " + limit, e);
                    } finally {
                        DatabaseUtils.closeCursorQuietly(cursor);
                    }
                }
            }).get();
        } catch (InterruptedException e) {
            logger.log(Level.SEVERE, "Failed to get changes", e);
        } catch (ExecutionException e) {
            logger.log(Level.SEVERE, "Failed to get changes", e);
            if (e.getCause() != null) {
                if (e.getCause() instanceof IllegalStateException) {
                    throw (IllegalStateException) e.getCause();
                }
            }
        }

        return null;

    }

    /**
     * Get list of documents for given list of numeric ids. The result list is ordered by sequence number,
     * and only the current revisions are returned.
     *
     * @param docIds given list of internal ids
     * @return list of documents ordered by sequence number
     */
    List<DocumentRevision> getDocumentsWithInternalIds(final List<Long> docIds) {
        Preconditions.checkNotNull(docIds, "Input document internal id list cannot be null");

        try {
            return queue.submit(new SQLQueueCallable<List<DocumentRevision>>() {
                @Override
                public List<DocumentRevision> call(SQLDatabase db) throws Exception {
                    return getDocumentsWithInternalIdsInQueue(db, docIds);
                }
            }).get();
        } catch (InterruptedException e) {
            logger.log(Level.SEVERE, "Failed to get documents using internal ids", e);
        } catch (ExecutionException e) {
            logger.log(Level.SEVERE, "Failed to get documents using internal ids", e);
        }
        return null;

    }

    private List<DocumentRevision> getDocumentsWithInternalIdsInQueue(SQLDatabase db, final List<Long> docIds)
            throws AttachmentException, DocumentNotFoundException, DocumentException, DatastoreException {

        if (docIds.size() == 0) {
            return Collections.emptyList();
        }

        final String GET_DOCUMENTS_BY_INTERNAL_IDS = "SELECT " + FULL_DOCUMENT_COLS + " FROM revs, docs "
                + "WHERE revs.doc_id IN ( %s ) AND current = 1 AND docs.doc_id = revs.doc_id";

        // Split into batches because SQLite has a limit on the number
        // of placeholders we can use in a single query. 999 is the default
        // value, but it can be lower. It's hard to find this out from Java,
        // so we use a value much lower.
        List<DocumentRevision> result = new ArrayList<DocumentRevision>(docIds.size());

        List<List<Long>> batches = Lists.partition(docIds, SQLITE_QUERY_PLACEHOLDERS_LIMIT);
        for (List<Long> batch : batches) {
            String sql = String.format(GET_DOCUMENTS_BY_INTERNAL_IDS, DatabaseUtils.makePlaceholders(batch.size()));
            String[] args = new String[batch.size()];
            for (int i = 0; i < batch.size(); i++) {
                args[i] = Long.toString(batch.get(i));
            }
            result.addAll(getRevisionsFromRawQuery(db, sql, args));
        }

        // Contract is to sort by sequence number, which we need to do
        // outside the sqlDb as we're batching requests.
        Collections.sort(result, new Comparator<DocumentRevision>() {
            @Override
            public int compare(DocumentRevision documentRevision, DocumentRevision documentRevision2) {
                long a = documentRevision.getSequence();
                long b = documentRevision2.getSequence();
                return (int) (a - b);
            }
        });

        return result;
    }

    @Override
    public List<DocumentRevision> getAllDocuments(final int offset, final int limit, final boolean descending) {
        Preconditions.checkState(this.isOpen(), "Database is closed");
        if (offset < 0) {
            throw new IllegalArgumentException("offset must be >= 0");
        }
        if (limit < 0) {
            throw new IllegalArgumentException("limit must be >= 0");
        }
        try {
            return queue.submit(new SQLQueueCallable<List<DocumentRevision>>() {
                @Override
                public List<DocumentRevision> call(SQLDatabase db) throws Exception {
                    // Generate the SELECT statement, based on the options:
                    String sql = String.format(
                            "SELECT " + FULL_DOCUMENT_COLS + " FROM revs, docs "
                                    + "WHERE deleted = 0 AND current = 1 AND docs.doc_id = revs.doc_id "
                                    + "ORDER BY docs.doc_id %1$s, revid DESC LIMIT %2$s OFFSET %3$s ",
                            (descending ? "DESC" : "ASC"), limit, offset);
                    return getRevisionsFromRawQuery(db, sql, new String[] {});
                }
            }).get();
        } catch (InterruptedException e) {
            logger.log(Level.SEVERE, "Failed to get all documents", e);
        } catch (ExecutionException e) {
            logger.log(Level.SEVERE, "Failed to get all documents", e);
        }
        return null;

    }

    @Override
    public List<String> getAllDocumentIds() {
        Preconditions.checkState(this.isOpen(), "Database is closed");
        try {
            return queue.submit(new GetAllDocumentIdsCallable()).get();
        } catch (InterruptedException e) {
            logger.log(Level.SEVERE, "Failed to get all document ids", e);
        } catch (ExecutionException e) {
            logger.log(Level.SEVERE, "Failed to get all document ids", e);
        }
        return null;
    }

    @Override
    public List<DocumentRevision> getDocumentsWithIds(final List<String> docIds) throws DocumentException {
        Preconditions.checkState(this.isOpen(), "Database is closed");
        Preconditions.checkNotNull(docIds, "Input document id list cannot be null");
        try {
            return queue.submit(new SQLQueueCallable<List<DocumentRevision>>() {
                @Override
                public List<DocumentRevision> call(SQLDatabase db) throws Exception {
                    String sql = String.format("SELECT " + FULL_DOCUMENT_COLS + " FROM revs, docs"
                            + " WHERE docid IN ( %1$s ) AND current = 1 AND docs.doc_id = revs.doc_id "
                            + " ORDER BY docs.doc_id ", DatabaseUtils.makePlaceholders(docIds.size()));
                    String[] args = docIds.toArray(new String[docIds.size()]);
                    List<DocumentRevision> docs = getRevisionsFromRawQuery(db, sql, args);
                    // Sort in memory since seems not able to sort them using SQL
                    return sortDocumentsAccordingToIdList(docIds, docs);
                }
            }).get();
        } catch (InterruptedException e) {
            logger.log(Level.SEVERE, "Failed to get documents with ids", e);
            throw new DocumentException("Failed to get documents with ids", e);
        } catch (ExecutionException e) {
            logger.log(Level.SEVERE, "Failed to get documents with ids", e);
            throw new DocumentException("Failed to get documents with ids", e);
        }
    }

    public List<String> getPossibleAncestorRevisionIDs(final String docId, final String revId, final int limit) {
        try {
            return queue.submit(new GetPossibleAncestorRevisionIdsCallable(docId, revId, limit)).get();
        } catch (InterruptedException e) {
            throw new RuntimeException(e);
        } catch (ExecutionException e) {
            throw new RuntimeException(e);
        }
    }

    private List<DocumentRevision> sortDocumentsAccordingToIdList(List<String> docIds,
            List<DocumentRevision> docs) {
        Map<String, DocumentRevision> idToDocs = putDocsIntoMap(docs);
        List<DocumentRevision> results = new ArrayList<DocumentRevision>();
        for (String id : docIds) {
            if (idToDocs.containsKey(id)) {
                results.add(idToDocs.remove(id));
            } else {
                logger.fine("No document found for id: " + id);
            }
        }
        assert idToDocs.size() == 0;
        return results;
    }

    private Map<String, DocumentRevision> putDocsIntoMap(List<DocumentRevision> docs) {
        Map<String, DocumentRevision> map = new HashMap<String, DocumentRevision>();
        for (DocumentRevision doc : docs) {
            // id should be unique cross all docs
            assert !map.containsKey(doc.getId());
            map.put(doc.getId(), doc);
        }
        return map;
    }

    /**
     * <p>Returns the current winning revision of a local document.</p>
     *
     * @param docId id of the local document
     * @return {@code LocalDocument} of the document
     * @throws DocumentNotFoundException if the document ID doesn't exist
     */
    public LocalDocument getLocalDocument(final String docId) throws DocumentNotFoundException {
        Preconditions.checkState(this.isOpen(), "Database is closed");
        try {
            return queue.submit(new SQLQueueCallable<LocalDocument>() {
                @Override
                public LocalDocument call(SQLDatabase db) throws Exception {
                    return doGetLocalDocument(db, docId);
                }
            }).get();
        } catch (InterruptedException e) {
            logger.log(Level.SEVERE, "Failed to get local document", e);
        } catch (ExecutionException e) {
            throw new DocumentNotFoundException(e);
        }
        return null;
    }

    private DocumentRevision createDocumentBody(SQLDatabase db, String docId, final DocumentBody body)
            throws AttachmentException, ConflictException, DatastoreException {
        Preconditions.checkState(this.isOpen(), "Database is closed");
        CouchUtils.validateDocumentId(docId);
        Preconditions.checkNotNull(body, "Input document body cannot be null");
        this.validateDBBody(body);

        // check if the docid exists first:

        // if it does exist:
        // * if winning leaf deleted, root the 'created' document there
        // * else raise error
        // if it does not exist:
        // * normal insert logic for a new document

        InsertRevisionCallable callable = new InsertRevisionCallable();
        DocumentRevision potentialParent = null;

        try {
            potentialParent = this.getDocumentInQueue(db, docId, null);
        } catch (DocumentNotFoundException e) {
            //this is an expected exception, it just means we are
            // resurrecting the document
        }

        if (potentialParent != null) {
            if (!potentialParent.isDeleted()) {
                // current winner not deleted, can't insert
                throw new ConflictException(
                        String.format("Cannot create doc, document with id %s already exists ", docId));
            }
            // if we got here, parent rev was deleted
            this.setCurrent(db, potentialParent, false);
            callable.revId = CouchUtils.generateNextRevisionId(potentialParent.getRevision());
            callable.docNumericId = potentialParent.getInternalNumericId();
            callable.parentSequence = potentialParent.getSequence();
        } else {
            // otherwise we are doing a normal create document
            long docNumericId = insertDocumentID(db, docId);
            callable.revId = CouchUtils.getFirstRevisionId();
            callable.docNumericId = docNumericId;
            callable.parentSequence = -1l;
        }
        callable.deleted = false;
        callable.current = true;
        callable.data = body.asBytes();
        callable.available = true;
        callable.call(db);

        try {
            DocumentRevision doc = getDocumentInQueue(db, docId, callable.revId);
            logger.finer("New document created: " + doc.toString());
            return doc;
        } catch (DocumentNotFoundException e) {
            throw new RuntimeException(String.format("Could not get document we just inserted "
                    + "(id: %s); this should not happen, please file an issue with as much detail "
                    + "as possible.", docId), e);
        }

    }

    private void validateDBBody(DocumentBody body) {
        for (String name : body.asMap().keySet()) {
            if (name.startsWith("_")) {
                throw new InvalidDocumentException("Field name start with '_' is not allowed. ");
            }
        }
    }

    /**
     * <p>Inserts a local document with an ID and body. Replacing the current local document of the
     * same id if one is present. </p>
     *
     * <p>Local documents are not replicated between datastores.</p>
     *
     * @param docId      The document id for the document
     * @param body       JSON body for the document
     * @return {@code DocumentRevision} of the newly created document
     * @throws DocumentException if there is an error inserting the local document into the database
     */
    public LocalDocument insertLocalDocument(final String docId, final DocumentBody body) throws DocumentException {
        Preconditions.checkState(this.isOpen(), "Database is closed");
        CouchUtils.validateDocumentId(docId);
        Preconditions.checkNotNull(body, "Input document body cannot be null");
        try {
            return queue.submitTransaction(new SQLQueueCallable<LocalDocument>() {
                @Override
                public LocalDocument call(SQLDatabase db) throws Exception {
                    ContentValues values = new ContentValues();
                    values.put("docid", docId);
                    values.put("json", body.asBytes());

                    long rowId = db.insertWithOnConflict("localdocs", values, SQLDatabase.CONFLICT_REPLACE);
                    if (rowId < 0) {
                        throw new DocumentException("Failed to insert local document");
                    } else {
                        logger.finer(String.format("Local doc inserted: %d , %s", rowId, docId));
                    }

                    return doGetLocalDocument(db, docId);
                }
            }).get();
        } catch (InterruptedException e) {
            logger.log(Level.SEVERE, "Failed to insert local document", e);
        } catch (ExecutionException e) {
            logger.log(Level.SEVERE, "Failed to insert local document", e);
            throw new DocumentException("Cannot insert local document", e);
        }

        return null;
    }

    private DocumentRevision updateDocumentBody(SQLDatabase db, String docId, String prevRevId,
            final DocumentBody body)
            throws ConflictException, AttachmentException, DocumentNotFoundException, DatastoreException {
        Preconditions.checkState(this.isOpen(), "Database is closed");
        Preconditions.checkArgument(!Strings.isNullOrEmpty(docId), "Input document id cannot be empty");
        Preconditions.checkArgument(!Strings.isNullOrEmpty(prevRevId),
                "Input previous revision id cannot be empty");
        Preconditions.checkNotNull(body, "Input document body cannot be null");

        this.validateDBBody(body);
        CouchUtils.validateRevisionId(prevRevId);

        DocumentRevision preRevision = this.getDocumentInQueue(db, docId, prevRevId);

        if (!preRevision.isCurrent()) {
            throw new ConflictException("Revision to be updated is not current revision.");
        }

        this.setCurrent(db, preRevision, false);
        String newRevisionId = this.insertNewWinnerRevision(db, body, preRevision);
        return this.getDocumentInQueue(db, preRevision.getId(), newRevisionId);
    }

    private DocumentRevision deleteDocumentInQueue(SQLDatabase db, final String docId, final String prevRevId)
            throws ConflictException, DocumentNotFoundException, DatastoreException {
        Preconditions.checkState(this.isOpen(), "Database is closed");
        Preconditions.checkArgument(!Strings.isNullOrEmpty(docId), "Input document id cannot be empty");
        Preconditions.checkArgument(!Strings.isNullOrEmpty(prevRevId),
                "Input previous revision id cannot be empty");

        CouchUtils.validateRevisionId(prevRevId);

        DocumentRevision prevRevision;
        try {
            prevRevision = getDocumentInQueue(db, docId, prevRevId);
        } catch (AttachmentException e) {
            throw new DocumentNotFoundException(e);
        }

        DocumentRevisionTree revisionTree;
        try {
            revisionTree = getAllRevisionsOfDocumentInQueue(db, docId);
        } catch (AttachmentException e) {
            // we can't get load the document, due to an attachment error,
            // throw an exception saying we couldn't find the document.
            throw new DocumentNotFoundException(e);
        }

        if (!revisionTree.leafRevisionIds().contains(prevRevId)) {
            throw new ConflictException("Document has newer revisions than the revision "
                    + "passed to delete; get the newest revision of the document and try again.");
        }

        if (prevRevision.isDeleted()) {
            throw new DocumentNotFoundException("Previous Revision is already deleted");
        }
        setCurrent(db, prevRevision, false);
        String newRevisionId = CouchUtils.generateNextRevisionId(prevRevision.getRevision());
        // Previous revision to be deleted could be winner revision ("current" == true),
        // or a non-winner leaf revision ("current" == false), the new inserted
        // revision must have the same flag as it previous revision.
        // Deletion of non-winner leaf revision is mainly used when resolving
        // conflicts.
        InsertRevisionCallable callable = new InsertRevisionCallable();
        callable.docNumericId = prevRevision.getInternalNumericId();
        callable.revId = newRevisionId;
        callable.parentSequence = prevRevision.getSequence();
        callable.deleted = true;
        callable.current = prevRevision.isCurrent();
        callable.data = JSONUtils.emptyJSONObjectAsBytes();
        callable.available = false;
        callable.call(db);

        try {
            //get the deleted document revision to return to the user
            return getDocumentInQueue(db, prevRevision.getId(), newRevisionId);
        } catch (AttachmentException e) {
            //throw document not found since we failed to load the document
            throw new DocumentNotFoundException(e);
        }
    }

    /**
     * <p>Deletes a local document.</p>
     *
     * @param docId documentId of the document to be deleted
     *
     * @throws DocumentNotFoundException if the document ID doesn't exist
     */
    public void deleteLocalDocument(final String docId) throws DocumentNotFoundException {
        Preconditions.checkState(this.isOpen(), "Database is closed");
        Preconditions.checkArgument(!Strings.isNullOrEmpty(docId), "Input document id cannot be empty");

        try {
            queue.submit(new SQLQueueCallable<Object>() {
                @Override
                public Object call(SQLDatabase db) throws Exception {
                    String[] whereArgs = { docId };
                    int rowsDeleted = db.delete("localdocs", "docid=? ", whereArgs);
                    if (rowsDeleted == 0) {
                        throw new DocumentNotFoundException(docId, (String) null);
                    }
                    return null;
                }
            }).get();
        } catch (InterruptedException e) {
            throw new RuntimeException(e);
        } catch (ExecutionException e) {
            throw new DocumentNotFoundException(docId, null, e);
        }

    }

    private long insertDocumentID(SQLDatabase db, String docId) {
        ContentValues args = new ContentValues();
        args.put("docid", docId);
        return db.insert("docs", args);
    }

    private long insertStubRevision(SQLDatabase db, long docNumericId, String revId, long parentSequence)
            throws AttachmentException {
        // don't copy attachments
        InsertRevisionCallable callable = new InsertRevisionCallable();
        callable.docNumericId = docNumericId;
        callable.revId = revId;
        callable.parentSequence = parentSequence;
        callable.deleted = false;
        callable.current = false;
        callable.data = JSONUtils.emptyJSONObjectAsBytes();
        callable.available = false;
        return callable.call(db);
    }

    /**
     * <p>Returns the current winning revision of a local document.</p>
     *
     * @param documentId id of the local document
     * @return {@code LocalDocument} of the document
     * @throws DocumentNotFoundException if the document ID doesn't exist
     */
    private LocalDocument doGetLocalDocument(SQLDatabase db, String docId)
            throws DocumentNotFoundException, DocumentException, DatastoreException {
        assert !Strings.isNullOrEmpty(docId);
        Cursor cursor = null;
        try {
            String[] args = { docId };
            cursor = db.rawQuery("SELECT json FROM localdocs WHERE docid=?", args);
            if (cursor.moveToFirst()) {
                byte[] json = cursor.getBlob(0);

                return new LocalDocument(docId, DocumentBodyImpl.bodyWith(json));
            } else {
                throw new DocumentNotFoundException(String.format("No local document found with id: %s", docId));
            }
        } catch (SQLException e) {
            logger.log(Level.SEVERE, String.format("Error getting local document with id: %s", docId), e);
            throw new DatastoreException("Error getting local document with id: " + docId, e);
        } finally {
            DatabaseUtils.closeCursorQuietly(cursor);
        }
    }

    private List<DocumentRevision> getRevisionsFromRawQuery(SQLDatabase db, String sql, String[] args)
            throws DocumentNotFoundException, AttachmentException, DocumentException, DatastoreException {
        List<DocumentRevision> result = new ArrayList<DocumentRevision>();
        Cursor cursor = null;
        try {
            cursor = db.rawQuery(sql, args);
            while (cursor.moveToNext()) {
                long sequence = cursor.getLong(3);
                List<? extends Attachment> atts = AttachmentManager.attachmentsForRevision(db, this.attachmentsDir,
                        this.attachmentStreamFactory, sequence);
                DocumentRevision row = getFullRevisionFromCurrentCursor(cursor, atts);
                result.add(row);
            }
        } catch (SQLException e) {
            throw new DatastoreException(e);
        } finally {
            DatabaseUtils.closeCursorQuietly(cursor);
        }
        return result;
    }

    /**
     * <p>Returns the datastore's unique identifier.</p>
     *
     * <p>This is used for the checkpoint document in a remote datastore
     * during replication.</p>
     *
     * @return a unique identifier for the datastore.
     * @throws DatastoreException if there was an error retrieving the unique identifier from the
     * database
     */
    public String getPublicIdentifier() throws DatastoreException {
        Preconditions.checkState(this.isOpen(), "Database is closed");
        try {
            return queue.submit(new SQLQueueCallable<String>() {
                @Override
                public String call(SQLDatabase db) throws Exception {
                    Cursor cursor = null;
                    try {
                        cursor = db.rawQuery("SELECT value FROM info WHERE key='publicUUID'", null);
                        if (cursor.moveToFirst()) {
                            return "touchdb_" + cursor.getString(0);
                        } else {
                            throw new IllegalStateException("Error querying PublicUUID, "
                                    + "it is probably because the sqlDatabase is not probably initialized.");
                        }
                    } finally {
                        DatabaseUtils.closeCursorQuietly(cursor);
                    }
                }
            }).get();
        } catch (InterruptedException e) {
            logger.log(Level.SEVERE, "Failed to get public ID", e);
            throw new RuntimeException(e);
        } catch (ExecutionException e) {
            logger.log(Level.SEVERE, "Failed to get public ID", e);
            throw new DatastoreException("Failed to get public ID", e);
        }
    }

    /**
     * This method has been deprecated and should not be used.
     * @see #forceInsert(List)
     */
    @Deprecated
    public void forceInsert(final DocumentRevision rev, final List<String> revisionHistory,
            final Map<String, Object> attachments,
            final Map<String[], List<PreparedAttachment>> preparedAttachments, final boolean pullAttachmentsInline)
            throws DocumentException {
        forceInsert(Collections.singletonList(new ForceInsertItem(rev, revisionHistory, attachments,
                preparedAttachments, pullAttachmentsInline)));
    }

    /**
     * <p>
     * Inserts one or more revisions of a document into the database. For efficiency, this is
     * performed as one database transaction.
     * </p>
     * <p>
     * Each revision is inserted at a point in the tree expressed by the path described in the
     * {@code revisionHistory} field. If any non-leaf revisions do not exist locally, then they are
     * created as "stub" revisions.
     * </p>
     * <p>
     * This method should only be called by the replicator. It is designed
     * to allow revisions from remote databases to be added to this
     * database during the replication process: the documents in the remote database already have
     * revision IDs that need to be preserved for the two databases to be in sync (otherwise it
     * would not be possible to tell that the two represent the same revision). This is analogous to
     * using the _new_edits false option in CouchDB
     * (see <a href="https://wiki.apache.org/couchdb/HTTP_Bulk_Document_API#Posting_Existing_Revisions">
     * the CouchDB wiki</a> for more detail).
     * <p>
     * If the document was successfully inserted, a
     * {@link com.cloudant.sync.notifications.DocumentCreated DocumentCreated},
     * {@link com.cloudant.sync.notifications.DocumentModified DocumentModified}, or
     * {@link com.cloudant.sync.notifications.DocumentDeleted DocumentDeleted}
     * event is posted on the event bus. The event will depend on the nature
     * of the update made.
     * </p>
     *
     *
     * @param items one or more revisions to insert. Each {@code ForceInsertItem} consists of:
     * <ul>
     * <li>
     * <b>rev</b> A {@code DocumentRevision} containing the information for a revision
     * from a remote datastore.
     * </li>
     * <li>
     * <b>revisionHistory</b> The history of the revision being inserted,
     * including the rev ID of {@code rev}. This list
     * needs to be sorted in ascending order
     * </li>
     * <li>
     * <b>attachments</b> Attachments metadata and optionally data if {@code pullAttachmentsInline} true
     * </li>
     * <li>
     * <b>preparedAttachments</b> Non-empty if {@code pullAttachmentsInline} false.
     * Attachments that have already been prepared, this is a
     * Map of String[docId,revId]  list of attachments
     * </li>
     * <li>
     * <b>pullAttachmentsInline</b> If true, use {@code attachments} metadata and data directly
     * from received JSON to add new attachments for this revision.
     * Else use {@code preparedAttachments} which were previously
     * downloaded and prepared by processOneChangesBatch in
     * BasicPullStrategy
     * </li>
     * </ul>
     *
     * @see Datastore#getEventBus()
     * @throws DocumentException if there was an error inserting the revision or its attachments
     * into the database
     */
    public void forceInsert(final List<ForceInsertItem> items) throws DocumentException {
        Preconditions.checkState(this.isOpen(), "Database is closed");

        for (ForceInsertItem item : items) {
            Preconditions.checkNotNull(item.rev, "Input document revision cannot be null");
            Preconditions.checkNotNull(item.revisionHistory, "Input revision history must not be null");
            Preconditions.checkArgument(item.revisionHistory.size() > 0,
                    "Input revision history must not be empty");

            Preconditions.checkArgument(checkCurrentRevisionIsInRevisionHistory(item.rev, item.revisionHistory),
                    "Current revision must exist in revision history.");
            Preconditions.checkArgument(checkRevisionIsInCorrectOrder(item.revisionHistory),
                    "Revision history must be in right order.");
            CouchUtils.validateDocumentId(item.rev.getId());
            CouchUtils.validateRevisionId(item.rev.getRevision());
        }

        // for raising events after completing database transaction
        final List<DocumentModified> events = new LinkedList<DocumentModified>();

        try {
            queue.submitTransaction(new SQLQueueCallable<Object>() {
                @Override
                public Object call(SQLDatabase db) throws Exception {
                    for (ForceInsertItem item : items) {

                        logger.finer("forceInsert(): " + item.rev.toString());

                        DocumentCreated documentCreated = null;
                        DocumentUpdated documentUpdated = null;

                        boolean ok = true;

                        long docNumericId = getNumericIdInQueue(db, item.rev.getId());
                        long seq = 0;

                        if (docNumericId != -1) {
                            seq = doForceInsertExistingDocumentWithHistory(db, item.rev, docNumericId,
                                    item.revisionHistory, item.attachments);
                            item.rev.initialiseSequence(seq);
                            // TODO fetch the parent doc?
                            documentUpdated = new DocumentUpdated(null, item.rev);
                        } else {
                            seq = doForceInsertNewDocumentWithHistory(db, item.rev, item.revisionHistory);
                            item.rev.initialiseSequence(seq);
                            documentCreated = new DocumentCreated(item.rev);
                        }

                        // now deal with any attachments
                        if (item.pullAttachmentsInline) {
                            if (item.attachments != null) {
                                for (String att : item.attachments.keySet()) {
                                    Map attachmentMetadata = (Map) item.attachments.get(att);
                                    Boolean stub = (Boolean) attachmentMetadata.get("stub");

                                    if (stub != null && stub) {
                                        // stubs get copied forward at the end of
                                        // insertDocumentHistoryIntoExistingTree - nothing to do here
                                        continue;
                                    }
                                    String data = (String) attachmentMetadata.get("data");
                                    String type = (String) attachmentMetadata.get("content_type");
                                    InputStream is = Base64InputStreamFactory
                                            .get(new ByteArrayInputStream(data.getBytes("UTF-8")));
                                    // inline attachments are automatically decompressed,
                                    // so we don't have to worry about that
                                    UnsavedStreamAttachment usa = new UnsavedStreamAttachment(is, att, type);
                                    try {
                                        PreparedAttachment pa = AttachmentManager.prepareAttachment(attachmentsDir,
                                                attachmentStreamFactory, usa);
                                        AttachmentManager.addAttachment(db, attachmentsDir, item.rev, pa);
                                    } catch (Exception e) {
                                        logger.log(Level.SEVERE, "There was a problem adding the " + "attachment "
                                                + usa + "to the datastore for document " + item.rev, e);
                                        throw e;
                                    }
                                }
                            }
                        } else {

                            try {
                                if (item.preparedAttachments != null) {
                                    for (String[] key : item.preparedAttachments.keySet()) {
                                        String id = key[0];
                                        String rev = key[1];
                                        try {
                                            DocumentRevision doc = getDocumentInQueue(db, id, rev);
                                            if (doc != null) {
                                                AttachmentManager.addAttachmentsToRevision(db, attachmentsDir, doc,
                                                        item.preparedAttachments.get(key));
                                            }
                                        } catch (DocumentNotFoundException e) {
                                            //safe to continue, previously getDocumentInQueue could return
                                            // null and this was deemed safe and expected behaviour
                                            // DocumentNotFoundException is thrown instead of returning
                                            // null now.
                                            continue;
                                        }
                                    }
                                }
                            } catch (Exception e) {
                                logger.log(Level.SEVERE,
                                        "There was a problem adding an " + "attachment to the datastore", e);
                                throw e;
                            }

                        }
                        if (ok) {
                            logger.log(Level.FINER, "Inserted revision: %s", item.rev);
                            if (documentCreated != null) {
                                events.add(documentCreated);
                            } else if (documentUpdated != null) {
                                events.add(documentUpdated);
                            }
                        }
                    }
                    return null;
                }
            }).get();

            // if we got here, everything got written to the database successfully
            // now raise any events we stored up
            for (DocumentModified event : events) {
                eventBus.post(event);
            }

        } catch (InterruptedException e) {
            throw new RuntimeException(e);
        } catch (ExecutionException e) {
            throw new DocumentException(e);
        }

    }

    /**
     * <p>Inserts a revision of a document with an existing revision ID</p>
     *
     * <p>Equivalent to:</p>
     *
     * <code>
     *    forceInsert(rev, Arrays.asList(revisionHistory), null, null, false);
     * </code>
     *
     * @param rev A {@code DocumentRevision} containing the information for a revision
     *            from a remote datastore.
     * @param revisionHistory The history of the revision being inserted,
     *                        including the rev ID of {@code rev}. This list
     *                        needs to be sorted in ascending order
     *
     * @see DatastoreImpl#forceInsert(DocumentRevision, java.util.List,java.util.Map, java.util.Map, boolean)
     * @throws DocumentException if there was an error inserting the revision into the database
     */
    public void forceInsert(DocumentRevision rev, String... revisionHistory) throws DocumentException {
        Preconditions.checkState(this.isOpen(), "Database is closed");
        this.forceInsert(rev, Arrays.asList(revisionHistory), null, null, false);
    }

    private boolean checkRevisionIsInCorrectOrder(List<String> revisionHistory) {
        for (int i = 0; i < revisionHistory.size() - 1; i++) {
            CouchUtils.validateRevisionId(revisionHistory.get(i));
            int l = CouchUtils.generationFromRevId(revisionHistory.get(i));
            int m = CouchUtils.generationFromRevId(revisionHistory.get(i + 1));
            if (l >= m) {
                return false;
            }
        }
        return true;
    }

    private boolean checkCurrentRevisionIsInRevisionHistory(DocumentRevision rev, List<String> revisionHistory) {
        return revisionHistory.get(revisionHistory.size() - 1).equals(rev.getRevision());
    }

    /**
     *
     * @param newRevision DocumentRevision to insert
     * @param revisions   revision history to insert, it includes all revisions (include the revision of the DocumentRevision
     *                    as well) sorted in ascending order.
     */
    private long doForceInsertExistingDocumentWithHistory(SQLDatabase db, DocumentRevision newRevision,
            long docNumericId, List<String> revisions, Map<String, Object> attachments)
            throws AttachmentException, DocumentNotFoundException, DatastoreException {
        logger.entering("BasicDatastore", "doForceInsertExistingDocumentWithHistory",
                new Object[] { newRevision, revisions, attachments });
        Preconditions.checkNotNull(newRevision, "New document revision must not be null.");
        Preconditions.checkArgument(this.getDocumentInQueue(db, newRevision.getId(), null) != null,
                "DocumentRevisionTree must exist.");
        Preconditions.checkNotNull(revisions, "Revision history should not be null.");
        Preconditions.checkArgument(revisions.size() > 0, "Revision history should have at least one revision.");

        // do we have a common ancestor?
        long ancestorSequence = getSequenceInQueue(db, newRevision.getId(), revisions.get(0));

        long sequence;

        if (ancestorSequence == -1) {
            sequence = insertDocumentHistoryToNewTree(db, newRevision, revisions, docNumericId);
        } else {
            sequence = insertDocumentHistoryIntoExistingTree(db, newRevision, revisions, docNumericId, attachments);
        }
        return sequence;
    }

    private long insertDocumentHistoryIntoExistingTree(SQLDatabase db, DocumentRevision newRevision,
            List<String> revisions, Long docNumericID, Map<String, Object> attachments)
            throws AttachmentException, DocumentNotFoundException, DatastoreException {

        // get info about previous "winning" rev
        long previousLeafSeq = getSequenceInQueue(db, newRevision.getId(), null);
        Preconditions.checkArgument(previousLeafSeq > 0, "Parent revision must exist");

        // Insert the new stub revisions, going down the tree
        // at the end of the loop, parentSeq will be the parent of our doc to insert
        long parentSeq = 0L;
        for (int i = 0; i < revisions.size() - 1; i++) {
            String revId = revisions.get(i);
            long seq = getSequenceInQueue(db, newRevision.getId(), revId);
            if (seq == -1) {
                seq = insertStubRevision(db, docNumericID, revId, parentSeq);
                this.changeDocumentToBeNotCurrent(db, parentSeq);
            }
            parentSeq = seq;
        }

        // Insert the new leaf revision
        String newLeafRev = revisions.get(revisions.size() - 1);
        logger.finer("Inserting new revision, id: " + docNumericID + ", rev: " + newLeafRev);
        this.changeDocumentToBeNotCurrent(db, parentSeq);
        // don't copy over attachments
        InsertRevisionCallable callable = new InsertRevisionCallable();
        callable.docNumericId = docNumericID;
        callable.revId = newLeafRev;
        callable.parentSequence = parentSeq;
        callable.deleted = newRevision.isDeleted();
        callable.current = false; // we'll call pickWinnerOfConflicts to set this if it needs it
        callable.data = newRevision.asBytes();
        callable.available = true;
        long newLeafSeq = callable.call(db);

        pickWinnerOfConflicts(db, docNumericID, newRevision.getId(), previousLeafSeq);

        // copy stubbed attachments forward from last real revision to this revision
        if (attachments != null) {
            for (Map.Entry<String, Object> att : attachments.entrySet()) {
                Boolean stub = ((Map<String, Boolean>) att.getValue()).get("stub");
                if (stub != null && stub.booleanValue()) {
                    try {
                        AttachmentManager.copyAttachment(db, previousLeafSeq, newLeafSeq, att.getKey());
                    } catch (SQLException sqe) {
                        logger.log(Level.SEVERE, "Error copying stubbed attachments", sqe);
                        throw new DatastoreException("Error copying stubbed attachments", sqe);
                    }
                }
            }
        }

        return newLeafSeq;
    }

    private long insertDocumentHistoryToNewTree(SQLDatabase db, DocumentRevision newRevision,
            List<String> revisions, Long docNumericID)
            throws AttachmentException, DocumentNotFoundException, DatastoreException {
        Preconditions.checkArgument(checkCurrentRevisionIsInRevisionHistory(newRevision, revisions),
                "Current revision must exist in revision history.");

        // get info about previous "winning" rev
        long previousLeafSeq = getSequenceInQueue(db, newRevision.getId(), null);

        // Adding a brand new tree
        logger.finer("Inserting a brand new tree for an existing document.");
        long parentSequence = 0L;
        for (int i = 0; i < revisions.size() - 1; i++) {
            //we copy attachments here so allow the exception to propagate
            parentSequence = insertStubRevision(db, docNumericID, revisions.get(i), parentSequence);
        }
        // don't copy attachments
        String newLeafRev = newRevision.getRevision();
        InsertRevisionCallable callable = new InsertRevisionCallable();
        callable.docNumericId = docNumericID;
        callable.revId = newLeafRev;
        callable.parentSequence = parentSequence;
        callable.deleted = newRevision.isDeleted();
        callable.current = false; // we'll call pickWinnerOfConflicts to set this if it needs it
        callable.data = newRevision.asBytes();
        callable.available = !newRevision.isDeleted();
        long newLeafSeq = callable.call(db);

        pickWinnerOfConflicts(db, docNumericID, newRevision.getId(), previousLeafSeq);
        return newLeafSeq;
    }

    private void pickWinnerOfConflicts(SQLDatabase db, long docNumericId, String docId, long previousWinnerSeq)
            throws DatastoreException {

        /*
         Pick winner and mark the appropriate revision with the 'current' flag set
         - There can only be one winner in a tree (or set of trees - if there is no common root)
           at any one time, so if there is a new winner, we only have to mark the old winner as
           no longer 'current'. This is the 'previousWinner' object
         - The new winner is determined by:
           * consider only non-deleted leafs
           * sort according to the CouchDB sorting algorithm: highest rev wins, if there is a tie
         then do a lexicographical compare of the revision id strings
           * we do a reverse sort (highest first) and pick the 1st and mark it 'current'
           * special case: if all leafs are deleted, then apply sorting and selection criteria
         above to all leafs
         */

        // first get all non-deleted leafs
        List<String> leafs = new ArrayList<String>();
        Cursor cursor = null;
        try {
            cursor = db.rawQuery(GET_NON_DELETED_LEAFS, new String[] { Long.toString(docNumericId) });
            while (cursor.moveToNext()) {
                leafs.add(cursor.getString(0));
            }
        } catch (SQLException sqe) {
            throw new DatastoreException(
                    "Exception thrown whilst trying to fetch non-deleted leaf nodes in pickWinnerOfConflicts", sqe);
        } finally {
            DatabaseUtils.closeCursorQuietly(cursor);
        }

        // this is a corner case - all leaf nodes are deleted
        // re-get with the same query but without the revs.delete clause
        if (leafs.size() == 0) {
            try {
                cursor = db.rawQuery(GET_ALL_LEAFS, new String[] { Long.toString(docNumericId) });
                while (cursor.moveToNext()) {
                    leafs.add(cursor.getString(0));
                }
            } catch (SQLException sqe) {
                throw new DatastoreException(
                        "Exception thrown whilst trying to fetch all leaf nodes in pickWinnerOfConflicts", sqe);
            } finally {
                DatabaseUtils.closeCursorQuietly(cursor);
            }
        }

        Collections.sort(leafs, new Comparator<String>() {
            @Override
            public int compare(String r1, String r2) {
                int generationCompare = CouchUtils.generationFromRevId(r1) - CouchUtils.generationFromRevId(r2);
                // note that the return statements have a unary minus since we are reverse sorting
                if (generationCompare != 0) {
                    return -generationCompare;
                } else {
                    return -CouchUtils.getRevisionIdSuffix(r1).compareTo(CouchUtils.getRevisionIdSuffix(r2));
                }
            }
        });
        // new winner will be at the top of the list
        String leaf = leafs.get(0);
        long newWinnerSeq = getSequenceInQueue(db, docId, leaf);
        if (previousWinnerSeq != newWinnerSeq) {
            this.changeDocumentToBeNotCurrent(db, previousWinnerSeq);
            this.changeDocumentToBeCurrent(db, newWinnerSeq);
        }
    }

    /**
     * @param rev        DocumentRevision to insert
     * @param revHistory revision history to insert, it includes all revisions (include the revision of the DocumentRevision
     *                   as well) sorted in ascending order.
     */
    private long doForceInsertNewDocumentWithHistory(SQLDatabase db, DocumentRevision rev, List<String> revHistory)
            throws AttachmentException {
        logger.entering("DocumentRevision", "doForceInsertNewDocumentWithHistory()",
                new Object[] { rev, revHistory });

        long docNumericID = insertDocumentID(db, rev.getId());
        long parentSequence = 0L;
        for (int i = 0; i < revHistory.size() - 1; i++) {
            // Insert stub node
            parentSequence = insertStubRevision(db, docNumericID, revHistory.get(i), parentSequence);
        }
        // Insert the leaf node (don't copy attachments)
        InsertRevisionCallable callable = new InsertRevisionCallable();
        callable.docNumericId = docNumericID;
        callable.revId = revHistory.get(revHistory.size() - 1);
        callable.parentSequence = parentSequence;
        callable.deleted = rev.isDeleted();
        callable.current = true;
        callable.data = rev.getBody().asBytes();
        callable.available = true;
        long sequence = callable.call(db);
        return sequence;
    }

    private void changeDocumentToBeCurrent(SQLDatabase db, long sequence) {
        ContentValues args = new ContentValues();
        args.put("current", 1);
        String[] whereArgs = { Long.toString(sequence) };
        db.update("revs", args, "sequence=?", whereArgs);
    }

    private void changeDocumentToBeNotCurrent(SQLDatabase db, long sequence) {
        ContentValues args = new ContentValues();
        args.put("current", 0);
        String[] whereArgs = { Long.toString(sequence) };
        db.update("revs", args, "sequence=?", whereArgs);
    }

    @Override
    public void compact() {
        try {
            queue.submit(new SQLQueueCallable<Object>() {
                @Override
                public Object call(SQLDatabase db) {
                    logger.finer("Deleting JSON of old revisions...");
                    ContentValues args = new ContentValues();
                    args.put("json", (String) null);
                    int i = db.update("revs", args, "current=0", null);

                    logger.finer("Deleting old attachments...");
                    AttachmentManager.purgeAttachments(db, attachmentsDir);
                    logger.finer("Vacuuming SQLite database...");
                    db.compactDatabase();
                    return null;
                }
            }).get();
        } catch (InterruptedException e) {
            logger.log(Level.SEVERE, "Failed to compact database", e);
        } catch (ExecutionException e) {
            logger.log(Level.SEVERE, "Failed to compact database", e);
        }

    }

    @Override
    public void close() {
        queue.shutdown();
        eventBus.post(new DatabaseClosed(datastoreName));

    }

    boolean isOpen() {
        return !queue.isShutdown();
    }

    /**
     * Returns the subset of given the document id/revisions that are not stored in the database.
     *
     * The input revisions is a map, whose key is document id, and value is a list of revisions.
     * An example input could be (in json format):
     *
     * { "03ee06461a12f3c288bb865b22000170":
     *     [
     *       "1-b2e54331db828310f3c772d6e042ac9c",
     *       "2-3a24009a9525bde9e4bfa8a99046b00d"
     *     ],
     *   "82e04f650661c9bdb88c57e044000a4b":
     *     [
     *       "3-bb39f8c740c6ffb8614c7031b46ac162"
     *     ]
     * }
     *
     * The output is in same format.
     *
     * @see <a href="http://wiki.apache.org/couchdb/HttpPostRevsDiff">HttpPostRevsDiff documentation</a>
     * @param revisions a Multimap of document id  revision id
     * @return a Map of document id  collection of revision id: the subset of given the document
     * id/revisions that are not stored in the database
     */
    public Map<String, Collection<String>> revsDiff(final Multimap<String, String> revisions) {
        Preconditions.checkState(this.isOpen(), "Database is closed");
        Preconditions.checkNotNull(revisions, "Input revisions must not be null");

        try {
            return queue.submit(new SQLQueueCallable<Map<String, Collection<String>>>() {
                @Override
                public Map<String, Collection<String>> call(SQLDatabase db) throws Exception {
                    Multimap<String, String> missingRevs = ArrayListMultimap.create();
                    // Break the potentially big multimap into small ones so for each map,
                    // a single query can be use to check if the <id, revision> pairs in sqlDb or not
                    List<Multimap<String, String>> batches = multiMapPartitions(revisions,
                            SQLITE_QUERY_PLACEHOLDERS_LIMIT);
                    for (Multimap<String, String> batch : batches) {
                        revsDiffBatch(db, batch);
                        missingRevs.putAll(batch);
                    }
                    return missingRevs.asMap();
                }
            }).get();
        } catch (InterruptedException e) {
            logger.log(Level.SEVERE, "Failed to do revsdiff", e);
        } catch (ExecutionException e) {
            logger.log(Level.SEVERE, "Failed to do revsdiff", e);
        }

        return null;
    }

    List<Multimap<String, String>> multiMapPartitions(Multimap<String, String> revisions, int size) {

        List<Multimap<String, String>> partitions = new ArrayList<Multimap<String, String>>();
        Multimap<String, String> current = HashMultimap.create();
        for (Map.Entry<String, String> e : revisions.entries()) {
            current.put(e.getKey(), e.getValue());
            // the query uses below (see revsDiffBatch())
            // `multimap.size() + multimap.keySet().size()` placeholders
            // and SQLite has limit on the number of placeholders on a single query.
            if (current.size() + current.keySet().size() >= size) {
                partitions.add(current);
                current = HashMultimap.create();
            }
        }

        if (current.size() > 0) {
            partitions.add(current);
        }

        return partitions;
    }

    /**
     * Removes revisions present in the datastore from the input map.
     *
     * @param revisions an multimap from document id to set of revisions. The
     *                  map is modified in place for performance consideration.
     */
    void revsDiffBatch(SQLDatabase db, Multimap<String, String> revisions) throws DatastoreException {

        final String sql = String.format(
                "SELECT docs.docid, revs.revid FROM docs, revs "
                        + "WHERE docs.doc_id = revs.doc_id AND docs.docid IN (%s) AND revs.revid IN (%s) "
                        + "ORDER BY docs.docid",
                DatabaseUtils.makePlaceholders(revisions.keySet().size()),
                DatabaseUtils.makePlaceholders(revisions.size()));

        String[] args = new String[revisions.keySet().size() + revisions.size()];
        String[] keys = revisions.keySet().toArray(new String[revisions.keySet().size()]);
        String[] values = revisions.values().toArray(new String[revisions.size()]);
        System.arraycopy(keys, 0, args, 0, revisions.keySet().size());
        System.arraycopy(values, 0, args, revisions.keySet().size(), revisions.size());

        Cursor cursor = null;
        try {
            cursor = db.rawQuery(sql, args);
            while (cursor.moveToNext()) {
                String docId = cursor.getString(0);
                String revId = cursor.getString(1);
                revisions.remove(docId, revId);
            }
        } catch (SQLException e) {
            throw new DatastoreException(e);
        } finally {
            DatabaseUtils.closeCursorQuietly(cursor);
        }
    }

    public String extensionDataFolder(String extensionName) {
        Preconditions.checkState(this.isOpen(), "Database is closed");
        Preconditions.checkArgument(!Strings.isNullOrEmpty(extensionName),
                "extension name cannot be null or empty");
        return FilenameUtils.concat(this.extensionsDir, extensionName);
    }

    @Override
    public Iterator<String> getConflictedDocumentIds() {

        // the "SELECT DISTINCT ..." subquery selects all the parent
        // sequence, and so the outer "SELECT ..." practically selects
        // all the leaf nodes. The "GROUP BY" and "HAVING COUNT(*) > 1"
        // make sure only those document with more than one leafs are
        // returned.
        final String sql = "SELECT docs.docid, COUNT(*) FROM docs,revs " + "WHERE revs.doc_id = docs.doc_id "
                + "AND deleted = 0 AND revs.sequence NOT IN "
                + "(SELECT DISTINCT parent FROM revs WHERE parent NOT NULL) "
                + "GROUP BY docs.docid HAVING COUNT(*) > 1";

        try {
            return queue.submit(new SQLQueueCallable<Iterator<String>>() {
                @Override
                public Iterator<String> call(SQLDatabase db) throws Exception {

                    List<String> conflicts = new ArrayList<String>();
                    Cursor cursor = null;
                    try {
                        cursor = db.rawQuery(sql, new String[] {});
                        while (cursor.moveToNext()) {
                            String docId = cursor.getString(0);
                            conflicts.add(docId);
                        }
                    } catch (SQLException e) {
                        logger.log(Level.SEVERE, "Error getting conflicted document: ", e);
                        throw new DatastoreException(e);
                    } finally {
                        DatabaseUtils.closeCursorQuietly(cursor);
                    }
                    return conflicts.iterator();
                }
            }).get();
        } catch (InterruptedException e) {
            logger.log(Level.SEVERE, "Failed to get conflicted document Ids", e);
        } catch (ExecutionException e) {
            logger.log(Level.SEVERE, "Failed to get conflicted document Ids", e);
        }
        return null;

    }

    @Override
    public void resolveConflictsForDocument(final String docId, final ConflictResolver resolver)
            throws ConflictException {

        // before starting the tx, get the 'new winner' and see if we need to prepare its attachments

        try {
            queue.submitTransaction(new SQLQueueCallable<Object>() {
                @Override
                public Object call(SQLDatabase db) throws Exception {
                    DocumentRevisionTree docTree = getAllRevisionsOfDocumentInQueue(db, docId);
                    if (!docTree.hasConflicts()) {
                        return null;
                    }
                    DocumentRevision newWinner = null;
                    try {
                        newWinner = resolver.resolve(docId, docTree.leafRevisions(true));
                    } catch (Exception e) {
                        logger.log(Level.SEVERE, "Exception when calling ConflictResolver", e);
                    }
                    if (newWinner == null) {
                        // resolve() threw an exception or returned null, exit early
                        return null;
                    }

                    String revIdKeep = newWinner.getRevision();
                    if (revIdKeep == null) {
                        throw new IllegalArgumentException("Winning revision must have a revision id");
                    }

                    for (DocumentRevision revision : docTree.leafRevisions()) {
                        if (revision.getRevision().equals(revIdKeep)) {
                            // this is the one we want to keep, set it to current
                            setCurrent(db, revision, true);
                        } else {
                            if (revision.isDeleted()) {
                                // if it is deleted, just make it non-current
                                setCurrent(db, revision, false);
                            } else {
                                // if it's not deleted, deleted and make it non-current
                                DocumentRevision deleted = deleteDocumentInQueue(db, revision.getId(),
                                        revision.getRevision());
                                setCurrent(db, deleted, false);
                            }
                        }
                    }

                    // if this is a new or modified revision: graft the new revision on
                    if (newWinner.bodyModified
                            || (newWinner.attachments != null && newWinner.attachments.hasChanged())) {

                        // We need to work out which of the attachments for the revision are ones
                        // we can copy over because they exist in the attachment store already and
                        // which are new, that we need to prepare for insertion.
                        Collection<Attachment> attachments = newWinner.getAttachments() != null
                                ? newWinner.getAttachments().values()
                                : new ArrayList<Attachment>();
                        final List<PreparedAttachment> preparedNewAttachments = AttachmentManager
                                .prepareAttachments(attachmentsDir, attachmentStreamFactory,
                                        AttachmentManager.findNewAttachments(attachments));
                        final List<SavedAttachment> existingAttachments = AttachmentManager
                                .findExistingAttachments(attachments);

                        updateDocumentFromRevision(db, newWinner, preparedNewAttachments, existingAttachments);
                    }

                    return null;
                }
            }).get();
        } catch (InterruptedException e) {
            logger.log(Level.SEVERE, "Failed to resolve conflicts", e);
        } catch (ExecutionException e) {
            logger.log(Level.SEVERE, "Failed to resolve Conflicts", e);
            if (e.getCause() != null) {
                if (e.getCause() instanceof IllegalArgumentException) {
                    throw (IllegalArgumentException) e.getCause();
                }
            }
        }

    }

    private String insertNewWinnerRevision(SQLDatabase db, DocumentBody newWinner, DocumentRevision oldWinner)
            throws AttachmentException, DatastoreException {
        String newRevisionId = CouchUtils.generateNextRevisionId(oldWinner.getRevision());

        InsertRevisionCallable callable = new InsertRevisionCallable();
        callable.docNumericId = oldWinner.getInternalNumericId();
        callable.revId = newRevisionId;
        callable.parentSequence = oldWinner.getSequence();
        callable.deleted = false;
        callable.current = true;
        callable.data = newWinner.asBytes();
        callable.available = true;
        callable.call(db);

        return newRevisionId;
    }

    private void setCurrent(SQLDatabase db, DocumentRevision winner, boolean currentValue) {
        ContentValues updateContent = new ContentValues();
        updateContent.put("current", currentValue ? 1 : 0);
        String[] whereArgs = new String[] { String.valueOf(winner.getSequence()) };
        db.update("revs", updateContent, "sequence=?", whereArgs);
    }

    private static DocumentRevision getFullRevisionFromCurrentCursor(Cursor cursor,
            List<? extends Attachment> attachments) {
        String docId = cursor.getString(cursor.getColumnIndex("docid"));
        long internalId = cursor.getLong(cursor.getColumnIndex("doc_id"));
        String revId = cursor.getString(cursor.getColumnIndex("revid"));
        long sequence = cursor.getLong(cursor.getColumnIndex("sequence"));
        byte[] json = cursor.getBlob(cursor.getColumnIndex("json"));
        boolean current = cursor.getInt(cursor.getColumnIndex("current")) > 0;
        boolean deleted = cursor.getInt(cursor.getColumnIndex("deleted")) > 0;

        long parent = -1L;
        if (cursor.columnType(cursor.getColumnIndex("parent")) == Cursor.FIELD_TYPE_INTEGER) {
            parent = cursor.getLong(cursor.getColumnIndex("parent"));
        } else if (cursor.columnType(cursor.getColumnIndex("parent")) == Cursor.FIELD_TYPE_NULL) {
        } else {
            throw new RuntimeException("Unexpected type: " + cursor.columnType(cursor.getColumnIndex("parent")));
        }

        DocumentRevisionBuilder builder = new DocumentRevisionBuilder().setDocId(docId).setRevId(revId)
                .setBody(DocumentBodyImpl.bodyWith(json)).setDeleted(deleted).setSequence(sequence)
                .setInternalId(internalId).setCurrent(current).setParent(parent).setAttachments(attachments);

        return builder.build();
    }

    /**
     * <p>
     * Read attachment stream to a temporary location and calculate sha1,
     * prior to being added to the datastore.
     * </p>
     * <p>
     * Used by replicator when receiving new/updated attachments
     *</p>
     *
     * @param att           Attachment to be prepared, providing data either from a file or a stream
     * @param length        Size in bytes of attachment as signalled by "length" metadata property
     * @param encodedLength Size in bytes of attachment, after encoding, as signalled by
     *                      "encoded_length" metadata property
     * @return A prepared attachment, ready to be added to the datastore
     * @throws AttachmentException if there was an error preparing the attachment, e.g., reading
     *                             attachment data.
     */
    public PreparedAttachment prepareAttachment(Attachment att, long length, long encodedLength)
            throws AttachmentException {
        PreparedAttachment pa = AttachmentManager.prepareAttachment(attachmentsDir, attachmentStreamFactory, att,
                length, encodedLength);
        return pa;
    }

    /**
     * <p>Returns attachment <code>attachmentName</code> for the revision.</p>
     *
     * <p>Used by replicator when pushing attachments</p>
     *
     * @param id The revision ID with which the attachment is associated
     * @param rev The document ID with which the attachment is associated
     * @param attachmentName Name of the attachment
     * @return <code>Attachment</code> or null if there is no attachment with that name.
     */
    public Attachment getAttachment(final String id, final String rev, final String attachmentName) {
        try {
            return queue.submit(new SQLQueueCallable<Attachment>() {
                @Override
                public Attachment call(SQLDatabase db) throws Exception {
                    long sequence = getSequenceInQueue(db, id, rev);
                    return AttachmentManager.getAttachment(db, attachmentsDir, attachmentStreamFactory, sequence,
                            attachmentName);
                }
            }).get();
        } catch (InterruptedException e) {
            logger.log(Level.SEVERE, "Failed to get attachment", e);
        } catch (ExecutionException e) {
            logger.log(Level.SEVERE, "Failed to get attachment", e);
        }

        return null;
    }

    /**
     * <p>Returns all attachments for the revision.</p>
     *
     * <p>Used by replicator when pulling attachments</p>
     *
     * @param rev The revision with which the attachments are associated
     * @return List of <code>Attachment</code>
     * @throws AttachmentException if there was an error reading the attachment metadata from the
     * database
     */
    public List<? extends Attachment> attachmentsForRevision(final DocumentRevision rev)
            throws AttachmentException {
        try {
            return queue.submit(new SQLQueueCallable<List<? extends Attachment>>() {

                @Override
                public List<? extends Attachment> call(SQLDatabase db) throws Exception {
                    return AttachmentManager.attachmentsForRevision(db, attachmentsDir, attachmentStreamFactory,
                            rev.getSequence());
                }
            }).get();
        } catch (InterruptedException e) {
            logger.log(Level.SEVERE, "Failed to get attachments for revision");
            throw new RuntimeException(e);
        } catch (ExecutionException e) {
            logger.log(Level.SEVERE, "Failed to get attachments for revision");
            throw new AttachmentException(e);
        }

    }

    @Override
    public EventBus getEventBus() {
        Preconditions.checkState(this.isOpen(), "Database is closed");
        return eventBus;
    }

    @Override
    public DocumentRevision createDocumentFromRevision(final DocumentRevision rev) throws DocumentException {
        Preconditions.checkNotNull(rev, "DocumentRevision cannot be null");
        Preconditions.checkState(isOpen(), "Datastore is closed");
        Preconditions.checkArgument(rev.getRevision() == null,
                "Revision ID must be null for new DocumentRevisions");
        Preconditions.checkArgument(rev.isFullRevision(), "Projected revisions cannot be used to create documents");
        final String docId;
        // create docid if docid is null
        if (rev.getId() == null) {
            docId = CouchUtils.generateDocumentId();
        } else {
            docId = rev.getId();
        }

        // We need to work out which of the attachments for the revision are ones
        // we can copy over because they exist in the attachment store already and
        // which are new, that we need to prepare for insertion.
        Collection<Attachment> attachments = rev.getAttachments() != null ? rev.getAttachments().values()
                : new ArrayList<Attachment>();
        final List<PreparedAttachment> preparedNewAttachments = AttachmentManager.prepareAttachments(attachmentsDir,
                attachmentStreamFactory, AttachmentManager.findNewAttachments(attachments));
        final List<SavedAttachment> existingAttachments = AttachmentManager.findExistingAttachments(attachments);

        DocumentRevision created = null;
        try {
            created = queue.submitTransaction(new SQLQueueCallable<DocumentRevision>() {
                @Override
                public DocumentRevision call(SQLDatabase db) throws Exception {

                    // Save document with new JSON body, add new attachments and copy over existing attachments
                    DocumentRevision saved = createDocumentBody(db, docId, rev.getBody());
                    AttachmentManager.addAttachmentsToRevision(db, attachmentsDir, saved, preparedNewAttachments);
                    AttachmentManager.copyAttachmentsToRevision(db, existingAttachments, saved);

                    // now re-fetch the revision with updated attachments
                    DocumentRevision updatedWithAttachments = getDocumentInQueue(db, saved.getId(),
                            saved.getRevision());
                    return updatedWithAttachments;
                }
            }).get();
            return created;
        } catch (InterruptedException e) {
            logger.log(Level.SEVERE, "Failed to create document", e);
        } catch (ExecutionException e) {
            logger.log(Level.SEVERE, "Failed to create document", e);
            throw new DocumentException(e);
        } finally {
            if (created != null) {
                eventBus.post(new DocumentCreated(created));
            }
        }
        return null;

    }

    @Override
    public DocumentRevision updateDocumentFromRevision(final DocumentRevision rev) throws DocumentException {

        Preconditions.checkNotNull(rev, "DocumentRevision cannot be null");
        Preconditions.checkState(isOpen(), "Datastore is closed");
        Preconditions.checkArgument(rev.isFullRevision(), "Projected revisions cannot be used to create documents");

        // We need to work out which of the attachments for the revision are ones
        // we can copy over because they exist in the attachment store already and
        // which are new, that we need to prepare for insertion.
        Collection<Attachment> attachments = rev.getAttachments() != null ? rev.getAttachments().values()
                : new ArrayList<Attachment>();
        final List<PreparedAttachment> preparedNewAttachments = AttachmentManager.prepareAttachments(attachmentsDir,
                attachmentStreamFactory, AttachmentManager.findNewAttachments(attachments));
        final List<SavedAttachment> existingAttachments = AttachmentManager.findExistingAttachments(attachments);

        try {
            DocumentRevision revision = queue.submitTransaction(new SQLQueueCallable<DocumentRevision>() {
                @Override
                public DocumentRevision call(SQLDatabase db) throws Exception {
                    return updateDocumentFromRevision(db, rev, preparedNewAttachments, existingAttachments);
                }
            }).get();

            if (revision != null) {
                eventBus.post(new DocumentUpdated(getDocument(rev.getId(), rev.getRevision()), revision));
            }

            return revision;
        } catch (InterruptedException e) {
            logger.log(Level.SEVERE, "Failed to update document", e);
            throw new RuntimeException(e);
        } catch (ExecutionException e) {
            logger.log(Level.SEVERE, "Failed to updated document", e);
            throw new DocumentException(e);
        }

    }

    private DocumentRevision updateDocumentFromRevision(SQLDatabase db, DocumentRevision rev,
            List<PreparedAttachment> preparedNewAttachments, List<SavedAttachment> existingAttachments)
            throws ConflictException, AttachmentException, DocumentNotFoundException, DatastoreException {
        Preconditions.checkNotNull(rev, "DocumentRevision cannot be null");

        DocumentRevision updated = updateDocumentBody(db, rev.getId(), rev.getRevision(), rev.getBody());
        AttachmentManager.addAttachmentsToRevision(db, attachmentsDir, updated, preparedNewAttachments);

        AttachmentManager.copyAttachmentsToRevision(db, existingAttachments, updated);

        // now re-fetch the revision with updated attachments
        DocumentRevision updatedWithAttachments = this.getDocumentInQueue(db, updated.getId(),
                updated.getRevision());
        return updatedWithAttachments;
    }

    @Override
    public DocumentRevision deleteDocumentFromRevision(final DocumentRevision rev) throws ConflictException {
        Preconditions.checkNotNull(rev, "DocumentRevision cannot be null");
        Preconditions.checkState(isOpen(), "Datastore is closed");

        try {
            DocumentRevision deletedRevision = queue.submitTransaction(new SQLQueueCallable<DocumentRevision>() {
                @Override
                public DocumentRevision call(SQLDatabase db) throws Exception {
                    return deleteDocumentInQueue(db, rev.getId(), rev.getRevision());
                }
            }).get();

            if (deletedRevision != null) {
                eventBus.post(new DocumentDeleted(rev, deletedRevision));
            }

            return deletedRevision;
        } catch (InterruptedException e) {
            logger.log(Level.SEVERE, "Failed to delete document", e);
        } catch (ExecutionException e) {
            logger.log(Level.SEVERE, "Failed to delete document", e);
            if (e.getCause() != null) {
                if (e.getCause() instanceof ConflictException) {
                    throw (ConflictException) e.getCause();
                }
            }
        }

        return null;
    }

    // delete all leaf nodes
    @Override
    public List<DocumentRevision> deleteDocument(final String id) throws DocumentException {
        Preconditions.checkNotNull(id, "id cannot be null");
        // to return

        try {
            return queue.submitTransaction(new SQLQueueCallable<List<DocumentRevision>>() {

                @Override
                public List<DocumentRevision> call(SQLDatabase db) throws Exception {
                    ArrayList<DocumentRevision> deleted = new ArrayList<DocumentRevision>();
                    Cursor cursor = null;
                    // delete all in one tx
                    try {
                        // get revid for each leaf
                        final String sql = "SELECT revs.revid FROM docs,revs " + "WHERE revs.doc_id = docs.doc_id "
                                + "AND docs.docid = ? " + "AND deleted = 0 AND revs.sequence NOT IN "
                                + "(SELECT DISTINCT parent FROM revs WHERE parent NOT NULL) ";

                        cursor = db.rawQuery(sql, new String[] { id });
                        while (cursor.moveToNext()) {
                            String revId = cursor.getString(0);
                            deleted.add(deleteDocumentInQueue(db, id, revId));
                        }
                        return deleted;
                    } catch (SQLException sqe) {
                        throw new DatastoreException("SQLException in deleteDocument, not deleting revisions", sqe);
                    } finally {
                        DatabaseUtils.closeCursorQuietly(cursor);
                    }
                }
            }).get();
        } catch (InterruptedException e) {
            logger.log(Level.SEVERE, "Failed to delete document", e);
        } catch (ExecutionException e) {
            throw new DocumentException("Failed to delete document", e);
        }

        return null;
    }

    <T> Future<T> runOnDbQueue(SQLQueueCallable<T> callable) {
        return queue.submit(callable);
    }
}