ezbake.data.mongo.EzMongoHandler.java Source code

Java tutorial

Introduction

Here is the source code for ezbake.data.mongo.EzMongoHandler.java

Source

/*   Copyright (C) 2013-2014 Computer Sciences Corporation
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License. */

package ezbake.data.mongo;

import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Timer;
import com.mongodb.AggregationOutput;
import com.mongodb.BasicDBList;
import com.mongodb.BasicDBObject;
import com.mongodb.Cursor;
import com.mongodb.DB;
import com.mongodb.DBCursor;
import com.mongodb.DBObject;
import com.mongodb.Mongo;
import com.mongodb.MongoClient;
import com.mongodb.ServerAddress;
import com.mongodb.WriteResult;
import com.mongodb.util.JSON;
import ezbake.base.thrift.EzSecurityToken;
import ezbake.base.thrift.Visibility;
import ezbake.configuration.constants.EzBakePropertyConstants;
import ezbake.data.base.EzbakeBaseDataService;
import ezbake.data.base.thrift.PurgeItems;
import ezbake.data.base.thrift.PurgeOptions;
import ezbake.data.base.thrift.PurgeResult;
import ezbake.data.common.TokenUtils;
import ezbake.data.common.classification.ClassificationUtils;
import ezbake.data.mongo.conversion.MongoConverter;
import ezbake.data.mongo.driver.thrift.EzAggregationRequest;
import ezbake.data.mongo.driver.thrift.EzCreateIndexRequest;
import ezbake.data.mongo.driver.thrift.EzFindRequest;
import ezbake.data.mongo.driver.thrift.EzGetMoreRequest;
import ezbake.data.mongo.driver.thrift.EzGetMoreResponse;
import ezbake.data.mongo.driver.thrift.EzInsertRequest;
import ezbake.data.mongo.driver.thrift.EzMongoDriverException;
import ezbake.data.mongo.driver.thrift.EzMongoDriverService;
import ezbake.data.mongo.driver.thrift.EzParallelScanOptions;
import ezbake.data.mongo.driver.thrift.EzParallelScanResponse;
import ezbake.data.mongo.driver.thrift.EzRemoveRequest;
import ezbake.data.mongo.driver.thrift.EzUpdateRequest;
import ezbake.data.mongo.driver.thrift.EzWriteResult;
import ezbake.data.mongo.driver.thrift.ResultsWrapper;
import ezbake.data.mongo.helper.MongoFindHelper;
import ezbake.data.mongo.helper.MongoInsertHelper;
import ezbake.data.mongo.helper.MongoUpdateHelper;
import ezbake.data.mongo.redact.RedactHelper;
import ezbake.data.mongo.thrift.EzMongoBaseException;
import ezbake.data.mongo.thrift.MongoDistinctParams;
import ezbake.data.mongo.thrift.MongoEzbakeDocument;
import ezbake.data.mongo.thrift.MongoFindParams;
import ezbake.data.mongo.thrift.MongoUpdateParams;
import ezbake.util.AuditEvent;
import ezbake.util.AuditEventType;
import ezbakehelpers.ezconfigurationhelpers.mongo.MongoConfigurationHelper;
import ezbakehelpers.mongoutils.MongoHelper;
import org.apache.accumulo.core.security.VisibilityParseException;
import org.apache.commons.lang.StringUtils;
import org.apache.thrift.TException;
import org.apache.thrift.TProcessor;
import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.ObjectOutputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;
import java.util.Properties;
import java.util.Set;

@SuppressWarnings({ "rawtypes", "unchecked" })
public class EzMongoHandler extends EzbakeBaseDataService implements EzMongoDriverService.Iface {

    MongoDriverHandler driver;

    private final Logger appLog = LoggerFactory.getLogger(EzMongoHandler.class);
    private Mongo mongo;
    protected String appName;
    private String appId;
    protected String dbName;
    private MongoConfigurationHelper mongoConfigurationHelper;
    private MongoFindHelper mongoFindHelper;
    private MongoInsertHelper mongoInsertHelper;
    private MongoUpdateHelper mongoUpdateHelper;
    private static final int DEFAULT_MONGODB_PORT = 27017;
    protected DB db;

    public final static String MONGODB_APP_DATABASE_NAME = "mongodb.app.database.name";
    public final static String READ_OPERATION = "read";
    public final static String WRITE_OPERATION = "write";
    public final static String DISCOVER_OPERATION = "discover";
    public final static String MANAGE_OPERATION = "manage";

    // Metrics
    private static final String PURGE_TIMER_NAME = MetricRegistry.name(EzMongoHandler.class, "PURGE");
    private static final String FIND_TIMER_NAME = MetricRegistry.name(EzMongoHandler.class, "FIND");
    private static final String UPDATE_DOCUMENT_TIMER_NAME = MetricRegistry.name(EzMongoHandler.class, "UPDATE",
            "DOCUMENT");
    private static final String INSERT_TIMER_NAME = MetricRegistry.name(EzMongoHandler.class, "INSERT");
    private static final String REMOVE_TIMER_NAME = MetricRegistry.name(EzMongoHandler.class, "REMOVE");

    //purge tracking collection name
    private static final String PURGE_TRACKING_COLL_NAME = "purgetracker";

    @SuppressWarnings("resource")
    public void init() throws Exception {
        final Properties properties = getConfigurationProperties();
        mongoConfigurationHelper = new MongoConfigurationHelper(properties);
        appLog.info("ezmongo configuration: {}", properties);

        // we append the appName to the collection names to keep the apps separate within Mongo
        appName = properties.getProperty(EzBakePropertyConstants.EZBAKE_APPLICATION_NAME);
        appId = properties.getProperty(EzBakePropertyConstants.EZBAKE_SECURITY_ID);

        // see if we are connecting to MongoDB with username/password
        final String username = mongoConfigurationHelper.getMongoDBUserName();
        final String password = mongoConfigurationHelper.getMongoDBPassword();
        final String hostname = mongoConfigurationHelper.getMongoDBHostName();
        dbName = mongoConfigurationHelper.getMongoDBDatabaseName();

        appLog.debug("Mongo username/password: {}/{}", username, password);
        appLog.debug("Mongo host(s): {}", hostname);
        appLog.debug("Mongo dbName: {}", dbName);

        if (StringUtils.isNotBlank(username) && StringUtils.isNotBlank(password)) {
            MongoHelper mongoHelper = new MongoHelper(properties);
            mongo = mongoHelper.getMongo();
        } else {
            // connect to MongoDB that is not username/password protected
            mongo = new MongoClient(hostname);
        }
        db = mongo.getDB(dbName);

        mongoFindHelper = new MongoFindHelper(this);
        mongoInsertHelper = new MongoInsertHelper(appId);
        mongoUpdateHelper = new MongoUpdateHelper(this);

        //create purge tracking collection
        createPurgeTracker();

        //init metrics and audit logger
        initMetrics();
        initAuditLogger(EzMongoHandler.class);

        // TODO: add caching to Mongo
        // cacheManager = (CacheManager)context.getBean("cacheManager");
    }

    /**
     * Create Purge Tracker Collection to Track the purge activity
     * @throws EzMongoBaseException
     *
     */
    void createPurgeTracker() throws EzMongoBaseException {
        try {
            final String finalCollectionName = getCollectionName(PURGE_TRACKING_COLL_NAME);

            if (!db.collectionExists(finalCollectionName)) {
                db.createCollection(finalCollectionName, new BasicDBObject());
            }

            appLog.info("in createPurgeTracker, created collection: {}", finalCollectionName);
        } catch (final Exception e) {
            throw enrichException("createPurgeTracker", e);
        }
    }

    /**
     * Initialize our meters & timers so that we can gather statistics for EzBake
     *
     */
    private void initMetrics() {
        final MetricRegistry mr = getMetricRegistry();

        // Timers
        final Timer purgeTimer = mr.timer(EzMongoHandler.PURGE_TIMER_NAME);
        final Timer findTimer = mr.timer(EzMongoHandler.FIND_TIMER_NAME);
        final Timer updateDocTimer = mr.timer(EzMongoHandler.UPDATE_DOCUMENT_TIMER_NAME);
        final Timer insertTimer = mr.timer(EzMongoHandler.INSERT_TIMER_NAME);
        final Timer removeTimer = mr.timer(EzMongoHandler.REMOVE_TIMER_NAME);

        // Meters
        //final Meter getMeter = mr.meter(EzElasticHandler.GET_METER_NAME);
        //final Meter queryMeter = mr.meter(EzElasticHandler.QUERY_METER_NAME);
    }

    private List<ServerAddress> parseMongoServerString(String mongoServers) {
        final List<ServerAddress> servers = new ArrayList<ServerAddress>();

        for (final String server : mongoServers.split(",")) {
            final String[] tokens = server.split(":");
            final String host = tokens[0];
            int port = DEFAULT_MONGODB_PORT;
            try {
                if (tokens.length == 2) {
                    port = Integer.parseInt(tokens[1]);
                }
                servers.add(new ServerAddress(host, port));
            } catch (final NumberFormatException e) {
                appLog.error("Unable to parse port number from server string({})", server);
            } catch (final java.net.UnknownHostException e) {
                appLog.error("Unknown host exception when parsing server string({})", server);
            }
        }

        return servers;
    }

    @Override
    public boolean ping() {
        boolean result = true;
        appLog.info("ping!");

        try {
            db.getCollectionNames();
        } catch (final Exception e) {
            result = false;
        }

        return result;
    }

    @Override
    public TProcessor getThriftProcessor() {
        try {

            init();
            driver = new MongoDriverHandler(this);
            driver.init();
            return new EzMongoDriverService.Processor(this);
        } catch (final Exception e) {
            e.printStackTrace();
            return null;
        }
    }

    public MongoFindHelper getMongoFindHelper() {
        return mongoFindHelper;
    }

    public MongoInsertHelper getMongoInsertHelper() {
        return mongoInsertHelper;
    }

    public MongoUpdateHelper getMongoUpdateHelper() {
        return mongoUpdateHelper;
    }

    @Override
    public boolean authenticate_driver(EzSecurityToken ezSecurityToken) throws TException {
        return true;
    }

    protected void auditLog(EzSecurityToken userToken, AuditEventType eventType, Map<String, String> argsMap) {
        AuditEvent event = new AuditEvent(eventType, userToken);

        for (Map.Entry<String, String> entry : argsMap.entrySet()) {
            event.arg(entry.getKey(), entry.getValue());
        }
        logEvent(event);
    }

    protected String printMongoObject(Object obj) {
        return obj == null ? "null" : obj.toString();
    }

    @Override
    public ResultsWrapper aggregate_driver(String collection, EzAggregationRequest ezAggregationRequest,
            EzSecurityToken token) throws TException, EzMongoDriverException {
        return driver.aggregate_driver(collection, ezAggregationRequest, token);
    }

    protected void setResponseObjectWithCursor(ResultsWrapper rw, Cursor cursor) throws IOException {
        ByteArrayOutputStream bOut = new ByteArrayOutputStream();
        new ObjectOutputStream(bOut).writeObject(cursor);
        rw.setResponseData(bOut.toByteArray());
    }

    private boolean isNotSystemCollection(String collection) {
        return !collection.equals("$cmd") && !collection.equals("system.indexes") && !collection.equals("fs.chunks")
                && !collection.equals("fs.files") && !collection.equals("system.namespaces");
    }

    @Override
    public ResultsWrapper find_driver(String collection, EzFindRequest ezFindRequest, EzSecurityToken token)
            throws TException, EzMongoDriverException {
        return driver.find_driver(collection, ezFindRequest, token);
    }

    @Override
    public EzWriteResult insert_driver(String collection, EzInsertRequest req, EzSecurityToken token)
            throws TException, EzMongoDriverException {
        return driver.insert_driver(collection, req, token);
    }

    @Override
    public EzWriteResult update_driver(String collection, EzUpdateRequest req, EzSecurityToken token)
            throws TException, EzMongoDriverException {
        return driver.update_driver(collection, req, token);
    }

    @Override
    public ResultsWrapper drop_driver(String collection, EzSecurityToken token)
            throws TException, EzMongoDriverException {
        return driver.drop_driver(collection, token);
    }

    @Override
    public EzWriteResult createIndex_driver(String collection, EzCreateIndexRequest req, EzSecurityToken token)
            throws TException, EzMongoDriverException {
        return driver.createIndex_driver(collection, req, token);
    }

    @Override
    public EzGetMoreResponse getMore_driver(String collection, EzGetMoreRequest req, EzSecurityToken token)
            throws TException, EzMongoDriverException {
        return driver.getMore_driver(collection, req, token);
    }

    @Override
    public EzParallelScanResponse parallelScan_driver(String collection, EzParallelScanOptions options,
            EzSecurityToken token) throws EzMongoDriverException, TException {
        return driver.parallelScan_driver(collection, options, token);
    }

    @Override
    public EzWriteResult remove_driver(String collection, EzRemoveRequest req, EzSecurityToken token)
            throws EzMongoDriverException, TException {
        return driver.remove_driver(collection, req, token);
    }

    @Override
    public int getMaxBsonObjectSize_driver(EzSecurityToken token) throws EzMongoDriverException, TException {
        return driver.getMaxBsonObjectSize_driver(token);
    }

    private void addResponseException(EzGetMoreResponse rw, Exception ex) throws TException {
        ByteArrayOutputStream bOut = new ByteArrayOutputStream();
        try {
            new ObjectOutputStream(bOut).writeObject(ex);
            rw.setMongoexception(bOut.toByteArray());
        } catch (Exception e) {
            throw new TException(e);
        }
    }

    protected ByteArrayOutputStream addDBCursorResult(List<DBObject> list) throws IOException {
        ByteArrayOutputStream bOut = new ByteArrayOutputStream();
        new ObjectOutputStream(bOut).writeObject(list);
        return bOut;
    }

    /**** Below are 1.3.x Mongo Dataset methods (with the IDL structs changes) ****/

    public String getCollectionName(String collectionName) {
        return appName + "_" + collectionName;
    }

    public DB getDb() {
        return db;
    }

    @Override
    public void ensureIndex(String collectionName, String jsonKeys, String jsonOptions, EzSecurityToken security)
            throws TException, EzMongoBaseException {
        try {
            HashMap<String, String> auditParamsMap = new HashMap<>();
            auditParamsMap.put("action", "ensureIndex");
            auditParamsMap.put("collectionName", collectionName);
            auditParamsMap.put("jsonKeys", jsonKeys);
            auditParamsMap.put("jsonOptions", jsonOptions);
            auditLog(security, AuditEventType.FileObjectCreate, auditParamsMap);

            TokenUtils.validateSecurityToken(security, this.getConfigurationProperties());

            if (StringUtils.isEmpty(collectionName)) {
                throw new EzMongoBaseException("collectionName is required.");
            }

            final String finalCollectionName = getCollectionName(collectionName);

            final DBObject indexKeys = (DBObject) JSON.parse(jsonKeys);

            if (!StringUtils.isEmpty(jsonOptions)) {
                final DBObject indexOptions = (DBObject) JSON.parse(jsonOptions);
                db.getCollection(finalCollectionName).ensureIndex(indexKeys, indexOptions);
            } else {
                db.getCollection(finalCollectionName).ensureIndex(indexKeys);
            }

            appLog.info("ensured index with keys: {}, options: {}", jsonKeys, jsonOptions);

        } catch (final Exception e) {
            throw enrichException("ensureIndex", e);
        }
    }

    /**
     * Create MongoDB indexes for the given fields in some collection.
     *
     * @param collectionName The Name of the collection the document belongs to
     * @param jsonKeys The Mongo index keys
     * @param jsonOptions Optional options for the particular fields being indexed.
     * @param security A valid EzSecurity token
     * @throws TException If authentication fails
     * @throws EzMongoBaseException If some other error occurs
     */
    @Override
    public void createIndex(String collectionName, String jsonKeys, String jsonOptions, EzSecurityToken security)
            throws TException, EzMongoBaseException {
        try {
            HashMap<String, String> auditParamsMap = new HashMap<>();
            auditParamsMap.put("action", "createIndex");
            auditParamsMap.put("collectionName", collectionName);
            auditParamsMap.put("jsonKeys", jsonKeys);
            auditParamsMap.put("jsonOptions", jsonOptions);
            auditLog(security, AuditEventType.FileObjectCreate, auditParamsMap);

            TokenUtils.validateSecurityToken(security, this.getConfigurationProperties());

            if (StringUtils.isEmpty(collectionName)) {
                throw new EzMongoBaseException("collectionName is required.");
            }

            final String finalCollectionName = getCollectionName(collectionName);

            final DBObject indexKeys = (DBObject) JSON.parse(jsonKeys);

            // The 'options' object is optional
            if (StringUtils.isEmpty(jsonOptions)) {
                db.getCollection(finalCollectionName).createIndex(indexKeys);
            } else {
                final DBObject indexOptions = (DBObject) JSON.parse(jsonOptions);
                // check if "ns" and "name" fields are not in the jsonOptions - then need to put them in.
                // otherwise, mongodb throws this error:
                // "Cannot authorize inserting into system.indexes documents without a string-typed \"ns\" field."
                String ns = (String) indexOptions.get("ns");
                if (ns == null) {
                    ns = dbName + "." + finalCollectionName;

                    indexOptions.put("ns", ns);

                    appLog.info("putting index's ns as : {}", ns);
                }
                String name = (String) indexOptions.get("name");
                if (name == null) {
                    name = "";

                    final Set<String> keySet = indexKeys.keySet();
                    for (final String key : keySet) {
                        final Object keyValue = indexKeys.get(key);

                        if (name.length() > 0) {
                            name += "_";
                        }
                        name += key + "_" + keyValue.toString();
                    }

                    indexOptions.put("name", name);
                    appLog.info("putting index's name as : {}", name);
                }

                db.getCollection(finalCollectionName).createIndex(indexKeys, indexOptions);
            }
            appLog.info("created index with keys: {}, options: {}", jsonKeys, jsonOptions);

        } catch (final Exception e) {
            throw enrichException("createIndex", e);
        }
    }

    @Override
    public List<String> getIndexInfo(String collectionName, EzSecurityToken security)
            throws TException, EzMongoBaseException {
        try {
            HashMap<String, String> auditParamsMap = new HashMap<>();
            auditParamsMap.put("action", "getIndexInfo");
            auditParamsMap.put("collectionName", collectionName);
            auditLog(security, AuditEventType.FileObjectAccess, auditParamsMap);

            TokenUtils.validateSecurityToken(security, this.getConfigurationProperties());

            if (StringUtils.isEmpty(collectionName)) {
                throw new EzMongoBaseException("collectionName is required.");
            }

            final String finalCollectionName = getCollectionName(collectionName);

            final List<DBObject> indexList = db.getCollection(finalCollectionName).getIndexInfo();
            final List<String> results = new ArrayList<String>();

            for (final DBObject index : indexList) {
                results.add(JSON.serialize(index));
            }

            appLog.info("got index info results: {}", results);

            return results;
        } catch (final Exception e) {
            throw enrichException("getIndexInfo", e);
        }
    }

    @Override
    public String insert(String collectionName, MongoEzbakeDocument mongoDocument, EzSecurityToken security)
            throws TException, EzMongoBaseException {
        try {
            String document = mongoDocument.jsonDocument;
            Visibility vis = mongoDocument.getVisibility();

            HashMap<String, String> auditParamsMap = new HashMap<>();
            auditParamsMap.put("action", "insert");
            auditParamsMap.put("collectionName", collectionName);
            auditParamsMap.put("mongoDocument", mongoDocument.toString());
            auditLog(security, AuditEventType.FileObjectCreate, auditParamsMap);

            TokenUtils.validateSecurityToken(security, this.getConfigurationProperties());

            if (StringUtils.isEmpty(collectionName)) {
                throw new EzMongoBaseException("collectionName is required.");
            }

            final DBObject dbObject = MongoConverter.toDBObject(document);

            // check if the user can see the document visibility
            mongoInsertHelper.checkAbilityToInsert(security, vis, dbObject, null, false, false);

            final String finalCollectionName = getCollectionName(collectionName);

            Timer.Context context = getMetricRegistry().getTimers().get(INSERT_TIMER_NAME).time();
            try {
                db.getCollection(finalCollectionName).insert(dbObject);
            } finally {
                context.stop();
            }

            final String id = dbObject.get("_id").toString();

            appLog.info("inserted - id: {}", id);

            return id;
        } catch (final Exception e) {
            throw enrichException("insert", e);
        }
    }

    public void checkAbilityToInsert(EzSecurityToken security, Visibility vis, DBObject newObject,
            DBObject existingObject, boolean isManageOperation, boolean fromDriver)
            throws VisibilityParseException, EzMongoBaseException {

        validateRequiredFieldsForInsert(vis, newObject, fromDriver);
        validateFormalVisibilityForInsert(security, vis, newObject, fromDriver);
        validateExternalCommunityVisibilityForInsert(security, vis, newObject, fromDriver);
        if (isManageOperation) {
            validatePlatformVisibilitiesForManagingExistingDBObject(security, vis, existingObject, fromDriver);
        }
        validatePlatformVisibilitiesForInsertingNewDBObject(security, vis, newObject, fromDriver);
    }

    private void validatePlatformVisibilitiesForManagingExistingDBObject(EzSecurityToken security, Visibility vis,
            DBObject existingObject, boolean fromDriver) throws EzMongoBaseException {
        // to further check if we can "Manage",
        // get the intersection of user's Platform auths with the doc's "Manage" visibilities
        Set<Long> platformAuths = security.getAuthorizations().getPlatformObjectAuthorizations();
        Set<Long> platformViz = null;
        if (fromDriver) {
            // mongodb has this field saved as a List - convert it to Set.
            List manageVizList = (List) existingObject.get(RedactHelper.PLATFORM_OBJECT_MANAGE_VISIBILITY_FIELD);
            if (manageVizList != null) {
                platformViz = new HashSet<Long>(manageVizList);
            }
        } else {
            if (vis.isSetAdvancedMarkings() && vis.getAdvancedMarkings().isSetPlatformObjectVisibility()) {
                platformViz = vis.getAdvancedMarkings().getPlatformObjectVisibility()
                        .getPlatformObjectManageVisibility();
            }
        }

        Set<Long> origPlatformViz = null;
        if (platformAuths != null && platformViz != null) {
            origPlatformViz = new HashSet<Long>(platformViz);
            platformViz.retainAll(platformAuths);
        }

        final boolean canManageViz = platformViz == null || platformViz.size() > 0;
        if (!canManageViz) {
            // reset the platformViz to the original value for logging purposes
            existingObject.put(RedactHelper.PLATFORM_OBJECT_MANAGE_VISIBILITY_FIELD, origPlatformViz);
            final String message = "User does not have all the required Platform auths to manage the existing DBObject: "
                    + platformAuths + ", platformViz needed: " + origPlatformViz;
            appLog.error(message);
            throw new EzMongoBaseException(message);
        }
    }

    private void validatePlatformVisibilitiesForInsertingNewDBObject(EzSecurityToken security, Visibility vis,
            DBObject newObject, boolean fromDriver) throws EzMongoBaseException {
        // to further check if we can insert the new mongo doc,
        // get the intersection of user's Platform auths with the doc's "Write" visibilities
        Set<Long> platformAuths = security.getAuthorizations().getPlatformObjectAuthorizations();
        Set<Long> platformViz = null;
        if (fromDriver) {
            platformViz = (Set<Long>) newObject.get(RedactHelper.PLATFORM_OBJECT_WRITE_VISIBILITY_FIELD);
        } else {
            if (vis.isSetAdvancedMarkings() && vis.getAdvancedMarkings().isSetPlatformObjectVisibility()) {
                platformViz = vis.getAdvancedMarkings().getPlatformObjectVisibility()
                        .getPlatformObjectWriteVisibility();
            }
        }

        Set<Long> origPlatformViz = null;
        if (platformAuths != null && platformViz != null) {
            origPlatformViz = new HashSet<Long>(platformViz);
            platformViz.retainAll(platformAuths);
        }

        final boolean canInsertViz = platformViz == null || platformViz.size() > 0;
        if (!canInsertViz) {
            // reset the platformViz to the original value for logging purposes
            newObject.put(RedactHelper.PLATFORM_OBJECT_WRITE_VISIBILITY_FIELD, origPlatformViz);
            final String message = "User does not have all the required Platform auths to insert: " + platformAuths
                    + ", platformViz needed: " + origPlatformViz;
            appLog.error(message);
            throw new EzMongoBaseException(message);
        }
    }

    private void validateExternalCommunityVisibilityForInsert(EzSecurityToken security, Visibility vis,
            DBObject dbObject, boolean fromDriver) throws EzMongoBaseException, VisibilityParseException {
        if (fromDriver) {
            Object extCVFieldObj = dbObject.get(RedactHelper.EXTERNAL_COMMUNITY_VISIBILITY_FIELD);
            if (extCVFieldObj == null) {
                return;
            }
            // we already have set the _ezExtV field with the double array [[ ]] format.
            //   iterate through the inner list elements and see if the token's auths have
            //   any of them as a whole.
            boolean canInsertExtViz = false;
            List outerList = (List) extCVFieldObj;
            Set<String> tokenAuths = security.getAuthorizations().getExternalCommunityAuthorizations();
            for (Object innerListObj : outerList) {
                // check if the token auths have all of this inner list element
                List innerList = (List) innerListObj;
                Set<String> innerSet = new HashSet<String>(innerList);
                if (tokenAuths.containsAll(innerSet)) {
                    canInsertExtViz = true;
                    break;
                }
            }
            if (!canInsertExtViz) {
                final String message = "User does not have all the required External Community auths to insert: "
                        + outerList + ", auths List: " + tokenAuths;
                appLog.error(message);
                throw new EzMongoBaseException(message);
            }
        } else {
            if (!vis.isSetAdvancedMarkings()) {
                appLog.info("validateExternalCommunityVisibilityForInsert - AdvancedMarkings is not set.");
                return;
            }
            // check if the user can insert the External Community Visibility (boolean expression).
            String externalCommunityBooleanExpression = vis.getAdvancedMarkings().getExternalCommunityVisibility();
            if (!StringUtils.isEmpty(externalCommunityBooleanExpression)) {
                appLog.info("checking if the user has the required classification to insert: {}",
                        externalCommunityBooleanExpression);

                final boolean canInsertExternalCommunityViz = ClassificationUtils
                        .confirmAuthsForAccumuloClassification(security, externalCommunityBooleanExpression,
                                ClassificationUtils.USER_EXTERNAL_COMMUNITY_AUTHS);

                if (!canInsertExternalCommunityViz) {
                    final String message = "User does not have all the required External Community auths to insert: "
                            + externalCommunityBooleanExpression;
                    appLog.error(message);
                    throw new EzMongoBaseException(message);
                }
            }
        }
    }

    private void validateFormalVisibilityForInsert(EzSecurityToken security, Visibility vis, DBObject dbObject,
            boolean fromDriver) throws EzMongoBaseException, VisibilityParseException {

        if (fromDriver) {
            Object fvFieldObj = dbObject.get(RedactHelper.FORMAL_VISIBILITY_FIELD);
            if (fvFieldObj == null) {
                return;
            }
            // we already have set the _ezFV field with the double array [[ ]] format.
            //   iterate through the inner list elements and see if the token's auths have
            //   any of them as a whole.
            boolean canInsertBooleanExpression = false;
            List outerList = (List) fvFieldObj;
            Set<String> tokenAuths = security.getAuthorizations().getFormalAuthorizations();
            for (Object innerListObj : outerList) {
                // check if the token auths have all of this inner list element
                List innerList = (List) innerListObj;
                Set<String> innerSet = new HashSet<String>(innerList);
                if (tokenAuths.containsAll(innerSet)) {
                    canInsertBooleanExpression = true;
                    break;
                }
            }
            if (!canInsertBooleanExpression) {
                final String message = "User does not have all the required Formal Visibility auths to insert: "
                        + outerList + ", auths List: " + tokenAuths;
                appLog.error(message);
                throw new EzMongoBaseException(message);
            }

        } else {
            // check if the user can insert the Formal Visibility (Boolean expression).
            String classification = vis.getFormalVisibility();
            if (!StringUtils.isEmpty(classification)) {
                appLog.info("checking if the user has the required classification to insert: {}", classification);

                final boolean canInsertBooleanExpression = ClassificationUtils
                        .confirmAuthsForAccumuloClassification(security, classification,
                                ClassificationUtils.USER_FORMAL_AUTHS);

                if (!canInsertBooleanExpression) {
                    final String message = "User does not have all the required classifications to insert: "
                            + classification;
                    appLog.error(message);
                    throw new EzMongoBaseException(message);
                }
            }
        }
    }

    private void validateRequiredFieldsForInsert(Visibility vis, DBObject dbObject, boolean fromDriver)
            throws VisibilityParseException, EzMongoBaseException {
        // Check if the minimally required security fields exist:
        // at least one of: _ezFV, _ezExtV, _ezObjRV
        if (!fromDriver) {
            RedactHelper.setSecurityFieldsInDBObject(dbObject, vis, appId);
        }
        Object ezFV = dbObject.get(RedactHelper.FORMAL_VISIBILITY_FIELD);
        Object ezExtV = dbObject.get(RedactHelper.EXTERNAL_COMMUNITY_VISIBILITY_FIELD);
        Object ezObjRV = dbObject.get(RedactHelper.PLATFORM_OBJECT_READ_VISIBILITY_FIELD);
        if (ezFV == null && ezExtV == null && ezObjRV == null) {
            throw new EzMongoBaseException(
                    "At least one security field (_ezFV, _ezExtV, _ezObjRV) is required for inserts.");
        }
    }

    @Override
    public int remove(String collectionName, String jsonQuery, EzSecurityToken security)
            throws TException, EzMongoBaseException {
        try {
            HashMap<String, String> auditParamsMap = new HashMap<>();
            auditParamsMap.put("action", "remove");
            auditParamsMap.put("collectionName", collectionName);
            auditParamsMap.put("jsonQuery", jsonQuery);
            auditLog(security, AuditEventType.FileObjectDelete, auditParamsMap);

            TokenUtils.validateSecurityToken(security, this.getConfigurationProperties());

            if (StringUtils.isEmpty(collectionName)) {
                throw new EzMongoBaseException("collectionName is required.");
            }

            final String finalCollectionName = getCollectionName(collectionName);

            int removedCount = 0;

            // see if we are able to remove the data in db with user's classification in the user token
            final List<DBObject> results = mongoFindHelper.findElements(collectionName, jsonQuery, "{ _id: 1}",
                    null, 0, 0, false, security, false, WRITE_OPERATION);
            if (results.size() > 0) {

                // construct a list of Objects to use as the filter
                final List<Object> idList = new ArrayList<Object>();
                for (final DBObject result : results) {
                    appLog.info("can remove DBObject (_id): {}", result);

                    idList.add(result.get("_id"));
                }

                final DBObject inClause = new BasicDBObject("$in", idList);
                final DBObject query = new BasicDBObject("_id", inClause);

                Timer.Context context = getMetricRegistry().getTimers().get(REMOVE_TIMER_NAME).time();

                try {
                    final WriteResult writeResult = db.getCollection(finalCollectionName).remove(query);

                    appLog.info("removed - write result: {}", writeResult.toString());

                    removedCount = writeResult.getN();
                } finally {
                    context.stop();
                }
            } else {
                appLog.info("Did not find any documents to remove with the query {}", jsonQuery);
            }

            appLog.info("after remove, removedCount: {}", removedCount);

            return removedCount;
        } catch (final Exception e) {
            throw enrichException("remove", e);
        }
    }

    @Override
    public int update(String collectionName, MongoUpdateParams mongoUpdateParams, EzSecurityToken security)
            throws TException, EzMongoBaseException {
        try {
            String jsonQuery = StringUtils.isEmpty(mongoUpdateParams.jsonQuery) ? "{}"
                    : mongoUpdateParams.jsonQuery;
            String jsonDocument = mongoUpdateParams.mongoDocument.jsonDocument;
            boolean updateWithOperators = mongoUpdateParams.updateWithOperators;
            Visibility vis = mongoUpdateParams.mongoDocument.getVisibility();

            HashMap<String, String> auditParamsMap = new HashMap<>();
            auditParamsMap.put("action", "update");
            auditParamsMap.put("collectionName", collectionName);
            auditParamsMap.put("mongoUpdateParams", mongoUpdateParams.toString());
            auditLog(security, AuditEventType.FileObjectModify, auditParamsMap);

            Timer.Context context = getMetricRegistry().getTimers().get(UPDATE_DOCUMENT_TIMER_NAME).time();
            int updatedCount;
            try {
                updatedCount = mongoUpdateHelper.updateDocument(collectionName, jsonQuery, jsonDocument,
                        updateWithOperators, vis, security);

                appLog.info("after update, updatedCount: {}", updatedCount);
            } finally {
                context.stop();
            }

            return updatedCount;
        } catch (final Exception e) {
            throw enrichException("update", e);
        }
    }

    @Override
    public long getCount(String collectionName, EzSecurityToken security) throws TException, EzMongoBaseException {
        try {
            HashMap<String, String> auditParamsMap = new HashMap<>();
            auditParamsMap.put("action", "getCount");
            auditParamsMap.put("collectionName", collectionName);
            auditLog(security, AuditEventType.FileObjectAccess, auditParamsMap);

            final String jsonQuery = "{}";
            return getCountFromQuery(collectionName, jsonQuery, security);

        } catch (final Exception e) {
            throw enrichException("getCount", e);
        }
    }

    @Override
    public long getCountFromQuery(String collectionName, String jsonQuery, EzSecurityToken security)
            throws TException, EzMongoBaseException {
        try {
            HashMap<String, String> auditParamsMap = new HashMap<>();
            auditParamsMap.put("action", "getCountFromQuery");
            auditParamsMap.put("collectionName", collectionName);
            auditParamsMap.put("jsonQuery", jsonQuery);
            auditLog(security, AuditEventType.FileObjectAccess, auditParamsMap);

            TokenUtils.validateSecurityToken(security, this.getConfigurationProperties());

            if (StringUtils.isEmpty(collectionName)) {
                throw new EzMongoBaseException("collectionName is required.");
            }

            final DBObject queryCommand = (DBObject) JSON.parse(jsonQuery);

            final DBObject query = new BasicDBObject("$match", queryCommand);

            final String finalCollectionName = getCollectionName(collectionName);

            appLog.info("getCountFromQuery, finalCollectionName: {}, {}", finalCollectionName, jsonQuery);

            final DBObject[] aggregationCommandsArray = mongoFindHelper
                    .getFindAggregationCommandsArray_withcounter(0, 0, null, null, security, true, READ_OPERATION);
            final AggregationOutput aggregationOutput = db.getCollection(finalCollectionName).aggregate(query,
                    aggregationCommandsArray);

            final BasicDBList resultsList = (BasicDBList) aggregationOutput.getCommandResult().get("result");
            long count = 0;
            if (resultsList.size() > 0) {
                final BasicDBObject resultsObj = (BasicDBObject) resultsList.get(0);
                count = resultsObj.getLong("count");
            }

            return count;
        } catch (final Exception e) {
            throw enrichException("getCountFromQuery", e);
        }
    }

    @Override
    public List<String> textSearch(String collectionName, String searchText, EzSecurityToken security)
            throws TException, EzMongoBaseException {
        try {
            HashMap<String, String> auditParamsMap = new HashMap<>();
            auditParamsMap.put("action", "textSearch");
            auditParamsMap.put("collectionName", collectionName);
            auditParamsMap.put("searchText", searchText);
            auditLog(security, AuditEventType.FileObjectAccess, auditParamsMap);

            TokenUtils.validateSecurityToken(security, this.getConfigurationProperties());

            if (StringUtils.isEmpty(collectionName)) {
                throw new EzMongoBaseException("collectionName is required.");
            }

            final DBObject searchObj = new BasicDBObject("$search", searchText);
            final DBObject textObj = new BasicDBObject("$text", searchObj);
            final DBObject query = new BasicDBObject("$match", textObj);

            final String finalCollectionName = getCollectionName(collectionName);

            appLog.info("textSearch, finalCollectionName: {}, query: {}", finalCollectionName, query);

            final DBObject[] aggregationCommandsArray = mongoFindHelper.getFindAggregationCommandsArray(0, 0, null,
                    null, security, READ_OPERATION);

            final AggregationOutput aggregationOutput = db.getCollection(finalCollectionName).aggregate(query,
                    aggregationCommandsArray);

            final BasicDBList commandResults = (BasicDBList) aggregationOutput.getCommandResult().get("result");
            final List<String> results = new ArrayList<String>();

            if (commandResults != null) {
                for (final Object obj : commandResults) {
                    final DBObject dbo = (DBObject) obj;

                    results.add(JSON.serialize(dbo));
                }
            } else {
                final String message = "Text search command results were null - there probably is no text index.";
                appLog.error(message);
                throw new EzMongoBaseException(message);
            }

            appLog.info("in textSearch, results size: {}", results.size());

            return results;
        } catch (final Exception e) {
            throw enrichException("textSearch", e);
        }
    }

    @Override
    public List<String> find(String collectionName, MongoFindParams mongoFindParams, EzSecurityToken security)
            throws EzMongoBaseException, TException {

        try {
            String jsonQuery = StringUtils.isEmpty(mongoFindParams.jsonQuery) ? "{}" : mongoFindParams.jsonQuery;
            String jsonProjection = mongoFindParams.jsonProjection;
            String jsonSort = mongoFindParams.jsonSort;
            int skip = mongoFindParams.skip;
            int limit = mongoFindParams.limit;
            boolean returnPlainObjectIdString = mongoFindParams.returnPlainObjectIdString;

            HashMap<String, String> auditParamsMap = new HashMap<>();
            auditParamsMap.put("action", "find");
            auditParamsMap.put("collectionName", collectionName);
            auditParamsMap.put("mongoFindParams", printMongoObject(mongoFindParams));
            auditLog(security, AuditEventType.FileObjectAccess, auditParamsMap);

            TokenUtils.validateSecurityToken(security, this.getConfigurationProperties());

            if (StringUtils.isEmpty(collectionName)) {
                throw new EzMongoBaseException("collectionName is required.");
            }

            Timer.Context context = getMetricRegistry().getTimers().get(FIND_TIMER_NAME).time();
            List<String> results = null;
            try {
                results = mongoFindHelper.findElements(collectionName, jsonQuery, jsonProjection, jsonSort, skip,
                        limit, returnPlainObjectIdString, security, true, READ_OPERATION);
                appLog.info("in find, results size: {}", results.size());
            } finally {
                context.stop();
            }

            return results;
        } catch (final Exception e) {
            throw enrichException("find", e);
        }

    }

    @Override
    public boolean collectionExists(String collectionName, EzSecurityToken security)
            throws TException, EzMongoBaseException {
        try {
            HashMap<String, String> auditParamsMap = new HashMap<>();
            auditParamsMap.put("action", "collectionExists");
            auditParamsMap.put("collectionName", collectionName);
            auditLog(security, AuditEventType.FileObjectAccess, auditParamsMap);

            TokenUtils.validateSecurityToken(security, this.getConfigurationProperties());

            if (StringUtils.isEmpty(collectionName)) {
                throw new EzMongoBaseException("collectionName is required.");
            }

            final String finalCollectionName = getCollectionName(collectionName);
            final boolean exists = db.collectionExists(finalCollectionName);

            appLog.info("in collectionExists, collection exists: {}", exists);

            return exists;
        } catch (final Exception e) {
            throw enrichException("collectionExists", e);
        }
    }

    @Override
    public void createCollection(String collectionName, EzSecurityToken security)
            throws TException, EzMongoBaseException {
        try {
            HashMap<String, String> auditParamsMap = new HashMap<>();
            auditParamsMap.put("action", "createCollection");
            auditParamsMap.put("collectionName", collectionName);
            auditLog(security, AuditEventType.FileObjectCreate, auditParamsMap);

            TokenUtils.validateSecurityToken(security, this.getConfigurationProperties());

            if (StringUtils.isEmpty(collectionName)) {
                throw new EzMongoBaseException("collectionName is required.");
            }

            final String finalCollectionName = getCollectionName(collectionName);

            db.createCollection(finalCollectionName, new BasicDBObject());

            appLog.info("in createCollection, created collection: {}", finalCollectionName);
        } catch (final Exception e) {
            throw enrichException("createCollection", e);
        }
    }

    @Override
    public void dropCollection(String collectionName, EzSecurityToken security)
            throws TException, EzMongoBaseException {
        try {
            HashMap<String, String> auditParamsMap = new HashMap<>();
            auditParamsMap.put("action", "dropCollection");
            auditParamsMap.put("collectionName", collectionName);
            auditLog(security, AuditEventType.FileObjectDelete, auditParamsMap);

            TokenUtils.validateSecurityToken(security, this.getConfigurationProperties());

            if (StringUtils.isEmpty(collectionName)) {
                throw new EzMongoBaseException("collectionName is required.");
            }

            final String finalCollectionName = getCollectionName(collectionName);
            db.getCollection(finalCollectionName).drop();

            appLog.info("in dropCollection, dropped collection: {}", finalCollectionName);
        } catch (final Exception e) {
            throw enrichException("dropCollection", e);
        }
    }

    @Override
    public List<String> distinct(String collectionName, MongoDistinctParams mongoDistinctParams,
            EzSecurityToken securityToken) throws TException, EzMongoBaseException {
        List<String> results = new ArrayList<>();

        String field = mongoDistinctParams.getField();
        String jsonQuery = mongoDistinctParams.getQuery();

        if (StringUtils.isEmpty(jsonQuery)) {
            jsonQuery = "{ }";
        }
        try {
            // call the redact pipeline
            List<DBObject> findResults = mongoFindHelper.findElements(collectionName, jsonQuery, null, null, 0, 0,
                    false, securityToken, false, READ_OPERATION);

            if (findResults.size() > 0) {
                // construct a list of Objects to use as the filter
                final List<Object> idList = new ArrayList<Object>();
                for (final DBObject findResult : findResults) {
                    appLog.info("can find DBObject (_id): {}", findResult);

                    idList.add(findResult.get("_id"));
                }

                final DBObject inClause = new BasicDBObject("$in", idList);
                final DBObject query = new BasicDBObject("_id", inClause);
                final String finalCollectionName = getCollectionName(collectionName);

                // get the distinct results from the redacted list
                List distinctResults = db.getCollection(finalCollectionName).distinct(field, query);

                // convert to Strings
                results = mongoFindHelper.addMongoResultsToList(distinctResults, false, true);

                appLog.info("Distinct values for field {}, jsonQuery {}: {}", field, jsonQuery, results);
            }

        } catch (Exception e) {
            throw enrichException("distinct", e);
        }

        return results;
    }

    /* Currently not used - if you want to create indexes dynamically, use the ezmongo CLI.
    private void createIndexes() throws Exception {
    InputStream inputStream = null;
    Resource resource = null;
        
    try {
        resource = new ClassPathResource("indexes.txt");
        inputStream = resource.getInputStream();
        final Scanner scanner0 = new Scanner(inputStream);
        DBCollection collection = null;
        
        while (scanner0.hasNext()) {
            final String index = scanner0.nextLine().trim();
            collection = null;
        
            if (index.startsWith("#") || index.length() == 0 || "".equals(index)) {
                // skip over the comments or empty lines
                continue;
            } else {
                final int spaceIdx = index.indexOf(" ");
                final String collectionName = index.substring(0, spaceIdx).trim();
                final String indexJson = index.substring(spaceIdx).trim();
        
                if (collectionName.startsWith("{")) {
                    // either someone didn't put in a collection name with the index
                    // JSON or they started the collection name with the "{"
                    // log it and continue
                    appLog.info("Invalid collection name to create index: {}", collectionName);
                    continue;
                }
        
                final String finalCollectionName = getCollectionName(collectionName);
                if (!mongoTemplate.collectionExists(finalCollectionName)) {
                    collection = mongoTemplate.createCollection(finalCollectionName);
                } else {
                    collection = mongoTemplate.getCollection(finalCollectionName);
                }
        
                // check the index JSON
                // TODO: this works but it's probably better to use a regex
                // to retrieve the parts of the JSON that we want
                if (indexJson.indexOf("},") != -1) {
                    final String[] tokens = indexJson.split("},");
        
                    final DBObject keys = retrieveDBObject(tokens[0]);
                    final DBObject optionsIn = retrieveDBObject(tokens[1]);
        
                    collection.ensureIndex(keys, optionsIn);
                } else {
                    // only have the keys portion
                    final DBObject keys = retrieveDBObject(indexJson);;
                    collection.ensureIndex(keys);
                }
            }
        }
    } catch (final IOException ioe) {
        appLog.error("Encountered an IOException: " + ioe.getCause());
    } catch (final Exception e) {
        e.printStackTrace();
    } finally {
        if (inputStream != null) {
            try {
                inputStream.close();
            } catch (final IOException ignored) {
            }
        }
    }
    }
        
    private DBObject retrieveDBObject(String jsonString) {
    final DBObject dbo = new BasicDBObject();
    String s = jsonString.trim();
    if (s.startsWith("{")) {
        s = s.substring(1);
    }
    if (s.endsWith("}")) {
        s = s.substring(0, s.indexOf("}"));
    }
        
    final String[] tokens = s.split(",");
    for (final String token : tokens) {
        final String[] keyValue = token.split(":");
        
        if (keyValue[1].toLowerCase().trim().equalsIgnoreCase("true")
                || keyValue[1].toLowerCase().trim().equalsIgnoreCase("false")) {
            dbo.put(keyValue[0], keyValue[1]);
        } else {
            dbo.put(keyValue[0], Integer.parseInt(keyValue[1].trim()));
        }
    }
        
    return dbo;
    }
    */

    /**
     * A method to add detail to the Thrift-serialized Exception.
     *
     * Since the underlying MongoDB instance may throw some unknown instance of java.lang.Exception in our usage, we
     * catch the base type in this handler and send it's classname and message back to the client.
     *
     * @param step
     * @param e
     * @return
     */
    private EzMongoBaseException enrichException(String step, Exception e) {
        final String classname = e.getClass().getCanonicalName(),
                messageStart = "Caught exception " + classname + " in " + step + ": ";
        appLog.error(messageStart, e);
        return new EzMongoBaseException(messageStart + e.getMessage());
    }

    String getCollNameOfPurgeId(long purgeId) {
        String collectionName = null;
        Set purgeIds = new HashSet();
        purgeIds.add(purgeId);

        appLog.info("Get Collection Name of Purge Id {} in Purge Tracker Collection", purgeId);
        DBObject query = new BasicDBObject();
        query.put(RedactHelper.APP_ID_FIELD, appId);
        query.put(RedactHelper.PURGE_ID, new BasicDBObject("$in", purgeIds));
        String trackingPurgeCollName = getCollectionName(PURGE_TRACKING_COLL_NAME);
        DBCursor cursor = db.getCollection(trackingPurgeCollName).find(query);
        if (cursor.length() == 1) {
            appLog.info("Found a Record getting collection name");
            collectionName = (String) cursor.one().get(RedactHelper.PURGE_TRACKING_COLL_FIELD);
        }
        appLog.info("getCollNameOfPurgeId, returning Collection Name {}", collectionName);
        return collectionName;
    }

    void updatePurgeTracker(long purgeId, String purgeCollName, EzSecurityToken token) {
        appLog.info("updatePurgeTracker with Purge Id {} and Collection Name {}", purgeId, purgeCollName);
        String trackingPurgeCollName = getCollectionName(PURGE_TRACKING_COLL_NAME);
        DBObject query = new BasicDBObject();
        query.put(RedactHelper.APP_ID_FIELD, appId);
        query.put(RedactHelper.PURGE_ID, purgeId);
        DBCursor cursor = db.getCollection(trackingPurgeCollName).find(query);
        if (cursor.length() == 0) {
            //insert
            appLog.info("No Records Found for Purge Id {} in Purge Collection {}", purgeId, trackingPurgeCollName);

            try {
                JSONObject jobj = new JSONObject();
                jobj.put(RedactHelper.PURGE_ID, purgeId);
                jobj.put(RedactHelper.PURGE_TRACKING_COLL_FIELD, purgeCollName);
                jobj.put(RedactHelper.APP_ID_FIELD, appId);
                appLog.info("Dumping JSON before Insert : {}", jobj.toString());
                this.insert(PURGE_TRACKING_COLL_NAME,
                        new MongoEzbakeDocument(jobj.toString(), new Visibility().setFormalVisibility("S")), token);

            } catch (TException e) {
                e.printStackTrace();
            } catch (org.codehaus.jettison.json.JSONException je) {
                je.printStackTrace();
                appLog.error("Error updating the purge record!");
            }

        } else {
            //update
            appLog.info("Updating Purge Collection {} with Purge Id", PURGE_TRACKING_COLL_NAME, purgeId);
            DBObject content = new BasicDBObject();
            content.put(RedactHelper.PURGE_IDS_FIELD, purgeId);
            content.put(RedactHelper.PURGE_TRACKING_COLL_FIELD, purgeCollName);
            mongoUpdateHelper.updateContent(getCollectionName(PURGE_TRACKING_COLL_NAME), query, content, false,
                    false);
        }

    }

    @Override
    public PurgeResult purge(PurgeItems items, PurgeOptions options, EzSecurityToken token) throws TException {
        HashMap<String, String> auditParamsMap = new HashMap<>();
        auditParamsMap.put("action", "purge");
        auditParamsMap.put("purgeItems", items.toString());
        auditParamsMap.put("purgeOptions", printMongoObject(options));
        auditLog(token, AuditEventType.FileObjectDelete, auditParamsMap);

        TokenUtils.validateSecurityToken(token, getConfigurationProperties());
        int batchSize = options == null ? 0 : options.getBatchSize();
        final Timer.Context context = getMetricRegistry().getTimers().get(PURGE_TIMER_NAME).time();
        try {
            return purge(items.getPurgeId(), items.getItems(), batchSize, token);
        } finally {
            context.stop();
        }
    }

    private PurgeResult purge(long id, Set<Long> toPurge, int batchSize, EzSecurityToken token) {
        appLog.info("Purging ID {} with batchSize {} with items:\n {}", id, batchSize, toPurge);

        final PurgeResult result = new PurgeResult(false);
        final Set<Long> purged = new HashSet<>();
        final Set<Long> unpurged = new HashSet<>();
        Set<String> collectionNames = db.getCollectionNames();
        //sort the collection names
        List<String> collNamesSortedList = MongoConverter.asSortedList(collectionNames);
        //get the collection name at which to start the purge
        String purgeStartCollName = getCollNameOfPurgeId(id);
        appLog.info(
                "Collection Name of Collection in Purge Tracker: " + purgeStartCollName + " For Purge Id: " + id);

        ListIterator<String> collNamesIterator = null;
        if (purgeStartCollName != null) {
            collNamesIterator = collNamesSortedList.listIterator(collNamesSortedList.indexOf(purgeStartCollName));
        } else {
            collNamesIterator = collNamesSortedList.listIterator();
        }

        //batch size tracker
        int batchTracker = 0;
        while ((collNamesIterator.hasNext()) && (batchTracker < batchSize)) {
            String collectionName = collNamesIterator.next();
            if (!isNotSystemCollection(collectionName)) {
                continue;
            }

            DBObject query = new BasicDBObject();
            query.put(RedactHelper.APP_ID_FIELD, appId);
            query.put(RedactHelper.ID_FIELD, new BasicDBObject("$in", toPurge));
            DBCursor cursor = db.getCollection(collectionName).find(query);

            while ((cursor.hasNext()) && (batchTracker < batchSize)) {
                DBObject dbObject = cursor.next();

                long purgeId = (Long) dbObject.get(RedactHelper.ID_FIELD);
                Object composite = dbObject.get(RedactHelper.COMPOSITE_FIELD);

                if (composite != null && (Boolean) composite) {
                    appLog.info("Composite item cannot be purged: _id {} ", dbObject.get("_id"));
                    unpurged.add(purgeId);
                } else {
                    appLog.info("Purging item _id {} and Purge Id {}", dbObject.get("_id"), purgeId);
                    purged.add(purgeId);
                    db.getCollection(collectionName).remove(dbObject);
                }

                batchTracker++;
            }

            if (cursor.hasNext()) {
                result.setIsFinished(false);
            } else {
                result.setIsFinished(true);
            }
            //update purge tracker collection name with purge id and collection name where purge stopped
            updatePurgeTracker(id, collectionName, token);
        }

        result.setPurged(purged);
        result.setUnpurged(unpurged);

        appLog.info("Size of Purged Items {}", result.getPurged().size());
        return result;
    }
}