ezbake.services.centralPurge.thrift.EzCentralPurgeServiceHandler.java Source code

Java tutorial

Introduction

Here is the source code for ezbake.services.centralPurge.thrift.EzCentralPurgeServiceHandler.java

Source

/*   Copyright (C) 2013-2015 Computer Sciences Corporation
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License. */

package ezbake.services.centralPurge.thrift;

import com.google.common.collect.Multimap;
import com.google.common.collect.Sets;
import com.mongodb.*;
import ezbake.base.thrift.*;
import ezbake.configuration.constants.EzBakePropertyConstants;
import ezbake.ezpurge.ServicePurgeClient;
import ezbake.security.client.EzSecurityTokenWrapper;
import ezbake.security.client.EzbakeSecurityClient;
import ezbake.services.centralPurge.helpers.DelayedServicePurgeState;
import ezbake.services.centralPurge.helpers.EzCentralPurgeServiceHelpers;
import ezbake.services.provenance.thrift.*;
import ezbake.thrift.ThriftClientPool;
import ezbake.util.AuditEvent;
import ezbake.util.AuditEventType;
import ezbake.util.AuditLogger;
import ezbakehelpers.ezconfigurationhelpers.mongo.MongoConfigurationHelper;
import ezbakehelpers.mongoutils.MongoHelper;
import org.apache.thrift.TException;
import org.apache.thrift.TProcessor;
import org.apache.thrift.TServiceClient;
import org.slf4j.Logger;

import java.net.UnknownHostException;
import java.util.*;
import java.util.concurrent.*;

import static ezbake.common.time.DateUtils.getCurrentDateTime;
import static ezbake.services.centralPurge.helpers.EzCentralPurgeServiceHelpers.*;
import static ezbake.util.AuditEvent.event;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.slf4j.LoggerFactory.getLogger;

//import ezbake.common.time.DateUtils;

/**
 * Created by jpercivall on 7/2/14.
 */

public class EzCentralPurgeServiceHandler extends ezbake.base.thrift.EzBakeBaseThriftService
        implements EzCentralPurgeService.Iface {

    private final Logger logger = getLogger(EzCentralPurgeServiceHandler.class);

    private static final String COMMON_APP_NAME = "common_services",
            PROVENANCE_SERVICE_NAME = ProvenanceServiceConstants.SERVICE_NAME,
            EZBAKE_BASE_PURGE_SERVICE_NAME = ezCentralPurgeServiceConstants.SERVICE_NAME,
            PURGE_COLLECTION = "purgeCollection", AGEOFF_COLLECTION = "ageOffCollection";

    //private final String COMMON_APP_NAME = "common_services";
    //private final String EZBAKE_BASE_PURGE_SERVICE_NAME = ezCentralPurgeServiceConstants.SERVICE_NAME;
    //private final String PROVENANCE_SERVICE_NAME = ProvenanceServiceConstants.SERVICE_NAME;
    private String purgeAppSecurityId;
    private EzbakeSecurityClient securityClient;
    private Properties configuration;
    private static final AuditLogger auditLogger = new AuditLogger(EzCentralPurgeServiceHandler.class);
    private boolean initialized = false;
    private DelayQueue<DelayedServicePurgeState> servicePollDelayQueue;

    // This class runs a purge for every age off rule with out of date documents
    private class AutomaticAgeOff implements Runnable {
        public void run() {
            try {
                ProvenanceService.Client client = null;
                EzSecurityToken centralTokenForProvenance = null;

                // Get app token and set up audit event
                try {
                    centralTokenForProvenance = securityClient.fetchAppToken();
                } catch (Exception e) {
                    e.printStackTrace();
                }
                AuditEvent evt = event(AuditEventType.FileObjectAccess.getName(), centralTokenForProvenance)
                        .arg("event", "automatic ageOff");
                List<Long> ageOffIds = new LinkedList<>();
                ThriftClientPool pool = null;
                try {
                    pool = new ThriftClientPool(configuration);
                    // Get every ageOffRule, then start an ageOffEvent for each.
                    centralTokenForProvenance = securityClient.fetchAppToken(getProvenanceSecurityId(pool));
                    client = getProvenanceThriftClient(pool);
                    List<AgeOffRule> ageOffRules = client.getAllAgeOffRules(centralTokenForProvenance, 0, 0);
                    for (AgeOffRule ageOffRule : ageOffRules) {
                        try {
                            executeAgeOff(centralTokenForProvenance, ageOffRule.getId(), true);
                            ageOffIds.add(ageOffRule.getId());
                        } catch (Exception e) {
                            logError(e, evt,
                                    "Automatic ageOff failed [" + e.getClass().getName() + ":" + e.getMessage()
                                            + "] for this rule:" + ageOffRule.getName() + "(id:"
                                            + ageOffRule.getId() + ")");
                        }
                    }
                } catch (Exception e) {
                    logError(e, evt,
                            "Automatic ageOff failed [" + e.getClass().getName() + ":" + e.getMessage() + "]");
                } finally {
                    evt.arg("ageOffIds", ageOffIds);
                    auditLogger.logEvent(evt);
                    logEventToPlainLogs(logger, evt);
                    if (client != null)
                        returnClientToPool(client, pool);
                    if (pool != null)
                        pool.close();
                }
            } catch (Exception e) {
                e.printStackTrace();
                logger.error("Automatic ageOff failed [" + e.getClass().getName() + ":" + e.getMessage() + "]");
                auditLogger.log("Automatic ageOff failed [" + e.getClass().getName() + ":" + e.getMessage() + "]");
            }
        }
    }

    // TODO: determine where logger/stdOut go in ezcentos
    // This class polls each purge for a status update
    private class AutomaticUpdate implements Runnable {
        @Override
        public void run() {
            try {
                logger.debug("Starting AutomaticUpdate");
                DelayedServicePurgeState delayedServicePurgeState;
                EzSecurityToken centralPurgeServiceToken = null;
                try {
                    centralPurgeServiceToken = securityClient.fetchAppToken();
                } catch (TException e) {
                    e.printStackTrace();
                }
                List<String> servicesUpdated = new LinkedList<>();
                AuditEvent evt = event(AuditEventType.FileObjectModify.getName(), centralPurgeServiceToken)
                        .arg("event", "automatic update");
                try {
                    // While there is still a delayedServicePurgeState in the queue with an expired poll
                    while ((delayedServicePurgeState = servicePollDelayQueue.poll()) != null) {
                        ServicePurgeState servicePurgeState = delayedServicePurgeState.getServicePurgeState();
                        PurgeState purgeState = servicePurgeState.getPurgeState();
                        PurgeStatus status = purgeState.getPurgeStatus();
                        String appName = delayedServicePurgeState.getApplicationName();
                        String serviceName = delayedServicePurgeState.getServiceName();

                        // If the last update indicates the service is still working on that purge then ask the service for an update
                        if (serviceStillRunning(status)) {
                            try {
                                purgeState = getServiceClientAndUpdate(purgeState, appName, serviceName);
                                status = purgeState.getPurgeStatus();
                                servicesUpdated.add(appName + "_" + serviceName + ":" + purgeState.getPurgeId()
                                        + ":" + purgeState.getPurgeStatus());

                                // If the purge is still being worked on by that purge then put it back in the queue
                                if (serviceStillRunning(status)) {
                                    DelayedServicePurgeState newDelayedServicePurgeState = new DelayedServicePurgeState(
                                            servicePurgeState, appName, serviceName);
                                    servicePollDelayQueue.offer(newDelayedServicePurgeState);
                                }
                            } catch (Exception e) {
                                logError(e, evt, "Automatic update failed [" + e.getClass().getName() + ":"
                                        + e.getMessage() + "] for this app/service:" + appName + "_" + serviceName);
                            }
                        } else {
                            // (usually will only get here if the service ran the initial beginPurge synchronously)
                            try {
                                updatePurge(securityClient.fetchAppToken().deepCopy(), purgeState,
                                        delayedServicePurgeState.getApplicationName(),
                                        delayedServicePurgeState.getServiceName());
                                servicesUpdated
                                        .add(appName + "_" + serviceName + ":" + purgeState.getPurgeStatus());
                            } catch (Exception e) {
                                logError(e, evt, "Automatic update failed [" + e.getClass().getName() + ":"
                                        + e.getMessage() + "] for this app/service:" + appName + "_" + serviceName);
                            }
                        }
                    }
                } catch (Exception e) {
                    logError(e, evt,
                            "Automatic update failed [" + e.getClass().getName() + ":" + e.getMessage() + "]");
                } finally {
                    evt.arg("Updated services", servicesUpdated);
                    auditLogger.logEvent(evt);
                    logEventToPlainLogs(logger, evt);
                }
            } catch (Exception e) {
                e.printStackTrace();
                auditLogger.log("Automatic update failed [" + e.getClass().getName() + ":" + e.getMessage() + "]");
                logger.error("Automatic update failed [" + e.getClass().getName() + ":" + e.getMessage() + "]");
            }
        }
    };

    // This class starts the initial Purge for all services, allows beginPurge to be started asynchronously
    private class Purger implements Runnable {
        PurgeInitiationResult result;
        EzSecurityToken token;
        CentralPurgeType centralPurgeType;

        Purger(PurgeInitiationResult result, EzSecurityToken token, CentralPurgeType centralPurgeType) {
            this.result = result;
            this.token = token;
            this.centralPurgeType = centralPurgeType;
        }

        public void run() {
            logger.debug("Starting Purger");
            CentralPurgeState centralPurgeState = null;
            AuditEvent evt = event(AuditEventType.FileObjectCreate.getName(), token).arg("event", "startingPurge")
                    .arg("PurgeId", result.getPurgeId()).arg("purge type", centralPurgeType);
            try {
                // starts the purge and updates the backend
                centralPurgeState = getCentralPurgeState(result.purgeId);

                Map<String, ApplicationPurgeState> appMap = centralPurgeState.getApplicationStates();
                appMap = servicePurger(token, result.getPurgeId(), result.getToBePurged(), appMap, false,
                        centralPurgeType, evt);
                centralPurgeState.setApplicationStates(appMap);

                updateCentralPurgeState(centralPurgeState, result.getPurgeId());
            } catch (UnknownHostException e) {
                logError(e, evt, "Purger unable to reach MongoDB:[" + e.getClass().getName() + ":" + e.getMessage()
                        + "] on purgeId=" + result.getPurgeId());
            } catch (Exception e) {
                logError(e, evt, "Purger failed [" + e.getClass().getName() + ":" + e.getMessage() + "] on purgeId="
                        + result.getPurgeId());
            } finally {
                auditLogger.logEvent(evt);
                logEventToPlainLogs(logger, evt);
            }
        }
    }

    // This class starts the initial ageOff for all services, allows ageOffs to be started asynchronously
    private class AgeOffEventPurger implements Runnable {
        AgeOffInitiationResult result;
        EzSecurityToken token;
        boolean synchronous = false;
        AuditEvent evt;

        AgeOffEventPurger(AgeOffInitiationResult result, EzSecurityToken token, AuditEvent evt) {
            this.result = result;
            this.token = token;
            this.evt = evt;
        }

        public void run() {
            logger.info("Starting AgeOffEventPurger");

            try {
                // starts the ageOff and updates the backend
                CentralAgeOffEventState centralAgeOffEventState = getCentralAgeOffEventState(result.getAgeOffId());

                Map<String, ApplicationPurgeState> appMap = centralAgeOffEventState.getApplicationStates();
                appMap = servicePurger(token, result.getAgeOffId(), result.getAgeOffDocumentIds(), appMap,
                        synchronous, CentralPurgeType.NORMAL, evt);

                centralAgeOffEventState.setApplicationStates(appMap);

                updateCentralAgeOffEventState(centralAgeOffEventState, result.getAgeOffId());
            } catch (UnknownHostException e) {
                logError(e, evt, "CentralPurgeService unable to reach MongoDB in AgeOffEventPurger:["
                        + e.getClass().getName() + ":" + e.getMessage() + "]");
            } catch (Exception e) {
                logError(e, evt, "AgeOffEventPurger failed [" + e.getClass().getName() + ":" + e.getMessage()
                        + "] on ageOffId=" + result.getAgeOffId());
            }
        }
    }

    public EzCentralPurgeServiceHandler() {
    }

    // A method used by ThriftRunner to start the service
    @Override
    public TProcessor getThriftProcessor() {
        try {
            logger.debug("Starting getThriftProcessor");
            init();
            return new EzCentralPurgeService.Processor(this);
        } catch (Exception e) {
            e.printStackTrace();
            return null;
        }
    }

    // TODO: unit test all the things
    // TODO: integration test
    /* Sets up the service with two other running threads. One runs every 5 seconds checking for updates on purges.
     * The other checks every midnight for documents that need to be aged off. Also checks to see if the CentralPurge
     * MongoDB backend is in sync with the Provenance titan backend
     */
    private void init() throws TException {
        configuration = getConfigurationProperties();
        securityClient = new EzbakeSecurityClient(configuration);
        EzSecurityToken centralPurgeServiceToken = securityClient.fetchAppToken();
        AuditEvent evt = event(AuditEventType.ApplicationInitialization.getName(), centralPurgeServiceToken)
                .arg("event", "init");
        try {
            logger.info("Starting init");
            servicePollDelayQueue = new DelayQueue<>();
            EzSecurityTokenWrapper ezSecurityTokenWrapper = new EzSecurityTokenWrapper(centralPurgeServiceToken);
            purgeAppSecurityId = ezSecurityTokenWrapper.getSecurityId();

            // Set an instance of Calendar for Midnight.
            Calendar c = Calendar.getInstance();
            c.add(Calendar.DAY_OF_MONTH, 1);
            c.set(Calendar.HOUR_OF_DAY, 0);
            c.set(Calendar.MINUTE, 0);
            c.set(Calendar.SECOND, 0);
            c.set(Calendar.MILLISECOND, 0);

            //c.add(Calendar.MINUTE, 1);

            // Starts each of the delayed/recurring threads
            final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(3);
            final Runnable automaticAgeOff = new AutomaticAgeOff();
            final Runnable automaticUpdate = new AutomaticUpdate();

            final ScheduledFuture<?> automaticAgeOffHandle = scheduler.scheduleAtFixedRate(automaticAgeOff,
                    c.getTimeInMillis() - System.currentTimeMillis(), 24 * 3600000, MILLISECONDS);
            final ScheduledFuture<?> automaticUpdateHandle = scheduler.scheduleAtFixedRate(automaticUpdate, 2, 2,
                    SECONDS);
            // Just for testing
            // final ScheduledFuture<?> automaticAgeOffHandle = scheduler.scheduleAtFixedRate(automaticAgeOff, c.getTimeInMillis() - System.currentTimeMillis(), 60000*10, MILLISECONDS);

            initialized = true;
            ProvenanceService.Client client = null;
            ThriftClientPool pool = null;
            try {
                pool = new ThriftClientPool(configuration);
                EzSecurityToken centralTokenForProvenance = securityClient
                        .fetchAppToken(getProvenanceSecurityId(pool));
                client = getProvenanceThriftClient(pool);
                if (client.ping()) {
                    boolean ageOffOutOfSync = false;
                    boolean purgeOutOfSync = false;

                    // Check to see if there are any ageOffEvents which reference ageOffRules not in Provenance's backend
                    List<AgeOffRule> ageOffRules = client.getAllAgeOffRules(centralTokenForProvenance, 0, 0);
                    List<Long> provenanceAgeOffRules = new LinkedList<>();
                    for (AgeOffRule ageOffRule : ageOffRules) {
                        provenanceAgeOffRules.add(ageOffRule.getId());
                    }
                    List<Long> ageOffEvents = this.getAllAgeOffEvents(centralPurgeServiceToken);
                    List<CentralAgeOffEventState> ageOffEventStates = this
                            .getAgeOffEventState(centralPurgeServiceToken, ageOffEvents);
                    // note: there can be rules in the provenance service that don't yet have an event trying to age them
                    for (CentralAgeOffEventState centralAgeOffEventState : ageOffEventStates) {
                        if (!provenanceAgeOffRules.contains(centralAgeOffEventState.getAgeOffRuleId())) {
                            ageOffOutOfSync = true;
                            break;
                        }
                    }

                    // Check to see if there are any purgeIds in either CentralPurge's MongoDB or Provenance's Titan that aren't in the other
                    List<Long> provenancePurgeIds = client.getAllPurgeIds(centralTokenForProvenance);
                    List<CentralPurgeState> centralPurgeStates = this.getPurgeState(centralPurgeServiceToken,
                            getAllPurgeIdsInMongo());
                    List<Long> centralPurgePurgeIds = new LinkedList<>();
                    for (CentralPurgeState centralPurgeState : centralPurgeStates) {
                        centralPurgePurgeIds.add(centralPurgeState.getPurgeInfo().getId());
                    } /*
                      for (CentralAgeOffEventState centralAgeOffEventState : ageOffEventStates){
                      centralPurgePurgeIds.add(centralAgeOffEventState.getAgeOffEventInfo().getId());
                      }*/
                    // Need to verify if either contain a purgeId that is not in the other
                    if (!centralPurgePurgeIds.containsAll(provenancePurgeIds)
                            || !provenancePurgeIds.containsAll(centralPurgePurgeIds)) {
                        purgeOutOfSync = true;
                    }

                    // If either out of sync, throw an exception indicating that the service is still running but the backends are out of sync
                    if (ageOffOutOfSync || purgeOutOfSync) {
                        throw new CentralPurgeServiceException(
                                "Initialized but MongoDB out of sync with Provenance's titan DB, check ageOffCollection & purgeCollection");
                    }

                } else {
                    throw new CentralPurgeServiceException("Initialized but unable to reach provenance service");
                }
            } catch (Exception e) {
                logError(e, evt, e.getMessage());
            } finally {
                if (client != null)
                    returnClientToPool(client, pool);
                if (pool != null)
                    pool.close();
            }
        } catch (Exception e) {
            logError(e, evt, "Init failed [" + e.getClass().getName() + ":" + e.getMessage() + "]");
            throw new CentralPurgeServiceException("CentralPurgeService encountered an error in init:["
                    + e.getClass().getName() + ":" + e.getMessage() + "]");
        } finally {
            auditLogger.logEvent(evt);
            logEventToPlainLogs(logger, evt);
        }
    }

    /*  beginPurge()
     * This method should use the provided URIs to start a new purge event within
     * the provenance service utilizing that services markForPurge(). It should
     * return the ezProvenance.PurgeInitiationResult generated by that service
     * as an indication that the purge has begun, then discover all purge
     * services within the system and begin calling the beginPurge() methods for
     * each of those applications.
     *
     * Initially, it is probably best to call one service, wait for it to complete,
     * then call the next service. The service must not block other
     * requests while the purge is running.
     *
     * Additionally, this method should create some state internal to the
     * EzCentralPurgeService (likely backed by MongoDB) to be able to support
     * getPurgeState() queries while the purge is ongoing and after it has completed.
     *
     * @param token The token must be validated and only requests from within the
     *              this same application should be allowed. Others should throw
     *              EzSecurityTokenException.
     * @param uris  This is the list of uris for which we must purge all descendant
     *              documents.
     * @param name  A human-readable name for this purge. This service should enforce
     *              the uniqueness of names.
     * @param description A human-readable (potentially long) description of why this
     *              purge is taking place
     * @returns The method returns the ezProvenance.PurgeInitiation result generated by
     *          the provenance services markForPurge() method that is called by this
     *          method.
     */
    @Override
    public PurgeInitiationResult beginPurge(final EzSecurityToken token, final List<String> uris, String name,
            final String description) throws EzSecurityTokenException, TException {
        return beginPurgeHelper(token, uris, name, description, CentralPurgeType.NORMAL);
    }

    // A begin Purge method for Virus purges
    @Override
    public PurgeInitiationResult beginVirusPurge(EzSecurityToken token, List<String> uris, String name,
            String description) throws EzSecurityTokenException, TException {
        return beginPurgeHelper(token, uris, name, description, CentralPurgeType.VIRUS);
    }

    // Just a helper method for the beginPurge methods
    private PurgeInitiationResult beginPurgeHelper(EzSecurityToken token, List<String> uris, String name,
            String description, CentralPurgeType centralPurgeType) throws EzSecurityTokenException, TException {

        ProvenanceService.Client client = null;
        PurgeInitiationResult result = null;
        securityClient = new EzbakeSecurityClient(configuration);
        EzSecurityToken centralTokenForProvenance = null;

        AuditEvent evt = event(AuditEventType.FileObjectCreate.getName(), token).arg("event", "beginPurge")
                .arg("name", name).arg("description", description).arg("purge type", centralPurgeType);

        ThriftClientPool pool = null;
        try {
            pool = new ThriftClientPool(configuration);
            validateCentralPurgeSecurityToken(token);
            centralTokenForProvenance = securityClient.fetchDerivedTokenForApp(token,
                    getProvenanceSecurityId(pool));
            client = getProvenanceThriftClient(pool);

            // Enforce uniqueness of names
            for (Long purgeId : client.getAllPurgeIds(centralTokenForProvenance)) {
                PurgeInfo purgeInfoInside = client.getPurgeInfo(centralTokenForProvenance, purgeId);
                if (purgeInfoInside != null && name.equals(purgeInfoInside.getName())) {
                    throw new CentralPurgeServiceException(
                            "A purge with that name has already been created. Purge name must be unique.");
                }
            }

            // Tell Provenance service that the purge is occurring and get the started purge's information
            result = client.markDocumentForPurge(centralTokenForProvenance, uris, name, description);
            evt.arg("purgeId", result.purgeId);
            PurgeInfo purgeInfo = client.getPurgeInfo(centralTokenForProvenance, result.getPurgeId());

            // Initialize the CentralPurgeState information
            Map<String, ApplicationPurgeState> appStates = initializeApplicationState(result.purgeId);
            CentralPurgeState centralPurgeState = new CentralPurgeState();
            centralPurgeState.setCentralStatus(CentralPurgeStatus.ACTIVE);
            centralPurgeState.setPurgeInfo(purgeInfo);
            centralPurgeState.setCentralPurgeType(centralPurgeType);
            centralPurgeState.setApplicationStates(appStates);

            updateCentralPurgeState(centralPurgeState, purgeInfo.getId());

            // Tell services to start the purge (asynchronously) and stops the thread when finished
            ExecutorService executorService = Executors.newSingleThreadExecutor();
            executorService.execute(new Purger(result, token, centralPurgeType));
            executorService.shutdown();

        } catch (CentralPurgeServiceException e) {
            logError(e, evt, e.getMessage());
            throw e;
        } catch (EzSecurityTokenException e) {
            logError(e, evt, "CentralPurgeService failed when trying to validate token:[" + e.getClass().getName()
                    + ":" + e.getMessage() + "]");
            throw e;
        } catch (UnknownHostException e) {
            logError(e, evt, "CentralPurgeService unable to reach MongoDB in beginPurge:[" + e.getClass().getName()
                    + ":" + e.getMessage() + "]");
            throw new CentralPurgeServiceException("CentralPurgeService unable to reach MongoDB in beginPurge:["
                    + e.getClass().getName() + ":" + e.getMessage() + "]");
        } catch (Exception e) {
            logError(e, evt, "CentralPurgeService encountered an exception in beginPurge:[" + e.getClass().getName()
                    + ":" + e.getMessage() + "]");
            throw new CentralPurgeServiceException("CentralPurgeService encountered an exception in beginPurge:["
                    + e.getClass().getName() + ":" + e.getMessage() + "]");
        } finally {
            if (client != null)
                returnClientToPool(client, pool);
            if (pool != null)
                pool.close();
            auditLogger.logEvent(evt);
            logEventToPlainLogs(logger, evt);
        }
        return result;
    }

    /* getPurgeState()
     * This method is used by the UI to get the detailed status of a list of purge
     * event IDs. The UI can fetch the list of all purge event IDs by calling
     * the provenance services getAllPurgeIds().
     *
     * The UI can achieve paging by fetching the entire list from the provenance
     * services getAllPurgeIds() and then only requesting a slice of that list
     * from the central purge service.
     *
     * @param token The token must be validated and only requests from within the
     *              this same application should be allowed. Others should throw
     *              EzSecurityTokenException.
     * @param purgIds This is the list of purges for which the UI is requesting
     *                detailed information
     * @returns PurgeState The state for the selected purgeIds
     */
    @Override
    public List<CentralPurgeState> getPurgeState(EzSecurityToken token, List<Long> purgeIds)
            throws EzSecurityTokenException, TException {

        DBCollection purgeColl = null;
        Mongo mongoClient = null;
        AuditEvent evt = event(AuditEventType.FileObjectAccess.getName(), token).arg("event", "getPurgeState")
                .arg("purgeIds", purgeIds);
        try {
            validateCentralPurgeSecurityToken(token);

            // Get access to the purge collection within Mongo
            MongoConfigurationHelper mongoConfigurationHelper = new MongoConfigurationHelper(configuration);
            MongoHelper mongoHelper = new MongoHelper(configuration);
            mongoClient = mongoHelper.getMongo();
            DB mongoDB = mongoClient.getDB(mongoConfigurationHelper.getMongoDBDatabaseName());
            purgeColl = mongoDB.getCollection(PURGE_COLLECTION);

            List<CentralPurgeState> result = new ArrayList<CentralPurgeState>();

            // Gets all centralPurgeStates that are in the purgeIds list
            BasicDBObject query = new BasicDBObject(EzCentralPurgeServiceHelpers.PurgeId,
                    new BasicDBObject("$in", purgeIds));
            DBCursor cursor = purgeColl.find(query);

            // Decodes each centralPurgeState form how it's stored in MongoDB and adds it to the return variable
            for (DBObject dbObject : cursor) {
                CentralPurgeState centralPurgeState = decodeCentralPurgeState(
                        (DBObject) dbObject.get(CentralPurgeStateString));
                result.add(centralPurgeState);
            }
            return result;
        } catch (EzSecurityTokenException e) {
            logError(e, evt, "CentralPurgeService failed when trying to validate token:[" + e.getClass().getName()
                    + ":" + e.getMessage() + "]");
            throw e;
        } catch (UnknownHostException e) {
            logError(e, evt, "CentralPurgeService unable to reach MongoDB in getPurgeState:["
                    + e.getClass().getName() + ":" + e.getMessage() + "]");
            throw new CentralPurgeServiceException("CentralPurgeService unable to reach MongoDB in getPurgeState:["
                    + e.getClass().getName() + ":" + e.getMessage() + "]");
        } catch (Exception e) {
            logError(e, evt, "CentralPurgeService encountered an exception in getPurgeState:["
                    + e.getClass().getName() + ":" + e.getMessage() + "]");
            throw new CentralPurgeServiceException("CentralPurgeService encountered an exception in getPurgeState:["
                    + e.getClass().getName() + ":" + e.getMessage() + "]");
        } finally {
            auditLogger.logEvent(evt);
            logEventToPlainLogs(logger, evt);
            if (mongoClient != null)
                mongoClient.close();
        }
    }

    /* beginManualAgeOff()
     * This method is used by the UI to initiate the manual execution of an age off
     * operation on the specified age off rule. The UI should call the provenance
     * service to get the list of age off rules.
     *
     *
     * @param token The token must be validated and only requests from within the
     *              this same application should be allowed. Others should throw
     *              EzSecurityTokenException.
     * @param purgeIds This is the list of purges for which the UI is requesting
     *                detailed information
     * @returns PurgeState The state for the selected purgeIds
     */
    @Override
    public AgeOffEventInfo beginManualAgeOff(EzSecurityToken token, long ruleId)
            throws EzSecurityTokenException, ProvenanceAgeOffRuleNotFoundException, TException {
        return executeAgeOff(token, ruleId, false);
    }

    /* updatePurge()
     * This method is called by the individual applications services that actually purge
     * data to inform the Central Purge Service of completion or error of a purge.
     *
     * The individual applications should call this method if their purge status changes
     * to one of:
     *        STOPPING,
     *        ERROR,
     *        FINISHED_COMPLETE,
     *        FINISHED_INCOMPLETE
     *
     * The Central Purge Service should also poll the state of the individual applications
     * periodically for this information.
     *
     * Keep in mind that to an application service there is no difference between an ageOff
     * event and a manual purge. So this method must handle both cases.
     * 
     * @param token The token must be validated and requests from ANY APPLICATION
     *              should be allowed. If the token is not valid, generate a
     *              EzSecurityTokenException.
     *              The calling applications security ID can be found in this
     *              token to identify the calling application.
     * @param state This is the purge state that the application is updating.
     */
    @Override
    public void updatePurge(EzSecurityToken token, PurgeState inputPurgeState, String applicationName,
            String serviceName) throws EzSecurityTokenException, TException {
        DBObject dbObject = null;
        Map<String, ApplicationPurgeState> appStatesMap = null;
        CentralPurgeState centralPurgeState = null;
        CentralAgeOffEventState centralAgeOffEventState = null;
        Set<Long> centralCompletelyPurgedSet = null;
        Set<Long> centralToBePurgedSet = null;
        DBCollection purgeColl = null;
        DBCollection ageOffColl = null;
        Mongo mongoClient = null;

        AuditEvent evt = event(AuditEventType.FileObjectModify.getName(), token).arg("event", "update purge")
                .arg("purgeId", inputPurgeState.getPurgeId()).arg("service name", serviceName)
                .arg("application name", applicationName);

        ThriftClientPool pool = null;
        try {
            pool = new ThriftClientPool(configuration);
            securityClient.validateReceivedToken(token);
            // Validates that the application that is calling update purge is allowed to update for the passed appName
            String securityId = "";
            EzSecurityTokenWrapper wrapper = new EzSecurityTokenWrapper(token);
            // note: if the  centralPurgeService is calling then any appName can be updated
            if (wrapper.getSecurityId().equals(purgeAppSecurityId)) {
                securityId = purgeAppSecurityId;
            } else {
                securityId = pool.getSecurityId(getSecurityName(applicationName, serviceName));
            }

            if (!securityId.equals(wrapper.getSecurityId())) {
                throw new EzSecurityTokenException(
                        "The security id for the token does match the applicationName passed");
            }

            // Get access to the ageoff collection within Mongo
            MongoConfigurationHelper mongoConfigurationHelper = new MongoConfigurationHelper(configuration);
            MongoHelper mongoHelper = new MongoHelper(configuration);
            mongoClient = mongoHelper.getMongo();
            DB mongoDB = mongoClient.getDB(mongoConfigurationHelper.getMongoDBDatabaseName());
            purgeColl = mongoDB.getCollection(PURGE_COLLECTION);
            ageOffColl = mongoDB.getCollection(AGEOFF_COLLECTION);

            Long purgeId = inputPurgeState.getPurgeId();

            // Attempt to get the CentralPurgeState
            BasicDBObject query = new BasicDBObject(EzCentralPurgeServiceHelpers.PurgeId, purgeId);
            DBCursor cursor = purgeColl.find(query);

            boolean ageOff = false;
            CentralPurgeStatus centralPurgeStatus;

            // Check to see if the id passed corresponds to a purge event
            if (cursor.hasNext()) {
                //Set the map of application states and the set of ids to purge
                dbObject = cursor.next();
                centralPurgeState = decodeCentralPurgeState((DBObject) dbObject.get(CentralPurgeStateString));
                appStatesMap = centralPurgeState.getApplicationStates();

                PurgeInfo purgeInfo = centralPurgeState.getPurgeInfo();
                centralCompletelyPurgedSet = purgeInfo.getPurgeDocumentIds();
                centralToBePurgedSet = purgeInfo.getPurgeDocumentIds();
                centralPurgeStatus = centralPurgeState.getCentralStatus();
            } else {
                query = new BasicDBObject(EzCentralPurgeServiceHelpers.AgeOffEventId, purgeId);
                // If it doesn't exist as a purge, check to see if it is an ageOffEvent
                cursor = ageOffColl.find(query);

                if (cursor.hasNext()) {
                    //Set the map of application states and the set of ids to purge
                    dbObject = cursor.next();
                    centralAgeOffEventState = decodeCentralAgeOffEventState(
                            (DBObject) dbObject.get(CentralAgeOffStateString));
                    appStatesMap = centralAgeOffEventState.getApplicationStates();
                    AgeOffEventInfo ageOffEventInfo = centralAgeOffEventState.getAgeOffEventInfo();
                    centralToBePurgedSet = ageOffEventInfo.getPurgeSet();
                    centralCompletelyPurgedSet = ageOffEventInfo.getPurgeSet();
                    centralPurgeStatus = centralAgeOffEventState.getCentralStatus();
                    ageOff = true;
                } else {
                    throw new CentralPurgeServiceException("No purge with purgeId:" + purgeId);
                }
            }

            ServicePurgeState servicePurgeState = null;
            Map<String, ServicePurgeState> servicePurgeStatesMap = null;
            ApplicationPurgeState applicationPurgeState = null;
            // Gets the mongoDB entry for the service that is updating it's purge status.
            try {
                applicationPurgeState = appStatesMap.get(applicationName);
                servicePurgeStatesMap = applicationPurgeState.getServicePurgestates();
                servicePurgeState = servicePurgeStatesMap.get(serviceName);
                if (servicePurgeState == null) {
                    throw new NullPointerException("Failed to find [" + applicationName + "_" + serviceName
                            + "] for purgeId" + inputPurgeState.getPurgeId() + " to update");
                }
            } catch (NullPointerException e) {
                throw e;
            }
            // Update the ServicePurgeState and put it back
            servicePurgeState.setTimeLastPoll(getCurrentDateTime());
            servicePurgeState.setPurgeState(inputPurgeState);
            servicePurgeStatesMap.put(serviceName, servicePurgeState);
            appStatesMap.put(applicationName, applicationPurgeState);
            boolean interventionNeeded = false;
            boolean stopped = true;
            Set<Long> servicePurged;

            /* These nested loops check each service to get an update of the CompletelyPurgedSet, see if any purge
             * service is still running and if manual intervention is/will be needed.
             */
            // Loop through all apps
            for (String appNameIter : appStatesMap.keySet()) {
                ApplicationPurgeState applicationPurgeStateInner = appStatesMap.get(appNameIter);
                Map<String, ServicePurgeState> servicePurgeStates = applicationPurgeStateInner
                        .getServicePurgestates();

                //Loop through all services
                for (String serviceNameIter : servicePurgeStates.keySet()) {
                    PurgeState applicationServicePurgeState = servicePurgeStates.get(serviceNameIter)
                            .getPurgeState();
                    servicePurged = applicationServicePurgeState.getPurged();
                    applicationServicePurgeState.getPurged().removeAll(applicationServicePurgeState.getNotPurged());

                    //update based on current service
                    centralCompletelyPurgedSet = Sets.intersection(centralCompletelyPurgedSet, servicePurged);
                    if (serviceStillRunning(applicationServicePurgeState.getPurgeStatus())) {
                        stopped = false;
                    }
                    if (!(applicationServicePurgeState.getNotPurged().isEmpty())) {
                        interventionNeeded = true;
                    }
                }
            }

            // If all of the ids that needed to be purged have been purged then it resolved automatically
            boolean resolved = false;
            if (centralCompletelyPurgedSet.containsAll(centralToBePurgedSet)) {
                resolved = true;
                centralPurgeStatus = CentralPurgeStatus.RESOLVED_AUTOMATICALLY;
            }
            // If one of the services has a document that couldn't be
            // automatically resolved, manual intervention is needed
            if (centralPurgeStatus != CentralPurgeStatus.RESOLVED_MANUALLY
                    && centralPurgeStatus != CentralPurgeStatus.RESOLVED_AUTOMATICALLY) {
                if (interventionNeeded) {
                    if (stopped) {
                        centralPurgeStatus = CentralPurgeStatus.STOPPED_MANUAL_INTERVENTION_NEEDED;
                    } else {
                        centralPurgeStatus = CentralPurgeStatus.ACTIVE_MANUAL_INTERVENTION_WILL_BE_NEEDED;
                    }
                }
            } else {
                resolved = true;
            }

            if (ageOff == false) {
                // If it is a purge event, update the CentralPurgeState in MongoDB
                centralPurgeState.setApplicationStates(appStatesMap);
                centralPurgeState.setCentralStatus(centralPurgeStatus);

                dbObject.put(CentralPurgeStateString, encodeCentralPurgeState(centralPurgeState));
                purgeColl.update(query, dbObject, true, false);

                // Also need to update the purge in the ProvenanceService
                ProvenanceService.Client provenanceClient = null;
                try {
                    provenanceClient = getProvenanceThriftClient(pool);
                    EzSecurityToken centralTokenForProvenance = securityClient.fetchDerivedTokenForApp(token,
                            getProvenanceSecurityId(pool));
                    provenanceClient.updatePurge(centralTokenForProvenance, purgeId, centralCompletelyPurgedSet,
                            null, resolved);
                    PurgeInfo purgeInfo = provenanceClient.getPurgeInfo(centralTokenForProvenance, purgeId);
                    centralPurgeState.setPurgeInfo(purgeInfo);
                    updateCentralPurgeState(centralPurgeState, purgeId);
                } finally {
                    if (provenanceClient != null)
                        returnClientToPool(provenanceClient, pool);
                }
            } else {
                // If it is an ageOffEvent, update the CentralAgeOffState in MongoDB
                centralAgeOffEventState.setApplicationStates(appStatesMap);
                centralAgeOffEventState.setCentralStatus(centralPurgeStatus);

                AgeOffEventInfo ageOffEventInfo = centralAgeOffEventState.getAgeOffEventInfo();
                ageOffEventInfo.setCompletelyPurgedSet(centralCompletelyPurgedSet);
                ageOffEventInfo.setResolved(resolved);
                centralAgeOffEventState.setAgeOffEventInfo(ageOffEventInfo);
                dbObject.put(CentralAgeOffStateString, encodeCentralAgeOffEventState(centralAgeOffEventState));
                ageOffColl.update(query, dbObject, true, false);

                // If there are ids aged by all services then tell the provenance service
                if (!centralCompletelyPurgedSet.isEmpty()) {
                    ProvenanceService.Client provenanceClient = null;
                    try {
                        provenanceClient = getProvenanceThriftClient(pool);
                        EzSecurityToken centralTokenForProvenance = securityClient.fetchDerivedTokenForApp(token,
                                getProvenanceSecurityId(pool));
                        provenanceClient.markDocumentAsAged(centralTokenForProvenance, centralCompletelyPurgedSet);
                    } finally {
                        if (provenanceClient != null)
                            returnClientToPool(provenanceClient, pool);
                    }
                }
            }
            evt.arg("status", servicePurgeState.getPurgeState().getPurgeStatus().name());
            logger.info("[" + applicationName + "_" + serviceName + "] purgeId:" + inputPurgeState.getPurgeId()
                    + " purgedIds:" + inputPurgeState.getPurged() + " status:" + inputPurgeState.getPurgeStatus());
        } catch (CentralPurgeServiceException e) {
            logError(e, evt, e.getMessage());
            throw e;
        } catch (NullPointerException e) {
            logError(e, evt, "CentralPurgeService encountered an exception in updatePurge:["
                    + e.getClass().getName() + ":" + e.getMessage() + "]");
            throw new CentralPurgeServiceException("CentralPurgeService encountered an exception in updatePurge:["
                    + e.getClass().getName() + ":" + e.getMessage() + "]");
        } catch (EzSecurityTokenException e) {
            logError(e, evt, "CentralPurgeService failed when trying to validate token:[" + e.getClass().getName()
                    + ":" + e.getMessage() + "]");
            throw e;
        } catch (UnknownHostException e) {
            logError(e, evt, "CentralPurgeService unable to reach MongoDB in updatePurge:[" + e.getClass().getName()
                    + ":" + e.getMessage() + "]");
            throw new CentralPurgeServiceException("CentralPurgeService unable to reach MongoDB in updatePurge:["
                    + e.getClass().getName() + ":" + e.getMessage() + "]");
        } catch (Exception e) {
            logError(e, evt, "CentralPurgeService encountered an exception in updatePurge:["
                    + e.getClass().getName() + ":" + e.getMessage() + "]");
            throw new CentralPurgeServiceException("CentralPurgeService encountered an exception in updatePurge:["
                    + e.getClass().getName() + ":" + e.getMessage() + "]");
        } finally {
            if (pool != null)
                pool.close();
            auditLogger.logEvent(evt);
            logEventToPlainLogs(logger, evt);
            if (mongoClient != null)
                mongoClient.close();
        }
    }

    /* resolvePurge()
     * This method allows the UI to mark a a purge as being resolved and supply a note
     * regarding  the resolution of the purge.
        
     * Manual resolution of a purge through the UI could happen for a couple reasons
     * - Not all documents could be automatically purged, so an admin is marking
     *   successful completion of the purge.
     * - One application is not properly responding to a purge and admin interaction
     *   is required
     *
     * When this method is called, the Central Purge Service should make final updates
     * to the state of the purge stored in the Provenance Service and any state regarding
     * the purge stored elsewhere. This is done via that services updatePurge() method.
     *
     * When calling the provenance services updatePurge() method, the note should be
     * included and resolved should be set to True.
     *
     * @param token The token must be validated and only requests from within the
     *              this same application should be allowed. Others should throw
     *              EzSecurityTokenException.
     * @param purgId This is the purge that is being marked as resolved
     * @param notes Notes as to why this purge was manually resolved.
     */
    @Override
    public void resolvePurge(EzSecurityToken token, long purgeId, String notes)
            throws EzSecurityTokenException, TException {
        ProvenanceService.Client provenanceClient = null;
        AuditEvent evt = event(AuditEventType.FileObjectModify.getName(), token).arg("event", "resolve purge")
                .arg("purgeId", purgeId).arg("notes", notes);

        ThriftClientPool pool = null;
        try {
            pool = new ThriftClientPool(configuration);
            validateCentralPurgeSecurityToken(token);
            EzSecurityToken centralTokenForProvenance = securityClient.fetchDerivedTokenForApp(token,
                    getProvenanceSecurityId(pool));

            // Get the centralPurgeState
            CentralPurgeState centralPurgeState = getCentralPurgeState(purgeId);
            if (centralPurgeState == null)
                throw new CentralPurgeServiceException("Did not find a purge with purgeID " + purgeId);
            CentralPurgeStatus centralStatus = centralPurgeState.getCentralStatus();
            if (centralStatus == CentralPurgeStatus.RESOLVED_AUTOMATICALLY
                    || centralStatus == CentralPurgeStatus.RESOLVED_MANUALLY)
                throw new CentralPurgeServiceException(
                        "The purge with purgeID " + purgeId + " has already been resolved");
            PurgeInfo purgeInfo = centralPurgeState.getPurgeInfo();

            // Update the purge in the provenance client
            provenanceClient = getProvenanceThriftClient(pool);
            notes = ", Manually resolved with note: " + notes;
            provenanceClient.updatePurge(centralTokenForProvenance, purgeId,
                    purgeInfo.getCompletelyPurgedDocumentIds(), notes, true);
            purgeInfo = provenanceClient.getPurgeInfo(centralTokenForProvenance, purgeId);

            // Update the purge in Mongo
            centralPurgeState.setCentralStatus(CentralPurgeStatus.RESOLVED_MANUALLY);
            centralPurgeState.setPurgeInfo(purgeInfo);

            // Cancel all services that are still running the purge
            Map<String, ApplicationPurgeState> appMap = centralPurgeState.getApplicationStates();
            appMap = cancelServices(token, appMap, purgeId, evt);
            centralPurgeState.setApplicationStates(appMap);

            updateCentralPurgeState(centralPurgeState, purgeId);
        } catch (CentralPurgeServiceException e) {
            logError(e, evt, e.getMessage());
            throw e;
        } catch (EzSecurityTokenException e) {
            logError(e, evt, "CentralPurgeService failed when trying to validate token:[" + e.getClass().getName()
                    + ":" + e.getMessage() + "]");
            throw e;
        } catch (UnknownHostException e) {
            logError(e, evt, "CentralPurgeService unable to reach MongoDB in resolvePurge:["
                    + e.getClass().getName() + ":" + e.getMessage() + "]");
            throw new CentralPurgeServiceException("CentralPurgeService unable to reach MongoDB in resolvePurge:["
                    + e.getClass().getName() + ":" + e.getMessage() + "]");
        } catch (Exception e) {
            logError(e, evt, "CentralPurgeService unable to reach MongoDB in resolvePurge:["
                    + e.getClass().getName() + ":" + e.getMessage() + "]");
            throw new CentralPurgeServiceException("CentralPurgeService encountered an exception in resolvePurge:["
                    + e.getClass().getName() + ":" + e.getMessage() + "]");
        } finally {
            if (pool != null)
                pool.close();
            if (provenanceClient != null)
                returnClientToPool(provenanceClient, pool);
            auditLogger.logEvent(evt);
            logEventToPlainLogs(logger, evt);
        }

    }

    /* getAgeOffEventState()
     * This method is used by the UI to get the detailed status of a list of ageOff
     * event IDs. The UI can fetch the list of all ageOff event IDs by calling
     * the getAllAgeOffEvents() method.
     *
     * The UI can achieve paging by fetching the entire list from the
     * getAllAgeOffEvents() and then only requesting a slice of that list
     * from this method.
     *
     * @param token The token must be validated and only requests from within the
     *              this same application should be allowed. Others should throw
     *              EzSecurityTokenException.
     * @param purgIds This is the list of purges for which the UI is requesting
     *                detailed information
     * @returns List<AgeOffEventInfo> The list of info for the selected ageOffEventIds
     */
    @Override
    public List<CentralAgeOffEventState> getAgeOffEventState(EzSecurityToken token, List<Long> ageOffEventIds)
            throws EzSecurityTokenException, TException {

        DBCollection ageOffColl = null;
        Mongo mongoClient = null;
        AuditEvent evt = event(AuditEventType.FileObjectAccess.getName(), token)
                .arg("event", "get age off event state").arg("age off event ids", ageOffEventIds);
        try {
            validateCentralPurgeSecurityToken(token);

            // Get access to the ageoff collection within Mongo

            MongoConfigurationHelper mongoConfigurationHelper = new MongoConfigurationHelper(configuration);
            MongoHelper mongoHelper = new MongoHelper(configuration);
            mongoClient = mongoHelper.getMongo();
            DB mongoDB = mongoClient.getDB(mongoConfigurationHelper.getMongoDBDatabaseName());
            ageOffColl = mongoDB.getCollection(AGEOFF_COLLECTION);

            List<CentralAgeOffEventState> result = new ArrayList<>();
            BasicDBObject query;
            DBCursor cursor;

            // Just need to get and decode the CentralAgeOffStates matching the ids from MongoDB
            query = new BasicDBObject(EzCentralPurgeServiceHelpers.AgeOffEventId,
                    new BasicDBObject("$in", ageOffEventIds));
            cursor = ageOffColl.find(query);
            for (DBObject dbObject : cursor) {
                CentralAgeOffEventState centralAgeOffEventState = decodeCentralAgeOffEventState(
                        (DBObject) dbObject.get(CentralAgeOffStateString));
                result.add(centralAgeOffEventState);
            }

            return result;
        } catch (EzSecurityTokenException e) {
            logError(e, evt, "CentralPurgeService failed when trying to validate token:[" + e.getClass().getName()
                    + ":" + e.getMessage() + "]");
            throw e;
        } catch (UnknownHostException e) {
            logError(e, evt, "CentralPurgeService unable to reach MongoDB in getAgeOffEventState:["
                    + e.getClass().getName() + ":" + e.getMessage() + "]");
            throw new CentralPurgeServiceException(
                    "CentralPurgeService unable to reach MongoDB in getAgeOffEventState:[" + e.getClass().getName()
                            + ":" + e.getMessage() + "]");
        } catch (Exception e) {
            logError(e, evt, "CentralPurgeService encountered an exception in getAgeOffEventState:["
                    + e.getClass().getName() + ":" + e.getMessage() + "]");
            throw new CentralPurgeServiceException(
                    "CentralPurgeService encountered an exception in getAgeOffEventState:[" + e.getClass().getName()
                            + ":" + e.getMessage() + "]");
        } finally {
            auditLogger.logEvent(evt);
            logEventToPlainLogs(logger, evt);
            if (mongoClient != null)
                mongoClient.close();
        }
    }

    /* getAllAgeOffEvents()
     * @param token The token must be validated and only requests from within the
     *              this same application should be allowed. Others should throw
     *              EzSecurityTokenException.
     *
     * @returns This method should find all AgeOffEvents and return the ageOffEvent id for
     * each
     */
    @Override
    public List<Long> getAllAgeOffEvents(EzSecurityToken token) throws TException {
        List<Long> result = null;
        DBCollection ageOffColl = null;
        Mongo mongoClient = null;

        AuditEvent evt = event(AuditEventType.FileObjectAccess.getName(), token).arg("event",
                "get all age off events");
        try {
            validateCentralPurgeSecurityToken(token);

            // Get access to the ageoff collection within Mongo

            MongoConfigurationHelper mongoConfigurationHelper = new MongoConfigurationHelper(configuration);
            MongoHelper mongoHelper = new MongoHelper(configuration);
            mongoClient = mongoHelper.getMongo();
            DB mongoDB = mongoClient.getDB(mongoConfigurationHelper.getMongoDBDatabaseName());
            ageOffColl = mongoDB.getCollection(AGEOFF_COLLECTION);

            // Just get all ids from MongoDB
            result = new ArrayList<Long>();
            DBCursor cursor = ageOffColl.find();
            while (cursor.hasNext()) {
                result.add((Long) cursor.next().get(EzCentralPurgeServiceHelpers.AgeOffEventId));
            }

            return result;

        } catch (EzSecurityTokenException e) {
            logError(e, evt, "CentralPurgeService failed when trying to validate token:[" + e.getClass().getName()
                    + ":" + e.getMessage() + "]");
            throw e;
        } catch (UnknownHostException e) {
            logError(e, evt, "CentralPurgeService unable to reach MongoDB in getAllAgeOffEvents:["
                    + e.getClass().getName() + ":" + e.getMessage() + "]");
            throw new CentralPurgeServiceException(
                    "CentralPurgeService unable to reach MongoDB in getAllAgeOffEvents:[" + e.getClass().getName()
                            + ":" + e.getMessage() + "]");
        } catch (Exception e) {
            logError(e, evt, "CentralPurgeService encountered an exception in getAllAgeOffEvents:["
                    + e.getClass().getName() + ":" + e.getMessage() + "]");
            throw new CentralPurgeServiceException(
                    "CentralPurgeService encountered an exception in getAllAgeOffEvents:[" + e.getClass().getName()
                            + ":" + e.getMessage() + "]");
        } finally {
            auditLogger.logEvent(evt);
            logEventToPlainLogs(logger, evt);
            if (mongoClient != null)
                mongoClient.close();
        }
    }

    /* resolveAgeOffEvent()
     * This method allows the UI to mark an ageOffEvent as being resolved and supply a note
     * regarding  the resolution of the purge.
        
     * Manual resolution of an ageOffEvent through the UI could happen for a couple reasons
     * - Not all documents could be automatically purged, so an admin is marking
     *   successful completion of the ageOffEvent.
     * - One application is not properly responding to a purge and admin interaction
     *   is required
     *
     * When this method is called, the Central Purge Service should make final updates
     * to any state regarding the ageOffEvent stored elsewhere. This is done via that
     * services updatePurge() method.
     *
     * When calling the provenance services updatePurge() method, the note should be
     * included and resolved should be set to True.
     *
     * @param token The token must be validated and only requests from within the
     *              this same application should be allowed. Others should throw
     *              EzSecurityTokenException.
     * @param purgId This is the purge that is being marked as resolved
     * @param notes Notes as to why this purge was manually resolved.
     */
    @Override
    public void resolveAgeOffEvent(EzSecurityToken token, long ageOffEventId, String notes)
            throws EzSecurityTokenException, TException {

        AuditEvent evt = event(AuditEventType.FileObjectModify.getName(), token)
                .arg("event", "resolve age off event").arg("age off event id", ageOffEventId).arg("notes", notes);

        try {
            validateCentralPurgeSecurityToken(token);

            // Get and update the CentralAgeOffStates from MongoDB
            CentralAgeOffEventState centralAgeOffEventState = getCentralAgeOffEventState(ageOffEventId);
            centralAgeOffEventState.setCentralStatus(CentralPurgeStatus.RESOLVED_MANUALLY);
            AgeOffEventInfo ageOffEventInfo = centralAgeOffEventState.getAgeOffEventInfo();
            notes = ageOffEventInfo.getDescription() + ", Manually resolved with note: " + notes;
            ageOffEventInfo.setDescription(notes);
            ageOffEventInfo.setResolved(true);
            centralAgeOffEventState.setAgeOffEventInfo(ageOffEventInfo);

            // Cancel all services that are still running the purge
            Map<String, ApplicationPurgeState> appMap = centralAgeOffEventState.getApplicationStates();
            appMap = cancelServices(token, appMap, ageOffEventId, evt);
            centralAgeOffEventState.setApplicationStates(appMap);

            updateCentralAgeOffEventState(centralAgeOffEventState, ageOffEventId);
        } catch (EzSecurityTokenException e) {
            logError(e, evt, "CentralPurgeService failed when trying to validate token:[" + e.getClass().getName()
                    + ":" + e.getMessage() + "]");
            throw e;
        } catch (UnknownHostException e) {
            logError(e, evt, "CentralPurgeService unable to reach MongoDB in resolveAgeOffEvent:["
                    + e.getClass().getName() + ":" + e.getMessage() + "]");
            throw new CentralPurgeServiceException(
                    "CentralPurgeService unable to reach MongoDB in resolveAgeOffEvent:[" + e.getClass().getName()
                            + ":" + e.getMessage() + "]");
        } catch (Exception e) {
            logError(e, evt, "CentralPurgeService encountered an exception in resolveAgeOffEvent:["
                    + e.getClass().getName() + ":" + e.getMessage() + "]");
            throw new CentralPurgeServiceException(
                    "CentralPurgeService encountered an exception in resolveAgeOffEvent:[" + e.getClass().getName()
                            + ":" + e.getMessage() + "]");
        } finally {
            auditLogger.logEvent(evt);
            logEventToPlainLogs(logger, evt);
        }
    }

    @Override
    public CentralPurgeQueryResults getPagedSortedFilteredPurgeStates(EzSecurityToken token,
            List<CentralPurgeStatus> statuses, int pageNum, int numPerPage)
            throws EzSecurityTokenException, CentralPurgeServiceException {

        DBCollection purgeColl = null;
        Mongo mongoClient = null;
        AuditEvent evt = event(AuditEventType.FileObjectAccess.getName(), token)
                .arg("event", "getPagedSortedFilteredPurgeStates").arg("statuses", statuses).arg("pageNum", pageNum)
                .arg("numPerPage", numPerPage);
        try {
            validateCentralPurgeSecurityToken(token);

            // Get access to the purge collection within Mongo
            MongoConfigurationHelper mongoConfigurationHelper = new MongoConfigurationHelper(configuration);
            MongoHelper mongoHelper = new MongoHelper(configuration);
            mongoClient = mongoHelper.getMongo();
            DB mongoDB = mongoClient.getDB(mongoConfigurationHelper.getMongoDBDatabaseName());
            purgeColl = mongoDB.getCollection(PURGE_COLLECTION);

            List<CentralPurgeState> result = new ArrayList<>();
            List<Integer> statusesValues = new LinkedList<>();
            for (CentralPurgeStatus status : statuses) {
                statusesValues.add(status.getValue());
            }

            // Gets all centralPurgeStates that are in the statuses and pages
            BasicDBObject query = new BasicDBObject(
                    EzCentralPurgeServiceHelpers.CentralPurgeStateString + "."
                            + EzCentralPurgeServiceHelpers.CentralPurgeStatusString,
                    new BasicDBObject("$in", statusesValues));
            DBCursor cursor = purgeColl.find(query).sort(new BasicDBObject(PurgeId, -1))
                    .skip(pageNum > 0 ? ((pageNum - 1) * numPerPage) : 0).limit(numPerPage);

            // Decodes each centralPurgeState from how it's stored in MongoDB and adds it to the return variable
            for (DBObject dbObject : cursor) {
                CentralPurgeState centralPurgeState = decodeCentralPurgeState(
                        (DBObject) dbObject.get(CentralPurgeStateString));
                result.add(centralPurgeState);
            }
            CentralPurgeQueryResults centralPurgeQueryResults = new CentralPurgeQueryResults();
            centralPurgeQueryResults.setPurgeStates(result);
            centralPurgeQueryResults.setCount(purgeColl.count(query));
            return centralPurgeQueryResults;
        } catch (EzSecurityTokenException e) {
            logError(e, evt, "CentralPurgeService failed when trying to validate token:[" + e.getClass().getName()
                    + ":" + e.getMessage() + "]");
            throw e;
        } catch (UnknownHostException e) {
            logError(e, evt, "CentralPurgeService unable to reach MongoDB in getPagedSortedFilteredPurgeStates:["
                    + e.getClass().getName() + ":" + e.getMessage() + "]");
            throw new CentralPurgeServiceException(
                    "CentralPurgeService unable to reach MongoDB in getPagedSortedFilteredPurgeStates:["
                            + e.getClass().getName() + ":" + e.getMessage() + "]");
        } catch (Exception e) {
            logError(e, evt, "CentralPurgeService encountered an exception in getPagedSortedFilteredPurgeStates:["
                    + e.getClass().getName() + ":" + e.getMessage() + "]");
            throw new CentralPurgeServiceException(
                    "CentralPurgeService encountered an exception in getPagedSortedFilteredPurgeStates:["
                            + e.getClass().getName() + ":" + e.getMessage() + "]");
        } finally {
            auditLogger.logEvent(evt);
            logEventToPlainLogs(logger, evt);
            if (mongoClient != null)
                mongoClient.close();
        }
    }

    @Override
    public CentralAgeOffEventQueryResults getPagedSortedFilteredAgeOffEventStates(EzSecurityToken token,
            List<CentralPurgeStatus> statuses, int pageNum, int numPerPage)
            throws EzSecurityTokenException, CentralPurgeServiceException {
        DBCollection ageOffColl = null;
        Mongo mongoClient = null;
        AuditEvent evt = event(AuditEventType.FileObjectAccess.getName(), token)
                .arg("event", "getPagedSortedFilteredAgeOffEventStates").arg("statuses", statuses)
                .arg("pageNum", pageNum).arg("numPerPage", numPerPage);
        try {
            validateCentralPurgeSecurityToken(token);

            // Get access to the ageoff collection within Mongo

            MongoConfigurationHelper mongoConfigurationHelper = new MongoConfigurationHelper(configuration);
            MongoHelper mongoHelper = new MongoHelper(configuration);
            mongoClient = mongoHelper.getMongo();
            DB mongoDB = mongoClient.getDB(mongoConfigurationHelper.getMongoDBDatabaseName());
            ageOffColl = mongoDB.getCollection(AGEOFF_COLLECTION);

            List<CentralAgeOffEventState> result = new ArrayList<>();
            List<Integer> statusesValues = new LinkedList<>();
            for (CentralPurgeStatus status : statuses) {
                statusesValues.add(status.getValue());
            }

            // Gets all centralPurgeStates that are in the statuses and pages
            BasicDBObject query = new BasicDBObject(
                    EzCentralPurgeServiceHelpers.CentralAgeOffStateString + "."
                            + EzCentralPurgeServiceHelpers.CentralPurgeStatusString,
                    new BasicDBObject("$in", statusesValues));
            DBCursor cursor = ageOffColl.find(query).sort(new BasicDBObject(AgeOffEventId, -1))
                    .skip(pageNum > 0 ? ((pageNum - 1) * numPerPage) : 0).limit(numPerPage);

            for (DBObject dbObject : cursor) {
                CentralAgeOffEventState centralAgeOffEventState = decodeCentralAgeOffEventState(
                        (DBObject) dbObject.get(CentralAgeOffStateString));
                result.add(centralAgeOffEventState);
            }
            CentralAgeOffEventQueryResults centralAgeOffEventQueryResults = new CentralAgeOffEventQueryResults();
            centralAgeOffEventQueryResults.setAgeOffEventStates(result);
            centralAgeOffEventQueryResults.setCount(ageOffColl.count(query));
            return centralAgeOffEventQueryResults;
        } catch (EzSecurityTokenException e) {
            logError(e, evt, "CentralPurgeService failed when trying to validate token:[" + e.getClass().getName()
                    + ":" + e.getMessage() + "]");
            throw e;
        } catch (UnknownHostException e) {
            logError(e, evt,
                    "CentralPurgeService unable to reach MongoDB in getPagedSortedFilteredAgeOffEventStates:["
                            + e.getClass().getName() + ":" + e.getMessage() + "]");
            throw new CentralPurgeServiceException(
                    "CentralPurgeService unable to reach MongoDB in getPagedSortedFilteredAgeOffEventStates:["
                            + e.getClass().getName() + ":" + e.getMessage() + "]");
        } catch (Exception e) {
            logError(e, evt,
                    "CentralPurgeService encountered an exception in getPagedSortedFilteredAgeOffEventStates:["
                            + e.getClass().getName() + ":" + e.getMessage() + "]");
            throw new CentralPurgeServiceException(
                    "CentralPurgeService encountered an exception in getPagedSortedFilteredAgeOffEventStates:["
                            + e.getClass().getName() + ":" + e.getMessage() + "]");
        } finally {
            auditLogger.logEvent(evt);
            logEventToPlainLogs(logger, evt);
            if (mongoClient != null)
                mongoClient.close();
        }
    }

    // This method is used by resolvePurge and resolveAgeOff event to  cancel the services still running that purge
    private Map<String, ApplicationPurgeState> cancelServices(EzSecurityToken token,
            Map<String, ApplicationPurgeState> appMap, long id, AuditEvent evt) throws EzSecurityTokenException {

        for (String appName : appMap.keySet()) {
            try {
                ApplicationPurgeState applicationPurgeState = appMap.get(appName);
                Map<String, ServicePurgeState> servicePurgeStateMap = applicationPurgeState.getServicePurgestates();

                for (String serviceName : servicePurgeStateMap.keySet()) {
                    EzBakeBasePurgeService.Client individualServicePurgeClient = null;
                    ThriftClientPool pool = null;
                    try {
                        pool = new ThriftClientPool(configuration);
                        ServicePurgeState servicePurgeState = servicePurgeStateMap.get(serviceName);
                        PurgeStatus status = servicePurgeState.getPurgeState().getPurgeStatus();

                        if (serviceStillRunning(status)) {
                            EzSecurityToken centralTokenForServices = null;
                            centralTokenForServices = securityClient.fetchDerivedTokenForApp(token,
                                    pool.getSecurityId(getSecurityName(appName, serviceName)));

                            if (centralTokenForServices == null) {
                                throw new PurgeException(
                                        "Failed when getting a token targeting " + appName + "_" + serviceName);
                            }
                            individualServicePurgeClient = getServicePurgeThriftClient(appName, serviceName, pool);
                            servicePurgeState.setPurgeState(
                                    individualServicePurgeClient.cancelPurge(centralTokenForServices, id));

                            servicePurgeStateMap.put(serviceName, servicePurgeState);
                        }
                    } catch (Exception e) {
                        logError(e, evt, "CentralPurgeService failed when trying to cancel:" + appName + "_"
                                + serviceName + " [" + e.getClass().getName() + ":" + e.getMessage() + "]");
                        e.printStackTrace();
                    } finally {
                        if (pool != null)
                            pool.close();
                        if (individualServicePurgeClient != null) {
                            returnClientToPool(individualServicePurgeClient, pool);
                        }
                    }
                }
                appMap.put(appName, applicationPurgeState);
            } catch (Exception e) {
                logError(e, evt, "CentralPurgeService failed when trying to cancel:" + appName + " ["
                        + e.getClass().getName() + ":" + e.getMessage() + "]");
                e.printStackTrace();
            }
        }
        return appMap;
    }

    // This event is run when an ageOffEvent or Purge is started. It starts the purge on all services
    private Map<String, ApplicationPurgeState> servicePurger(EzSecurityToken token, Long id, Set<Long> purgeSet,
            Map<String, ApplicationPurgeState> appMap, boolean synchronous, CentralPurgeType centralPurgeType,
            AuditEvent evt) throws Exception {
        EzBakeBasePurgeService.Client individualServicePurgeClient = null;

        try {
            // Get a map of every application to it's respective service
            ServicePurgeClient serviceDiscoveryPurgeClient = new ServicePurgeClient(
                    configuration.getProperty(EzBakePropertyConstants.ZOOKEEPER_CONNECTION_STRING));
            Multimap<String, String> allAppMap = serviceDiscoveryPurgeClient.getPurgeServices();
            Set<String> appNames = allAppMap.keySet();

            // Iterate through each app
            for (String appName : appNames) {
                try {
                    Collection<String> services = allAppMap.get(appName);
                    ApplicationPurgeState applicationPurgeState = new ApplicationPurgeState();
                    Map<String, ServicePurgeState> servicePurgeStateMap = new HashMap<>();
                    // Iterate through the services for that app
                    for (String serviceName : services) {
                        ThriftClientPool pool = null;
                        try {
                            pool = new ThriftClientPool(configuration);
                            EzSecurityToken centralTokenForServices = securityClient.fetchDerivedTokenForApp(token,
                                    pool.getSecurityId(getSecurityName(appName, serviceName)));

                            if (centralTokenForServices == null) {
                                throw new PurgeException(
                                        "Failed when getting a token targeting " + appName + "_" + serviceName);
                            }
                            individualServicePurgeClient = getServicePurgeThriftClient(appName, serviceName, pool);
                            PurgeState purgeState = null;
                            EzSecurityTokenWrapper ezSecurityTokenWrapper = new EzSecurityTokenWrapper(
                                    centralTokenForServices);
                            logger.info("App name passing for derivedToken:" + appName + " (it's securityID:"
                                    + pool.getSecurityId(appName) + ") Token details: target:"
                                    + ezSecurityTokenWrapper.getTargetSecurityId() + " tokenId:"
                                    + ezSecurityTokenWrapper.getSecurityId() + " username"
                                    + ezSecurityTokenWrapper.getUsername());
                            // Call the service for the respective purge type
                            switch (centralPurgeType) {
                            case NORMAL:
                                purgeState = individualServicePurgeClient.beginPurge(EZBAKE_BASE_PURGE_SERVICE_NAME,
                                        id, purgeSet, centralTokenForServices);
                                break;
                            case VIRUS:
                                purgeState = individualServicePurgeClient.beginVirusPurge(
                                        EZBAKE_BASE_PURGE_SERVICE_NAME, id, purgeSet, centralTokenForServices);
                            }
                            DateTime timeStamp = purgeState.getTimeStamp();
                            ServicePurgeState servicePurgeState = new ServicePurgeState();
                            servicePurgeState.setPurgeState(purgeState);
                            servicePurgeState.setTimeInitiated(timeStamp);
                            servicePurgeState.setTimeLastPoll(timeStamp);

                            // Should the thread wait until the service stops running?
                            if (!synchronous) {
                                // Add the service to the queue for updates
                                DelayedServicePurgeState delayedServicePurgeState = new DelayedServicePurgeState(
                                        servicePurgeState, appName, serviceName);
                                servicePollDelayQueue.offer(delayedServicePurgeState);
                                servicePurgeStateMap.put(serviceName, servicePurgeState);
                            } else {
                                // Wait for the service to stop purging, getting updates every suggestPollPeriod seconds
                                int suggestedPollPeriod = purgeState.getSuggestedPollPeriod();
                                while (serviceStillRunning(purgeState.getPurgeStatus())) {
                                    try {
                                        TimeUnit.MILLISECONDS.sleep(suggestedPollPeriod);
                                    } catch (InterruptedException ex) {
                                        Thread.currentThread().interrupt();
                                    }

                                    centralTokenForServices = securityClient.fetchDerivedTokenForApp(token,
                                            pool.getSecurityId(getSecurityName(appName, serviceName)));

                                    if (centralTokenForServices == null) {
                                        throw new PurgeException("Failed when getting a token targeting " + appName
                                                + "_" + serviceName);
                                    }
                                    purgeState = individualServicePurgeClient.purgeStatus(centralTokenForServices,
                                            id);
                                    updatePurge(token, purgeState, appName, serviceName);
                                }
                            }
                        } catch (EzSecurityTokenException e) {
                            logError(e, evt,
                                    "CentralPurgeService failed when trying to get a token to purge " + appName
                                            + "_" + serviceName + ":[" + e.getClass().getName() + ":"
                                            + e.getMessage() + "]");
                        } catch (PurgeException e) {
                            logError(e, evt, "CentralPurgeService failed when trying to purge " + appName + "_"
                                    + serviceName + ":[" + e.getClass().getName() + ":" + e.getMessage() + "]");
                        } catch (TException e) {
                            logError(e, evt, "CentralPurgeService failed when trying to purge " + appName + "_"
                                    + serviceName + ":[" + e.getClass().getName() + ":" + e.getMessage() + "]");
                        } catch (Exception e) {
                            logError(e, evt, "CentralPurgeService failed when trying to purge " + appName + "_"
                                    + serviceName + ":[" + e.getClass().getName() + ":" + e.getMessage() + "]");
                        } finally {
                            if (individualServicePurgeClient != null)
                                returnClientToPool(individualServicePurgeClient, pool);
                            if (pool != null)
                                pool.close();
                        }
                        applicationPurgeState.setServicePurgestates(servicePurgeStateMap);
                    }
                    appMap.put(appName, applicationPurgeState);
                } catch (Exception e) {
                    logError(e, evt, "CentralPurgeService failed when trying to get a token to purge " + appName
                            + ":[" + e.getClass().getName() + ":" + e.getMessage() + "]");
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
            throw new CentralPurgeServiceException(e.getMessage());
        }
        return appMap;
    }

    // A helper method for beginManualAgeOff to add a synchronous argument
    private AgeOffEventInfo executeAgeOff(EzSecurityToken token, long ruleId, boolean synchronous)
            throws TException {

        ProvenanceService.Client client = null;
        AgeOffEventInfo ageOffEventInfo = null;

        AuditEvent evt = event(AuditEventType.FileObjectCreate.getName(), token).arg("ruleId", ruleId)
                .arg("synchronous", synchronous);
        if (synchronous) {
            evt.arg("event", "ran age off event");
        } else {
            evt.arg("event", "starting age off event");
        }
        ThriftClientPool pool = null;
        try {
            pool = new ThriftClientPool(configuration);
            validateCentralPurgeSecurityToken(token);
            EzSecurityToken centralTokenForProvenance = securityClient.fetchDerivedTokenForApp(token,
                    getProvenanceSecurityId(pool));

            // Start the ageOffEvent in the provenance service
            client = getProvenanceThriftClient(pool);
            AgeOffInitiationResult ageOffInitiationResult = client.startAgeOffEvent(centralTokenForProvenance,
                    ruleId, null);
            AgeOffRule ageOffRule = client.getAgeOffRuleById(centralTokenForProvenance, ruleId);

            // Initialize ageOffEvent fields
            ageOffEventInfo = new AgeOffEventInfo();
            Set<Long> completelyPurgedSet = new HashSet<>();
            Set<Long> purgeSet = ageOffInitiationResult.getAgeOffDocumentIds();
            DateTime dateTime = getCurrentDateTime();
            ageOffEventInfo.setTimeCreated(dateTime);
            ageOffEventInfo.setCompletelyPurgedSet(completelyPurgedSet);
            ageOffEventInfo.setId(ageOffInitiationResult.getAgeOffId());
            ageOffEventInfo.setPurgeSet(purgeSet);
            ageOffEventInfo.setDescription("Automatic AgeOffEvent for ruleId:" + ruleId + "; AgeOffRule user:["
                    + ageOffRule.getUser() + "] AgeOffRule name:[" + ageOffRule.getName() + "]");
            evt.arg("ageOffId", ageOffInitiationResult.getAgeOffId());

            // If a user started the ageOff then put user's name into the user field, else use the app name
            if (token.getType() == TokenType.USER) {
                ageOffEventInfo.setUser(token.getTokenPrincipal().getPrincipal());
            } else {
                ageOffEventInfo.setUser(token.getTokenPrincipal().getName());
            }

            // If there are no documents that need to be aged off then the event is resolved
            boolean resolved = purgeSet.isEmpty();
            if (resolved) {
                ageOffEventInfo.setResolved(true);
            } else {
                ageOffEventInfo.setResolved(false);
            }

            // Initialize and set the CentralAgeOffEventState
            CentralAgeOffEventState centralAgeOffEventState = new CentralAgeOffEventState();
            Map<String, ApplicationPurgeState> appStates = initializeApplicationState(
                    ageOffInitiationResult.getAgeOffId());
            centralAgeOffEventState.setApplicationStates(appStates);
            centralAgeOffEventState.setAgeOffEventInfo(ageOffEventInfo);
            centralAgeOffEventState.setAgeOffRuleId(ruleId);
            if (resolved) {
                centralAgeOffEventState.setCentralStatus(CentralPurgeStatus.RESOLVED_AUTOMATICALLY);
            } else {
                centralAgeOffEventState.setCentralStatus(CentralPurgeStatus.ACTIVE);
            }
            updateCentralAgeOffEventState(centralAgeOffEventState, ageOffInitiationResult.getAgeOffId());

            // If the ageOffEvent is already resolved there is no need to run a purge
            if (!resolved) {
                if (synchronous) {
                    // If it is synchronous then run the purge and update the values
                    appStates = servicePurger(token, ageOffEventInfo.getId(), ageOffEventInfo.getPurgeSet(),
                            appStates, synchronous, CentralPurgeType.NORMAL, evt);
                    //centralAgeOffEventState=getCentralAgeOffEventState(ageOffInitiationResult.getAgeOffId());
                    //centralAgeOffEventState.setApplicationStates(appStates);
                    //updateCentralAgeOffEventState(centralAgeOffEventState, ageOffInitiationResult.getAgeOffId());
                } else {
                    // If not start a thread running the AgeOffEventPurger
                    ExecutorService executorService = Executors.newSingleThreadExecutor();
                    executorService.execute(new AgeOffEventPurger(ageOffInitiationResult, token, evt));
                    executorService.shutdown();
                }
            }

        } catch (ProvenanceAgeOffRuleNotFoundException e) {
            logError(e, evt, "CentralPurgeService failed when trying to ageOff" + e.getClass().getName() + ":"
                    + e.getMessage() + "]");
            throw e;
        } catch (EzSecurityTokenException e) {
            logError(e, evt, "CentralPurgeService failed when trying to get a token in executeAgeOff:["
                    + e.getClass().getName() + ":" + e.getMessage() + "]");
            throw e;
        } catch (UnknownHostException e) {
            logError(e, evt, "CentralPurgeService unable to reach MongoDB in beginAgeOff:[" + e.getClass().getName()
                    + ":" + e.getMessage() + "]");
            throw new CentralPurgeServiceException("CentralPurgeService unable to reach MongoDB in beginAgeOff:["
                    + e.getClass().getName() + ":" + e.getMessage() + "]");
        } catch (Exception e) {
            logError(e, evt, "CentralPurgeService encountered an exception in beginAgeOff:["
                    + e.getClass().getName() + ":" + e.getMessage() + "]");
            throw new CentralPurgeServiceException("CentralPurgeService encountered an exception in beginAgeOff:["
                    + e.getClass().getName() + ":" + e.getMessage() + "]");
        } finally {
            if (client != null)
                returnClientToPool(client, pool);
            auditLogger.logEvent(evt);
            logEventToPlainLogs(logger, evt);
        }
        return ageOffEventInfo;
    }

    // Gets a centralAgeOffEventState from mongoDB
    private CentralAgeOffEventState getCentralAgeOffEventState(Long ageOffId) throws UnknownHostException {

        Mongo mongoClient = null;
        DBCollection ageOffColl = null;
        try {
            MongoConfigurationHelper mongoConfigurationHelper = new MongoConfigurationHelper(configuration);
            MongoHelper mongoHelper = new MongoHelper(configuration);
            mongoClient = mongoHelper.getMongo();
            DB mongoDB = mongoClient.getDB(mongoConfigurationHelper.getMongoDBDatabaseName());
            ageOffColl = mongoDB.getCollection(AGEOFF_COLLECTION);

            //Get the centralAgeOffEventState and return it
            BasicDBObject query = new BasicDBObject(EzCentralPurgeServiceHelpers.AgeOffEventId, ageOffId);
            DBCursor cursor = ageOffColl.find(query);
            DBObject dbObject = null;
            if (cursor.hasNext()) {
                dbObject = cursor.next();
            } else {
                return null;
            }
            return decodeCentralAgeOffEventState((DBObject) dbObject.get(CentralAgeOffStateString));

        } finally {
            if (mongoClient != null)
                mongoClient.close();
        }
    }

    // Updates a centralAgeOffEventState in the DB (inserts if doesn't already exist)
    private void updateCentralAgeOffEventState(CentralAgeOffEventState centralAgeOffEventState, Long ageOffId)
            throws UnknownHostException {

        DBCollection ageOffColl = null;
        Mongo mongoClient = null;
        try {
            MongoConfigurationHelper mongoConfigurationHelper = new MongoConfigurationHelper(configuration);
            MongoHelper mongoHelper = new MongoHelper(configuration);
            mongoClient = mongoHelper.getMongo();
            DB mongoDB = mongoClient.getDB(mongoConfigurationHelper.getMongoDBDatabaseName());
            ageOffColl = mongoDB.getCollection(AGEOFF_COLLECTION);

            // Update the state if it exists, insert if not
            BasicDBObject query = new BasicDBObject(EzCentralPurgeServiceHelpers.AgeOffEventId, ageOffId);
            BasicDBObject ageOffEvent = new BasicDBObject().append(AgeOffEventId, ageOffId)
                    .append(CentralAgeOffStateString, encodeCentralAgeOffEventState(centralAgeOffEventState));
            boolean upsert = true;
            boolean multiUpdate = false;
            ageOffColl.update(query, ageOffEvent, upsert, multiUpdate);
        } catch (UnknownHostException e) {
            // TODO: log that couldn't connect to MongoDB
            throw e;
        } finally {
            if (mongoClient != null)
                mongoClient.close();
        }
    }

    private List<Long> getAllPurgeIdsInMongo() throws UnknownHostException {
        List<Long> result = null;
        DBCollection purgeColl = null;
        Mongo mongoClient = null;

        try {
            MongoConfigurationHelper mongoConfigurationHelper = new MongoConfigurationHelper(configuration);
            MongoHelper mongoHelper = new MongoHelper(configuration);
            mongoClient = mongoHelper.getMongo();
            DB mongoDB = mongoClient.getDB(mongoConfigurationHelper.getMongoDBDatabaseName());
            purgeColl = mongoDB.getCollection(PURGE_COLLECTION);

            // Just get all ids from MongoDB
            result = new ArrayList<Long>();
            DBCursor cursor = purgeColl.find();
            while (cursor.hasNext()) {
                result.add((Long) cursor.next().get(EzCentralPurgeServiceHelpers.PurgeId));
            }

            return result;
        } finally {
            if (mongoClient != null) {
                mongoClient.close();
            }
        }
    }

    // Gets a centralPurgeState from mongoDB
    private CentralPurgeState getCentralPurgeState(Long purgeId) throws UnknownHostException {

        DBCollection purgeColl = null;
        Mongo mongoClient = null;
        try {
            MongoConfigurationHelper mongoConfigurationHelper = new MongoConfigurationHelper(configuration);
            MongoHelper mongoHelper = new MongoHelper(configuration);
            mongoClient = mongoHelper.getMongo();
            DB mongoDB = mongoClient.getDB(mongoConfigurationHelper.getMongoDBDatabaseName());
            purgeColl = mongoDB.getCollection(PURGE_COLLECTION);

            //Get the centralPurgeState and return it
            BasicDBObject query = new BasicDBObject(PurgeId, purgeId);
            DBCursor cursor = purgeColl.find(query);
            DBObject dbObject = null;
            if (cursor.hasNext()) {
                dbObject = cursor.next();
            } else {
                return null;
            }
            return decodeCentralPurgeState((DBObject) dbObject.get(CentralPurgeStateString));
        } finally {
            if (mongoClient != null) {
                mongoClient.close();
            }
        }
    }

    // Updates a centralPurgeState in the DB (inserts if doesn't already exist)
    private void updateCentralPurgeState(CentralPurgeState centralPurgeState, Long purgeId)
            throws UnknownHostException {
        DBCollection purgeColl = null;
        Mongo mongoClient = null;
        try {
            MongoConfigurationHelper mongoConfigurationHelper = new MongoConfigurationHelper(configuration);
            MongoHelper mongoHelper = new MongoHelper(configuration);
            mongoClient = mongoHelper.getMongo();
            DB mongoDB = mongoClient.getDB(mongoConfigurationHelper.getMongoDBDatabaseName());
            purgeColl = mongoDB.getCollection(PURGE_COLLECTION);

            // Update the state if it exists, insert if not
            BasicDBObject query = new BasicDBObject(PurgeId, purgeId);
            BasicDBObject purgeStatus = new BasicDBObject()
                    .append(PurgeId, centralPurgeState.getPurgeInfo().getId())
                    .append(CentralPurgeStateString, encodeCentralPurgeState(centralPurgeState));
            boolean upsert = true;
            boolean multiUpdate = false;
            purgeColl.update(query, purgeStatus, upsert, multiUpdate);
        } finally {
            if (mongoClient != null)
                mongoClient.close();
        }
    }

    // Gets a service's client, then gets a status update for the specified purge and returns the purge state
    private PurgeState getServiceClientAndUpdate(PurgeState purgeState, String appName, String serviceName)
            throws TException {
        EzBakeBasePurgeService.Client appPurgeClient = null;
        ThriftClientPool pool = null;
        try {
            pool = new ThriftClientPool(configuration);

            EzSecurityToken centralTokenForServices = securityClient
                    .fetchAppToken(pool.getSecurityId(getSecurityName(appName, serviceName)));

            if (centralTokenForServices == null) {
                throw new PurgeException("Failed when getting a token targeting " + appName + "_" + serviceName);
            }
            appPurgeClient = getServicePurgeThriftClient(appName, serviceName, pool);
            purgeState = appPurgeClient.purgeStatus(centralTokenForServices, purgeState.getPurgeId());
            this.updatePurge(securityClient.fetchAppToken(), purgeState, appName, serviceName);

        } finally {
            if (appPurgeClient != null)
                returnClientToPool(appPurgeClient, pool);
            if (pool != null)
                pool.close();
        }
        return purgeState;
    }

    private void validateCentralPurgeSecurityToken(EzSecurityToken token) throws EzSecurityTokenException {
        securityClient.validateReceivedToken(token);
        EzSecurityTokenWrapper ezSecurityTokenWrapper = new EzSecurityTokenWrapper(token);
        if (!isPurgeAppSecurityId(ezSecurityTokenWrapper)) {
            logger.debug("Could not validate central purge security token:securityId=["
                    + ezSecurityTokenWrapper.getSecurityId() + "] actual=[" + this.purgeAppSecurityId + "]");
            throw new EzSecurityTokenException("Not central purge security token: securityId=["
                    + ezSecurityTokenWrapper.getSecurityId() + "] actual=[" + this.purgeAppSecurityId + "]");
        }
    }

    // This method takes gets the Service purge client for a specified application name and service name
    private EzBakeBasePurgeService.Client getServicePurgeThriftClient(String applicationName, String serviceName,
            ThriftClientPool pool) throws TException {
        if (applicationName.equals(COMMON_APP_NAME))
            return pool.getClient(serviceName, EzBakeBasePurgeService.Client.class);
        return pool.getClient(applicationName, serviceName, EzBakeBasePurgeService.Client.class);
    }

    // This method gets all services currently in the system. Then creates a purgeState for each within a Map.
    private Map<String, ApplicationPurgeState> initializeApplicationState(Long purgeId)
            throws CentralPurgeServiceException {
        ServicePurgeClient servicePurgeClient = null;
        LinkedHashMap<String, ApplicationPurgeState> appStates = null;
        try {
            // The ServicePurgeClient gets all the purge services currently in the system.
            appStates = new LinkedHashMap<String, ApplicationPurgeState>();
            servicePurgeClient = new ServicePurgeClient(
                    configuration.getProperty(EzBakePropertyConstants.ZOOKEEPER_CONNECTION_STRING));
            Multimap<String, String> allAppMap = servicePurgeClient.getPurgeServices();

            for (String app : allAppMap.keySet()) {
                ApplicationPurgeState applicationPurgeState = new ApplicationPurgeState();
                Map<String, ServicePurgeState> servicePurgeStateMap = new HashMap<String, ServicePurgeState>();

                for (String service : allAppMap.get(app)) {
                    // Create a ServicePurgeState for each service
                    ServicePurgeState servicePurgeState = new ServicePurgeState();
                    PurgeState purgeState = new PurgeState();
                    purgeState.setPurgeStatus(PurgeStatus.WAITING_TO_START);
                    purgeState.setCancelStatus(CancelStatus.NOT_CANCELED);
                    Set<Long> purgedSet = new HashSet<>();
                    Set<Long> notPurgedSet = new HashSet<>();
                    DateTime dateTime = getCurrentDateTime();
                    purgeState.setPurged(purgedSet);
                    purgeState.setNotPurged(notPurgedSet);
                    purgeState.setTimeStamp(dateTime);
                    purgeState.setPurgeId(purgeId);
                    servicePurgeState.setPurgeState(purgeState);
                    servicePurgeState.setTimeInitiated(dateTime);
                    servicePurgeState.setTimeLastPoll(dateTime);
                    servicePurgeStateMap.put(service, servicePurgeState);
                }

                // Put the new ServicePurgeState into it's respective applicationPurgeState
                applicationPurgeState.setServicePurgestates(servicePurgeStateMap);
                appStates.put(app, applicationPurgeState);
            }
        } catch (Exception e) {
            throw new CentralPurgeServiceException(e.getMessage());
        } finally {
            if (servicePurgeClient != null) {
                servicePurgeClient.close();
            }
        }
        return appStates;

    }

    // This method just checks if a Service purge status indicates that it is still running
    private boolean serviceStillRunning(PurgeStatus status) {
        return (status == PurgeStatus.WAITING_TO_START || status == PurgeStatus.STARTING
                || status == PurgeStatus.PURGING || status == PurgeStatus.STOPPING);
    }

    private ProvenanceService.Client getProvenanceThriftClient(ThriftClientPool pool) throws TException {
        return pool.getClient(PROVENANCE_SERVICE_NAME, ProvenanceService.Client.class);
    }

    private String getProvenanceSecurityId(ThriftClientPool pool) {
        return pool.getSecurityId(PROVENANCE_SERVICE_NAME);
    }

    private void returnClientToPool(TServiceClient client, ThriftClientPool pool) {
        pool.returnToPool(client);
    }

    public boolean ping() {
        return initialized;
    }

    private String getSecurityName(String applicationName, String serviceName) {
        String securityName;
        if (applicationName.equals(COMMON_APP_NAME)) {
            securityName = serviceName;
        } else {
            securityName = applicationName;
        }

        return securityName;
    }

    /**
     * <p>
     * Answers true if the given security token has an application security id
     * that is equal to the application security id from the purge service. If
     * they are not equivalent then false is returned.
     * </p>
     *
     * @param   ezSecurityTokenWrapper The security token that is checked to determine if
     *          it is from the purge service. Required.
     * @return  True if the token has an application security id that matches
     *          purge service's application security id and false if not.
     */
    private boolean isPurgeAppSecurityId(EzSecurityTokenWrapper ezSecurityTokenWrapper) {
        return ezSecurityTokenWrapper.getSecurityId().equals(this.purgeAppSecurityId);
    }

    private void logError(Exception e, AuditEvent evt, String loggerMessage) {
        evt.failed();
        e.printStackTrace();
        evt.arg(e.getClass().getName(), e);
        logger.error(loggerMessage);
    }
}