org.jumpmind.symmetric.service.impl.PurgeService.java Source code

Java tutorial

Introduction

Here is the source code for org.jumpmind.symmetric.service.impl.PurgeService.java

Source

/**
 * Licensed to JumpMind Inc under one or more contributor
 * license agreements.  See the NOTICE file distributed
 * with this work for additional information regarding
 * copyright ownership.  JumpMind Inc licenses this file
 * to you under the GNU General Public License, version 3.0 (GPLv3)
 * (the "License"); you may not use this file except in compliance
 * with the License.
 *
 * You should have received a copy of the GNU General Public License,
 * version 3.0 (GPLv3) along with this library; if not, see
 * <http://www.gnu.org/licenses/>.
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.jumpmind.symmetric.service.impl;

import java.sql.Timestamp;
import java.sql.Types;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Date;
import java.util.List;

import org.apache.commons.lang.time.DateUtils;
import org.jumpmind.db.sql.ISqlRowMapper;
import org.jumpmind.db.sql.Row;
import org.jumpmind.symmetric.common.ParameterConstants;
import org.jumpmind.symmetric.db.ISymmetricDialect;
import org.jumpmind.symmetric.model.ExtractRequest;
import org.jumpmind.symmetric.model.IncomingBatch;
import org.jumpmind.symmetric.model.OutgoingBatch;
import org.jumpmind.symmetric.model.RegistrationRequest;
import org.jumpmind.symmetric.service.ClusterConstants;
import org.jumpmind.symmetric.service.IClusterService;
import org.jumpmind.symmetric.service.IParameterService;
import org.jumpmind.symmetric.service.IPurgeService;
import org.jumpmind.symmetric.statistic.IStatisticManager;

/**
 * @see IPurgeService
 */
public class PurgeService extends AbstractService implements IPurgeService {

    enum MinMaxDeleteSql {
        DATA, DATA_EVENT, OUTGOING_BATCH, STRANDED_DATA
    };

    private IClusterService clusterService;

    private IStatisticManager statisticManager;

    public PurgeService(IParameterService parameterService, ISymmetricDialect symmetricDialect,
            IClusterService clusterService, IStatisticManager statisticManager) {
        super(parameterService, symmetricDialect);
        this.clusterService = clusterService;
        this.statisticManager = statisticManager;
        setSqlMap(new PurgeServiceSqlMap(symmetricDialect.getPlatform(), createSqlReplacementTokens()));
    }

    public long purgeOutgoing(boolean force) {
        long rowsPurged = 0;
        Calendar retentionCutoff = Calendar.getInstance();
        retentionCutoff.add(Calendar.MINUTE, -parameterService.getInt(ParameterConstants.PURGE_RETENTION_MINUTES));
        rowsPurged += purgeOutgoing(retentionCutoff, force);
        return rowsPurged;
    }

    public long purgeIncoming(boolean force) {
        long rowsPurged = 0;
        Calendar retentionCutoff = Calendar.getInstance();
        retentionCutoff.add(Calendar.MINUTE, -parameterService.getInt(ParameterConstants.PURGE_RETENTION_MINUTES));
        rowsPurged += purgeIncoming(retentionCutoff, force);
        return rowsPurged;
    }

    public long purgeOutgoing(Calendar retentionCutoff, boolean force) {
        long rowsPurged = 0;
        try {
            if (force || clusterService.lock(ClusterConstants.PURGE_OUTGOING)) {
                try {
                    log.info("The outgoing purge process is about to run for data older than {}",
                            SimpleDateFormat.getDateTimeInstance().format(retentionCutoff.getTime()));
                    rowsPurged += purgeStrandedBatches();
                    rowsPurged += purgeDataRows(retentionCutoff);
                    rowsPurged += purgeOutgoingBatch(retentionCutoff);
                    rowsPurged += purgeExtractRequests();
                } finally {
                    if (!force) {
                        clusterService.unlock(ClusterConstants.PURGE_OUTGOING);
                    }
                    log.info("The outgoing purge process has completed");
                }
            } else {
                log.debug("Could not get a lock to run an outgoing purge");
            }
        } catch (Exception ex) {
            log.error("", ex);
        }
        return rowsPurged;
    }

    private long purgeOutgoingBatch(final Calendar time) {
        log.info("Getting range for outgoing batch");
        long[] minMax = queryForMinMax(getSql("selectOutgoingBatchRangeSql"),
                new Object[] { time.getTime(), OutgoingBatch.Status.OK.name() });
        int maxNumOfBatchIdsToPurgeInTx = parameterService.getInt(ParameterConstants.PURGE_MAX_NUMBER_OF_BATCH_IDS);
        int maxNumOfDataEventsToPurgeInTx = parameterService
                .getInt(ParameterConstants.PURGE_MAX_NUMBER_OF_EVENT_BATCH_IDS);
        int dataEventsPurgedCount = purgeByMinMax(minMax, MinMaxDeleteSql.DATA_EVENT, time.getTime(),
                maxNumOfDataEventsToPurgeInTx);
        statisticManager.incrementPurgedDataEventRows(dataEventsPurgedCount);
        int outgoingbatchPurgedCount = purgeByMinMax(minMax, MinMaxDeleteSql.OUTGOING_BATCH, time.getTime(),
                maxNumOfBatchIdsToPurgeInTx);
        statisticManager.incrementPurgedBatchOutgoingRows(outgoingbatchPurgedCount);
        return dataEventsPurgedCount + outgoingbatchPurgedCount;
    }

    private long purgeStrandedBatches() {
        int updateStrandedBatchesCount = sqlTemplate.update(getSql("updateStrandedBatches"),
                OutgoingBatch.Status.OK.name(), 1, OutgoingBatch.Status.OK.name());
        if (updateStrandedBatchesCount > 0) {
            log.info("Set the status to {} for {} batches that no longer are associated with valid nodes",
                    OutgoingBatch.Status.OK.name(), updateStrandedBatchesCount);
            statisticManager.incrementPurgedBatchOutgoingRows(updateStrandedBatchesCount);
        }
        return updateStrandedBatchesCount;
    }

    private long purgeDataRows(final Calendar time) {
        log.info("Getting range for data");
        long[] minMax = queryForMinMax(getSql("selectDataRangeSql"), new Object[0]);
        int maxNumOfDataIdsToPurgeInTx = parameterService.getInt(ParameterConstants.PURGE_MAX_NUMBER_OF_DATA_IDS);
        int dataDeletedCount = purgeByMinMax(minMax, MinMaxDeleteSql.DATA, time.getTime(),
                maxNumOfDataIdsToPurgeInTx);
        statisticManager.incrementPurgedDataRows(dataDeletedCount);
        int strandedDeletedCount = purgeByMinMax(minMax, MinMaxDeleteSql.STRANDED_DATA, time.getTime(),
                maxNumOfDataIdsToPurgeInTx);
        statisticManager.incrementPurgedDataRows(strandedDeletedCount);
        return dataDeletedCount + strandedDeletedCount;

    }

    private long[] queryForMinMax(String sql, Object... params) {
        long[] minMax = sqlTemplate.queryForObject(sql, new ISqlRowMapper<long[]>() {
            public long[] mapRow(Row rs) {
                return new long[] { rs.getLong("min_id"), rs.getLong("max_id") - 1 };
            }
        }, params);
        return minMax;
    }

    private long purgeExtractRequests() {
        Calendar retentionCutoff = Calendar.getInstance();
        retentionCutoff.add(Calendar.MINUTE,
                -parameterService.getInt(ParameterConstants.PURGE_EXTRACT_REQUESTS_RETENTION_MINUTES));
        log.info("Purging extract requests that are older than {}", retentionCutoff.getTime());
        long count = sqlTemplate.update(getSql("deleteExtractRequestSql"), ExtractRequest.ExtractStatus.OK.name(),
                retentionCutoff.getTime());
        if (count > 0) {
            log.info("Purged {} extract requests", count);
        }
        return count;

    }

    private long purgeRegistrationRequests() {
        Calendar retentionCutoff = Calendar.getInstance();
        retentionCutoff.add(Calendar.MINUTE,
                -parameterService.getInt(ParameterConstants.PURGE_REGISTRATION_REQUEST_RETENTION_MINUTES));
        log.info("Purging registration requests that are older than {}", retentionCutoff.getTime());
        long count = sqlTemplate.update(getSql("deleteRegistrationRequestSql"),
                RegistrationRequest.RegistrationStatus.OK.name(), RegistrationRequest.RegistrationStatus.IG.name(),
                RegistrationRequest.RegistrationStatus.RR.name(), retentionCutoff.getTime());
        if (count > 0) {
            log.info("Purged {} registration requests", count);
        }
        return count;
    }

    private int purgeByMinMax(long[] minMax, MinMaxDeleteSql identifier, Date retentionTime,
            int maxNumtoPurgeinTx) {
        long minId = minMax[0];
        long purgeUpToId = minMax[1];
        long ts = System.currentTimeMillis();
        int totalCount = 0;
        int totalDeleteStmts = 0;
        int idSqlType = symmetricDialect.getSqlTypeForIds();
        Timestamp cutoffTime = new Timestamp(retentionTime.getTime());
        log.info("About to purge {}", identifier.toString().toLowerCase());

        while (minId <= purgeUpToId) {
            totalDeleteStmts++;
            long maxId = minId + maxNumtoPurgeinTx;
            if (maxId > purgeUpToId) {
                maxId = purgeUpToId;
            }

            String deleteSql = null;
            Object[] args = null;
            int[] argTypes = null;

            switch (identifier) {
            case DATA:
                deleteSql = getSql("deleteDataSql");
                args = new Object[] { minId, maxId, cutoffTime, minId, maxId, minId, maxId,
                        OutgoingBatch.Status.OK.name() };
                argTypes = new int[] { idSqlType, idSqlType, Types.TIMESTAMP, idSqlType, idSqlType, idSqlType,
                        idSqlType, Types.VARCHAR };
                break;
            case DATA_EVENT:
                deleteSql = getSql("deleteDataEventSql");
                args = new Object[] { minId, maxId, OutgoingBatch.Status.OK.name(), minId, maxId };
                argTypes = new int[] { idSqlType, idSqlType, Types.VARCHAR, idSqlType, idSqlType };

                break;
            case OUTGOING_BATCH:
                deleteSql = getSql("deleteOutgoingBatchSql");
                args = new Object[] { OutgoingBatch.Status.OK.name(), minId, maxId, minId, maxId };
                argTypes = new int[] { Types.VARCHAR, idSqlType, idSqlType, idSqlType, idSqlType };

                break;
            case STRANDED_DATA:
                deleteSql = getSql("deleteStrandedData");
                args = new Object[] { minId, maxId, cutoffTime, minId, maxId };
                argTypes = new int[] { idSqlType, idSqlType, Types.TIMESTAMP, idSqlType, idSqlType };
                break;
            }

            log.debug("Running the following statement: {} with the following arguments: {}", deleteSql,
                    Arrays.toString(args));
            int count = sqlTemplate.update(deleteSql, args, argTypes);
            log.debug("Deleted {} rows", count);
            totalCount += count;

            if (totalCount > 0 && (System.currentTimeMillis() - ts > DateUtils.MILLIS_PER_MINUTE * 5)) {
                log.info("Purged {} of {} rows so far using {} statements",
                        new Object[] { totalCount, identifier.toString().toLowerCase(), totalDeleteStmts });
                ts = System.currentTimeMillis();
            }
            minId = maxId + 1;
        }
        log.info("Done purging {} of {} rows", totalCount, identifier.toString().toLowerCase());
        return totalCount;
    }

    public long purgeIncoming(Calendar retentionCutoff, boolean force) {
        long purgedRowCount = 0;
        try {
            if (force || clusterService.lock(ClusterConstants.PURGE_INCOMING)) {
                try {
                    log.info("The incoming purge process is about to run");
                    purgedRowCount = purgeIncomingBatch(retentionCutoff);
                    purgedRowCount += purgeIncomingError();
                    purgedRowCount += purgeRegistrationRequests();
                } finally {
                    if (!force) {
                        clusterService.unlock(ClusterConstants.PURGE_INCOMING);
                    }
                    log.info("The incoming purge process has completed");
                }

            } else {
                log.debug("Could not get a lock to run an incoming purge");
            }
        } catch (Exception ex) {
            log.error("", ex);
        }
        return purgedRowCount;
    }

    private long purgeIncomingError() {
        log.info("Purging incoming error rows");
        long rowCount = sqlTemplate.update(getSql("deleteIncomingErrorsSql"));
        log.info("Purged {} incoming error rows", rowCount);
        return rowCount;

    }

    private long purgeIncomingBatch(final Calendar time) {
        log.info("Getting range for incoming batch");
        List<NodeBatchRange> nodeBatchRangeList = sqlTemplate.query(getSql("selectIncomingBatchRangeSql"),
                new ISqlRowMapper<NodeBatchRange>() {
                    public NodeBatchRange mapRow(Row rs) {
                        return new NodeBatchRange(rs.getString("node_id"), rs.getLong("min_id"),
                                rs.getLong("max_id"));
                    }
                }, time.getTime(), IncomingBatch.Status.OK.name());
        int incomingBatchesPurgedCount = purgeByNodeBatchRangeList(nodeBatchRangeList);
        statisticManager.incrementPurgedBatchIncomingRows(incomingBatchesPurgedCount);
        return incomingBatchesPurgedCount;
    }

    private int purgeByNodeBatchRangeList(List<NodeBatchRange> nodeBatchRangeList) {
        long ts = System.currentTimeMillis();
        int totalCount = 0;
        int totalDeleteStmts = 0;
        log.info("About to purge incoming batch");

        for (NodeBatchRange nodeBatchRange : nodeBatchRangeList) {
            int maxNumOfDataIdsToPurgeInTx = parameterService
                    .getInt(ParameterConstants.PURGE_MAX_NUMBER_OF_BATCH_IDS);
            long minBatchId = nodeBatchRange.getMinBatchId();
            long purgeUpToBatchId = nodeBatchRange.getMaxBatchId();

            while (minBatchId <= purgeUpToBatchId) {
                totalDeleteStmts++;
                long maxBatchId = minBatchId + maxNumOfDataIdsToPurgeInTx;
                if (maxBatchId > purgeUpToBatchId) {
                    maxBatchId = purgeUpToBatchId;
                }
                totalCount += sqlTemplate.update(getSql("deleteIncomingBatchSql"), new Object[] { minBatchId,
                        maxBatchId, nodeBatchRange.getNodeId(), IncomingBatch.Status.OK.name() });
                minBatchId = maxBatchId + 1;
            }

            if (totalCount > 0 && (System.currentTimeMillis() - ts > DateUtils.MILLIS_PER_MINUTE * 5)) {
                log.info("Purged {} incoming batch rows so far using {} statements",
                        new Object[] { totalCount, totalDeleteStmts });
                ts = System.currentTimeMillis();
            }
        }
        log.info("Done purging {} incoming batch rows", totalCount);
        return totalCount;
    }

    public void purgeStats(boolean force) {
        Calendar retentionCutoff = Calendar.getInstance();
        retentionCutoff.add(Calendar.MINUTE,
                -parameterService.getInt(ParameterConstants.PURGE_STATS_RETENTION_MINUTES));
        if (force || clusterService.lock(ClusterConstants.PURGE_STATISTICS)) {
            try {
                int purgedCount = sqlTemplate.update(getSql("purgeNodeHostChannelStatsSql"),
                        retentionCutoff.getTime());
                purgedCount += sqlTemplate.update(getSql("purgeNodeHostStatsSql"), retentionCutoff.getTime());
                purgedCount += sqlTemplate.update(getSql("purgeNodeHostJobStatsSql"), retentionCutoff.getTime());
                if (purgedCount > 0) {
                    log.debug("{} stats rows were purged", purgedCount);
                }
            } finally {
                if (!force) {
                    clusterService.unlock(ClusterConstants.PURGE_STATISTICS);
                }
            }
        }
    }

    class NodeBatchRange {
        private String nodeId;

        private long minBatchId;

        private long maxBatchId;

        public NodeBatchRange(String nodeId, long minBatchId, long maxBatchId) {
            this.nodeId = nodeId;
            this.minBatchId = minBatchId;
            this.maxBatchId = maxBatchId;
        }

        public String getNodeId() {
            return nodeId;
        }

        public long getMaxBatchId() {
            return maxBatchId;
        }

        public long getMinBatchId() {
            return minBatchId;
        }
    }

    public void purgeAllIncomingEventsForNode(String nodeId) {
        int count = sqlTemplate.update(getSql("deleteIncomingBatchByNodeSql"), new Object[] { nodeId });
        log.info("Purged all {} incoming batch for node {}", count, nodeId);
    }

}