com.ikanow.infinit.e.processing.custom.scheduler.CustomScheduleManager.java Source code

Java tutorial

Introduction

Here is the source code for com.ikanow.infinit.e.processing.custom.scheduler.CustomScheduleManager.java

Source

/*******************************************************************************
 * Copyright 2012, The Infinit.e Open Source Project.
 * 
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License, version 3,
 * as published by the Free Software Foundation.
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Affero General Public License for more details.
 * 
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 ******************************************************************************/
package com.ikanow.infinit.e.processing.custom.scheduler;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.bson.types.ObjectId;

import com.ikanow.infinit.e.data_model.api.knowledge.AdvancedQueryPojo;
import com.ikanow.infinit.e.data_model.control.DocumentQueueControlPojo;
import com.ikanow.infinit.e.data_model.store.DbManager;
import com.ikanow.infinit.e.data_model.store.MongoDbManager;
import com.ikanow.infinit.e.data_model.store.custom.mapreduce.CustomMapReduceJobPojo;
import com.ikanow.infinit.e.data_model.store.custom.mapreduce.CustomMapReduceJobPojo.SCHEDULE_FREQUENCY;
import com.ikanow.infinit.e.data_model.store.social.sharing.SharePojo;
import com.ikanow.infinit.e.data_model.store.social.sharing.SharePojo.ShareCommunityPojo;
import com.ikanow.infinit.e.data_model.utils.MongoApplicationLock;
import com.ikanow.infinit.e.processing.custom.utils.AuthUtils;
import com.ikanow.infinit.e.processing.custom.utils.PropertiesManager;
import com.mongodb.BasicDBObject;
import com.mongodb.CommandResult;
import com.mongodb.DBObject;
import com.mongodb.MongoException;
import com.mongodb.WriteConcern;
import com.mongodb.WriteResult;

public class CustomScheduleManager {

    /**
     * Check there are available slots for running
     */
    public static boolean availableSlots(PropertiesManager prop_custom) {
        int nMaxConcurrent = prop_custom.getHadoopMaxConcurrent();
        if (Integer.MAX_VALUE != nMaxConcurrent) {
            BasicDBObject maxQuery = new BasicDBObject(CustomMapReduceJobPojo.jobidS_,
                    new BasicDBObject(DbManager.ne_, null));
            int nCurrRunningJobs = (int) DbManager.getCustom().getLookup().count(maxQuery);
            if (nCurrRunningJobs >= nMaxConcurrent) {
                return false;
            }
        }
        return true;
    }//TESTED

    ////////////////////////////////////////////////////////////////////

    /**
     * Look for jobs that have not started yet but are scheduled for some point in the future
     */
    public static CustomMapReduceJobPojo getJobsToRun(PropertiesManager prop_custom, boolean bLocalMode,
            boolean bHadoopEnabled) {
        try {
            // First off, check the number of running jobs - don't exceed the max
            // (seem to run into memory problems if this isn't limited?)
            if (!availableSlots(prop_custom)) {
                return null;
            }

            BasicDBObject query = new BasicDBObject();
            query.append(CustomMapReduceJobPojo.jobidS_, null);
            query.append(CustomMapReduceJobPojo.waitingOn_, new BasicDBObject(MongoDbManager.size_, 0));
            query.append(CustomMapReduceJobPojo.nextRunTime_,
                    new BasicDBObject(MongoDbManager.lt_, new Date().getTime()));
            if (!bHadoopEnabled && !bLocalMode) {
                // Can only get shared queries:
                query.append("jarURL", null);
            }
            BasicDBObject updates = new BasicDBObject(CustomMapReduceJobPojo.jobidS_, "");
            updates.append("lastRunTime", new Date());
            BasicDBObject update = new BasicDBObject(MongoDbManager.set_, updates);
            DBObject dbo = DbManager.getCustom().getLookup().findAndModify(query, null, null, false, update, true,
                    false);

            if (dbo != null) {
                return CustomMapReduceJobPojo.fromDb(dbo, CustomMapReduceJobPojo.class);
            }
        } catch (Exception ex) {
            //oh noes!
            ex.printStackTrace();
        }

        return null;
    }

    /**
     * Look for running jobs, decide if they are complete
     */
    public static CustomMapReduceJobPojo getJobsToMakeComplete(boolean bHadoopEnabled,
            Map<ObjectId, String> incompleteJobsMap) {
        try {
            BasicDBObject query = new BasicDBObject();
            BasicDBObject nors[] = new BasicDBObject[3];
            nors[0] = new BasicDBObject(CustomMapReduceJobPojo.jobidS_, null);
            nors[1] = new BasicDBObject(CustomMapReduceJobPojo.jobidS_, "CHECKING_COMPLETION");
            nors[2] = new BasicDBObject(CustomMapReduceJobPojo.jobidS_, "");
            query.put(MongoDbManager.nor_, Arrays.asList(nors));
            BasicDBObject updates = new BasicDBObject(CustomMapReduceJobPojo.jobidS_, "CHECKING_COMPLETION");
            updates.put(CustomMapReduceJobPojo.lastChecked_, new Date());
            BasicDBObject update = new BasicDBObject(MongoDbManager.set_, updates);
            if (!bHadoopEnabled) {
                // Can only get shared queries:
                query.append(CustomMapReduceJobPojo.jarURL_, null);
            }
            DBObject dbo = DbManager.getCustom().getLookup().findAndModify(query, update);

            if (dbo != null) {
                CustomMapReduceJobPojo cmr = CustomMapReduceJobPojo.fromDb(dbo, CustomMapReduceJobPojo.class);
                incompleteJobsMap.put(cmr._id, cmr.jobidS);
                return cmr;
            }
        } catch (Exception ex) {
            //oh noes!
            ex.printStackTrace();
        }

        return null;
    }

    /**
     * Uses a map reduce jobs schedule frequency to determine when the next
     * map reduce job should be ran.
     * 
     * @param scheduleFreq
     * @param firstSchedule
     * @param iterations
     * @return
     */
    public static long getNextRunTime(SCHEDULE_FREQUENCY scheduleFreq, Date firstSchedule, long nextRuntime) {
        if ((null == firstSchedule) || (0 == firstSchedule.getTime())) {
            firstSchedule = new Date(nextRuntime);
        } //TESTED

        if (scheduleFreq == null || SCHEDULE_FREQUENCY.NONE == scheduleFreq) {
            return Long.MAX_VALUE;
        }
        Calendar cal = new GregorianCalendar();
        cal.setTime(firstSchedule);

        long now = new Date().getTime();
        while (cal.getTimeInMillis() < now) {
            if (SCHEDULE_FREQUENCY.HOURLY == scheduleFreq) {
                cal.add(Calendar.HOUR, 1);
            } else if (SCHEDULE_FREQUENCY.DAILY == scheduleFreq) {
                cal.add(Calendar.HOUR, 24);
            } else if (SCHEDULE_FREQUENCY.WEEKLY == scheduleFreq) {
                cal.add(Calendar.DATE, 7);
            } else if (SCHEDULE_FREQUENCY.MONTHLY == scheduleFreq) {
                cal.add(Calendar.MONTH, 1);
            }
        }
        return cal.getTimeInMillis();
    }

    ////////////////////////////////////////////////////////////////////
    ////////////////////////////////////////////////////////////////////
    ////////////////////////////////////////////////////////////////////

    // Saved Query Handling

    // Somewhat confusingly there are now 2 different types of saved query:
    // The original one, handled by CustomSavedQueryTaskLauncher
    // - this "re-uses" the CustomMapReducePojo object
    // - stores the entire query result in one object, including aggregations (can easily >16MB and break)
    // - you can store snapshots in append mode
    // - never really found a use
    // The new one, handled via DocumentQueueControlPojos embedded in shares
    // - writes the _ids into the shares, the existing API query is required to get them out

    private static MongoApplicationLock _appLock = null;

    public static void doneWithSavedQueryCache() {
        _appLock.release();
    }

    public static void createOrUpdatedSavedQueryCache() {
        if (null == _appLock) {
            _appLock = MongoApplicationLock.getLock(DbManager.getCustom().getSavedQueryCache().getDB().getName());
        }
        // the built-in applock acquisition code requires more persistent threads so we use the alternate mechanism that
        // just wipes out anything that hasn't been used in the last 2 minutes and then uses the existing contention handling to
        // allow one thread to grab it
        _appLock.clearStaleLocksOnTime(120);
        if (_appLock.acquire(100)) {
            BasicDBObject query = new BasicDBObject(SharePojo.type_, DocumentQueueControlPojo.SavedQueryQueue);
            List<SharePojo> savedQueries = SharePojo.listFromDb(DbManager.getSocial().getShare().find(query),
                    SharePojo.listType());
            if (null != savedQueries) {
                for (SharePojo savedQueryShare : savedQueries) {
                    if (null != savedQueryShare.getShare()) {
                        DocumentQueueControlPojo savedQuery = DocumentQueueControlPojo
                                .fromApi(savedQueryShare.getShare(), DocumentQueueControlPojo.class);

                        // Is this query well formed?
                        if ((null != savedQuery.getQueryInfo()) && ((null != savedQuery.getQueryInfo().getQuery())
                                || (null != savedQuery.getQueryInfo().getQueryId()))) {
                            Date now = new Date();
                            long freqOffset;
                            // Check if it's time to run the query
                            if (savedQuery.getQueryInfo()
                                    .getFrequency() == DocumentQueueControlPojo.SavedQueryInfo.DocQueueFrequency.Hourly) {
                                freqOffset = 3600L * 1000L;
                                //(nothing to do here, just run whenever)
                            } //TESTED (test3)
                            else if (savedQuery.getQueryInfo()
                                    .getFrequency() == DocumentQueueControlPojo.SavedQueryInfo.DocQueueFrequency.Daily) {
                                if (null != savedQuery.getQueryInfo().getFrequencyOffset()) { // hour of day
                                    freqOffset = 12L * 3600L * 1000L; // (already check vs hour of day so be more relaxed) 
                                    Calendar calendar = GregorianCalendar.getInstance();
                                    calendar.setTime(now);

                                    //DEBUG
                                    //System.out.println("DAILY: " + calendar.get(Calendar.HOUR_OF_DAY) + " VS " + savedQuery.getQueryInfo().getFrequencyOffset());   

                                    if (calendar.get(Calendar.HOUR_OF_DAY) != savedQuery.getQueryInfo()
                                            .getFrequencyOffset()) {
                                        continue;
                                    } //TESTED (test4)
                                } else {
                                    freqOffset = 24L * 3600L * 1000L; // (just run every 24 hours)                            
                                }
                            } //TESTED (test4)
                            else if (savedQuery.getQueryInfo()
                                    .getFrequency() == DocumentQueueControlPojo.SavedQueryInfo.DocQueueFrequency.Weekly) {
                                if (null != savedQuery.getQueryInfo().getFrequencyOffset()) { // day of week
                                    freqOffset = 3L * 24L * 3600L * 1000L; // (already check vs day of week so be more relaxed) 
                                    Calendar calendar = GregorianCalendar.getInstance();
                                    calendar.setTime(now);

                                    //DEBUG
                                    //System.out.println("WEEKLY: " + calendar.get(Calendar.DAY_OF_WEEK) + " VS " + savedQuery.getQueryInfo().getFrequencyOffset());

                                    if (calendar.get(Calendar.DAY_OF_WEEK) != savedQuery.getQueryInfo()
                                            .getFrequencyOffset()) {
                                        continue;
                                    }
                                } else {
                                    freqOffset = 7L * 24L * 3600L * 1000L; //(just run every 7 days)
                                }
                            } //TESTED (test5)
                            else
                                continue; // (no -supported- frequency, don't run)

                            long nowTime = now.getTime();

                            //DEBUG
                            //System.out.println("Comparing: " + savedQuery.getQueryInfo().getLastRun() + " VS " + now + " @ " + freqOffset/1000L);

                            if ((null == savedQuery.getQueryInfo().getLastRun()) || ((nowTime
                                    - savedQuery.getQueryInfo().getLastRun().getTime()) > freqOffset)) {
                                try {
                                    //(does nothing if the share already exists)
                                    WriteResult wr = DbManager.getCustom().getSavedQueryCache()
                                            .insert(savedQueryShare.toDb(), WriteConcern.ACKNOWLEDGED);

                                    if (wr.getN() > 0) { // if we've actually done something, update the main share table also
                                        savedQuery.getQueryInfo().setLastRun(now);
                                        savedQueryShare.setShare(savedQuery.toApi());
                                        // (this will overwrite the existing version)
                                        DbManager.getSocial().getShare().save(savedQueryShare.toDb());
                                    } //TESTED (by hand with prints)
                                } catch (MongoException e) {
                                } // just carry on - this is equivalent to getLastERror returning true

                            } //TESTED (test3-5)

                        } // (end saved query actually has a query)
                    }
                } //(end loop over saved queries)
            }
        } //(end acquired app lock)
    }//TESTED

    public static DocumentQueueControlPojo getSavedQueryToRun() {
        DocumentQueueControlPojo toReturn = null;
        try {
            SharePojo savedQueryShare = SharePojo.fromDb(
                    DbManager.getCustom().getSavedQueryCache().findAndRemove(new BasicDBObject()), SharePojo.class);
            if (null == savedQueryShare) { // nothing to process
                return null;
            } //TESTED (test1)
            toReturn = DocumentQueueControlPojo.fromApi(savedQueryShare.getShare(), DocumentQueueControlPojo.class);

            // Get the user communities and append the query if possible
            Set<ObjectId> userAccess = AuthUtils.getCommunities(savedQueryShare.getOwner().get_id());
            if (userAccess.isEmpty()) {
                return toReturn; // (_parentShare is null so will be discarded)
            }
            if (null != toReturn.getQueryInfo().getQueryId()) {
                BasicDBObject queryQuery = new BasicDBObject(SharePojo._id_, toReturn.getQueryInfo().getQueryId());
                queryQuery.put(ShareCommunityPojo.shareQuery_id_, new BasicDBObject(DbManager.in_, userAccess));
                SharePojo shareContainingQuery = SharePojo
                        .fromDb(DbManager.getSocial().getShare().findOne(queryQuery), SharePojo.class);

                if (null == shareContainingQuery) {
                    return toReturn; // (_parentShare is null so will be discarded)
                }
                toReturn.getQueryInfo().setQuery(
                        AdvancedQueryPojo.fromApi(shareContainingQuery.getShare(), AdvancedQueryPojo.class));
            } //TESTED (test6)

            if (null != toReturn.getQueryInfo().getQuery()) {
                // Check the communityIds...
                if (null != toReturn.getQueryInfo().getQuery().communityIds) {
                    ArrayList<ObjectId> revisedCommunityList = new ArrayList<ObjectId>(
                            toReturn.getQueryInfo().getQuery().communityIds.size());
                    for (ObjectId commId : toReturn.getQueryInfo().getQuery().communityIds) {
                        if (userAccess.contains(commId)) {
                            revisedCommunityList.add(commId);
                        }
                    } //(end loop over unchecked communities)
                    toReturn.getQueryInfo().getQuery().communityIds = revisedCommunityList;
                }
            } //TESTED (test1)
            toReturn._parentShare = savedQueryShare; // (if this is null then the subsequent processing will ignore this)
        } catch (Exception e) { // (this is some internal horror so log)
            e.printStackTrace();
        }
        return toReturn;
    }//TESTED (test1,test6)

}