org.opentestsystem.delivery.testreg.service.impl.TestRegUberEntityRelationshipServiceImpl.java Source code

Java tutorial

Introduction

Here is the source code for org.opentestsystem.delivery.testreg.service.impl.TestRegUberEntityRelationshipServiceImpl.java

Source

/*******************************************************************************
 * Educational Online Test Delivery System
 * Copyright (c) 2013 American Institutes for Research
 *
 * Distributed under the AIR Open Source License, Version 1.0
 * See accompanying file AIR-License-1_0.txt or at
 * http://www.smarterapp.org/documents/American_Institutes_for_Research_Open_Source_Software_License.pdf
 ******************************************************************************/
package org.opentestsystem.delivery.testreg.service.impl;

import com.google.common.collect.Lists;
import org.apache.commons.lang.StringUtils;
import org.opentestsystem.delivery.testreg.domain.CacheMap;
import org.opentestsystem.delivery.testreg.domain.ClientEntity;
import org.opentestsystem.delivery.testreg.domain.FormatType;
import org.opentestsystem.delivery.testreg.domain.HierarchyLevel;
import org.opentestsystem.delivery.testreg.domain.Sb11Entity;
import org.opentestsystem.delivery.testreg.domain.Student;
import org.opentestsystem.delivery.testreg.domain.TestRegistrationBase;
import org.opentestsystem.delivery.testreg.persistence.ClientEntityRepository;
import org.opentestsystem.delivery.testreg.persistence.InstitutionEntityRepository;
import org.opentestsystem.delivery.testreg.persistence.StudentRepository;
import org.opentestsystem.delivery.testreg.service.CacheMapService;
import org.opentestsystem.delivery.testreg.service.Sb11EntityRepositoryService;
import org.opentestsystem.delivery.testreg.service.TestRegUberEntityRelationshipService;
import org.opentestsystem.delivery.testreg.service.TestRegUserDetailsService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.cache.Cache;
import org.springframework.cache.CacheManager;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;

import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import java.beans.Introspector;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

/**
 * This class contains all the logic to hold all the Sb11Entity hierarchy in memory using spring and ehcache. It exposes
 * methods to build, get, and evict this cached map.
 */

@Service
public class TestRegUberEntityRelationshipServiceImpl implements TestRegUberEntityRelationshipService {

    private static final Logger LOGGER = LoggerFactory.getLogger(TestRegUberEntityRelationshipService.class);

    private static final String CACHE_NAME = "UberEntityRelationshipMap";

    @Autowired
    private CacheMapService cacheMapService;

    @Autowired
    private TestRegPersisterImpl testRegPersister;

    @Autowired
    private Sb11EntityRepositoryService sb11EntityService;

    @Qualifier("userDetailService")
    @Autowired
    private TestRegUserDetailsService testRegUserDetailsService;

    @Resource
    private CacheManager cacheManager;

    @Autowired
    private InstitutionEntityRepository institutionRepository;

    @Autowired
    private StudentRepository studentRepository;

    @Autowired
    private ClientEntityRepository clientRepository;

    @Value("${student.startup.validation:false}")
    private boolean enableStudentStartupValidation;

    @Override
    public Map<String, Set<String>> getUberEntityRelationshipMap() {
        return getUberEntityRelationshipMap(false);
    }

    @Override
    @Async
    public Map<String, Set<String>> buildUberEntityRelationshipMapAsync(final boolean evictAndRebuild) {
        // kick off an async method to build the map and cache it
        return getUberEntityRelationshipMap(evictAndRebuild);
    }

    @Override
    @PostConstruct
    public void initialize() {
        // kick off the building of the cache on startup
        buildUberEntityRelationshipMapAsync(false);
        loadInstitutionNcesIdCache();
    }

    @Override
    public void updateUberEntityRelationshipMap(final Sb11Entity entity) {
        final CacheMap cacheMap = this.cacheMapService.getCacheMap(CACHE_NAME);
        if (cacheMap == null) {
            // can't do it, it was never loaded yet...
            return;
        }

        final Map<String, Set<String>> map = cacheMap.getMap();
        if (!map.containsKey(entity.getId())) {
            map.put(entity.getId(), new HashSet<String>());
        }
        if (entity.getParentId() != null && map.containsKey(entity.getParentId())) {
            map.get(entity.getParentId()).add(entity.getId());
        }
        saveCacheMap(map);
    }

    @Override
    public void updateUberEntityRelationshipMapDeletion(final Sb11Entity entity) {
        if (entity != null) {
            updateUberEntityRelationshipMapDeletion(entity.getId());
        }
    }

    @Override
    public void updateUberEntityRelationshipMapDeletion(final String entityMongoId) {
        final CacheMap cacheMap = this.cacheMapService.getCacheMap(CACHE_NAME);
        if (cacheMap == null) {
            // can't do it, it was never loaded yet...
            return;
        }
        final Map<String, Set<String>> map = cacheMap.getMap();
        map.remove(entityMongoId);
        for (final Set<String> children : map.values()) {
            children.remove(entityMongoId);
        }

    }

    // ============================================================================

    private Map<String, Set<String>> getUberEntityRelationshipMap(final boolean evictAndRebuild) {
        if (!evictAndRebuild) {
            final CacheMap cacheMap = this.cacheMapService.getCacheMap(CACHE_NAME);
            if (cacheMap != null) {
                return cacheMap.getMap();
            }
        }

        LOGGER.debug((evictAndRebuild ? "***************** evict request; rebuilding the "
                : "*********************** (cache miss) starting to build ") + CACHE_NAME
                + "... *******************");
        final long start = System.currentTimeMillis();
        LOGGER.debug(
                "starting to build all Sb11Entity paths from hierarchy tree, from bottom upwards (i.e. INSTITUTION up to CLIENT...)");
        final Map<String, Set<String>> childLookup = buildChildLookup(buildPathwaysFromBottomUp(false, true));
        final long dur = System.currentTimeMillis() - start;
        LOGGER.debug(
                "done building all Sb11Entity paths from hierarchy tree, from bottom upwards (i.e. INSTITUTION up to CLIENT). Duration: "
                        + dur + "ms");
        LOGGER.debug("done building the " + CACHE_NAME + ". Duration: " + dur + "ms");
        LOGGER.debug(CACHE_NAME + " has " + childLookup.keySet().size() + " keys.");

        if (!enableStudentStartupValidation) {
            LOGGER.info("Student startup validation disabled. Skipping.");
        } else {
            LOGGER.info("Student startup validation enabled. Starting...");

            logHierarchies(buildPathwaysFromBottomUp(true, false));

            // check all student errors
            for (final Student student : this.studentRepository.findAll()) {

                final List<String> errorMessages = new ArrayList<>();
                final String stateAbbreviation = student.getStateAbbreviation();
                final String institutionMongoId = student.getInstitutionEntityMongoId();
                final String institutionEntityId = student.getInstitutionIdentifier();
                final String districtMongoId = student.getDistrictEntityMongoId();
                final String districtEntityId = student.getDistrictIdentifier();

                // check the associated institution
                if (StringUtils.isBlank(institutionMongoId)) {
                    errorMessages.add("missing institution: missing institutionMongoId");
                } else if (!childLookup.containsKey(institutionMongoId)) {
                    errorMessages.add("invalid institution: " + institutionMongoId);
                }

                if (StringUtils.isBlank(institutionEntityId)) {
                    errorMessages.add("invalid institution: missing institutionEntityId");
                } else {
                    final Sb11Entity institutionEntity = this.sb11EntityService.findByEntityIdAndStateAbbreviation(
                            institutionEntityId, stateAbbreviation, HierarchyLevel.INSTITUTION.getEntityClass());
                    if (institutionEntity == null) {
                        errorMessages.add("invalid institution: EntityId: " + institutionEntityId + ", State: "
                                + stateAbbreviation);
                    } else if (!institutionEntity.getId().equals(institutionMongoId)) {
                        errorMessages.add("institution discrepancy: Explicit: " + institutionMongoId
                                + ", Implicit: " + institutionEntity.getId());
                    }
                }

                // check the district (if one is associated)
                if (districtMongoId != null && districtMongoId.length() > 0
                        && !childLookup.containsKey(districtMongoId)) {
                    errorMessages.add("invalid district: " + districtMongoId);
                }

                if (districtEntityId != null && districtEntityId.length() > 0) {
                    final Sb11Entity districtEntity = this.sb11EntityService.findByEntityIdAndStateAbbreviation(
                            districtEntityId, stateAbbreviation, HierarchyLevel.DISTRICT.getEntityClass());
                    if (districtEntity == null) {
                        errorMessages.add("invalid district: EntityId: " + districtEntityId + ", State: "
                                + stateAbbreviation);
                    } else if (!districtEntity.getId().equals(districtMongoId)) {
                        errorMessages.add("district discrepancy: Explicit: " + districtMongoId + ", Implicit: "
                                + districtEntity.getId());
                    }
                }

                if (errorMessages.size() > 0) {
                    String combinedErrorMessage = "student error (invalid/corrupt): student (" + student.getId()
                            + ")";
                    for (final String errorMessage : errorMessages) {
                        combinedErrorMessage += "\n   student error --> " + errorMessage;
                    }
                    LOGGER.warn(combinedErrorMessage);
                }
            }
            LOGGER.info("Student startup validation complete.");
        }
        saveCacheMap(childLookup);
        return childLookup;
    }

    private void saveCacheMap(final Map<String, Set<String>> childLookup) {
        if (childLookup != null && childLookup.size() > 0) {
            this.cacheMapService.saveCacheMap(new CacheMap(CACHE_NAME, childLookup), true);
            loadInstitutionNcesIdCache(); // reload institution ncesid cache
            LOGGER.debug(
                    "***************** cache evict request, evicting user level entity to role cache ***********");
            this.testRegUserDetailsService.evictRoleToEntityCache();
            this.testRegUserDetailsService.evictAccessibleUsersCache();
        }
    }

    // create all the pathways (as lists) from bottom up in the entire tree
    private List<List<String>> buildPathwaysFromBottomUp(final boolean includeEntityTypeWithId,
            final boolean logEntityErrors) {

        final long start = System.currentTimeMillis();
        LOGGER.debug("running startup buildPathwaysFromBottomUp...");
        final List<TestRegUberEntityBottomUpPathwayBuilder> testRegUberEntityBottomUpPathwayBuilders = new ArrayList<>();

        final List<List<String>> pathwaysFromBottomUp = new ArrayList<>();
        for (final FormatType sb11EntityFormatType : Lists.reverse(getOrderedSb11EntityFormatTypes())) {
            for (final TestRegistrationBase testRegistrationBase : this.testRegPersister
                    .findAll(sb11EntityFormatType)) {
                LOGGER.trace(
                        "creating a testRegUberEntityBottomUpPathwayBuilder (thread) to handle building a pathway: "
                                + System.currentTimeMillis());
                final TestRegUberEntityBottomUpPathwayBuilder testRegUberEntityBottomUpPathwayBuilder = new TestRegUberEntityBottomUpPathwayBuilder(
                        (Sb11Entity) testRegistrationBase, includeEntityTypeWithId, this.testRegPersister,
                        this.sb11EntityService);
                testRegUberEntityBottomUpPathwayBuilders.add(testRegUberEntityBottomUpPathwayBuilder);
                testRegUberEntityBottomUpPathwayBuilder.run();
            }
        }

        if (logEntityErrors) {
            // collecting errors
            LOGGER.debug("collecting errors from testRegUberEntityBottomUpPathwayBuilder (threads)...");
            final Map<String, String> errorMessageMap = new HashMap<>();
            for (final TestRegUberEntityBottomUpPathwayBuilder testRegUberEntityBottomUpPathwayBuilder : testRegUberEntityBottomUpPathwayBuilders) {
                for (final Map.Entry<String, String> entry : testRegUberEntityBottomUpPathwayBuilder
                        .getErrorMessageMap().entrySet()) {
                    errorMessageMap.put(entry.getKey(), entry.getValue());
                }
            }
            // loop over errors and log them...
            for (final String entityError : errorMessageMap.values()) {
                LOGGER.warn(entityError);
            }
        }

        // ==================================================================================================
        // log proposed mongo commands to fix bad entity data
        // ==================================================================================================
        final boolean fixBadData = true;
        // there actually may not be a client yet
        final List<ClientEntity> clientEntities = this.clientRepository.findAll();
        if (!CollectionUtils.isEmpty(clientEntities) && fixBadData) {
            final Map<String, Sb11Entity> deleteTheseEntities = new HashMap<>();
            final Map<String, Sb11Entity> resetParentOnTheseEntities = new HashMap<>();
            LOGGER.debug(
                    "collecting invalid entity information results from testRegUberEntityBottomUpPathwayBuilder (threads)...");
            for (final TestRegUberEntityBottomUpPathwayBuilder testRegUberEntityBottomUpPathwayBuilder : testRegUberEntityBottomUpPathwayBuilders) {
                deleteTheseEntities.putAll(testRegUberEntityBottomUpPathwayBuilder.getDeleteTheseEntities());
                resetParentOnTheseEntities
                        .putAll(testRegUberEntityBottomUpPathwayBuilder.getResetParentOnTheseEntities());
            }

            final ClientEntity clientEntity = clientEntities.get(0);
            final StringBuilder sb = new StringBuilder();

            // loop thru and construct the update command to set all these entities to the CLIENT
            for (final Map.Entry<String, Sb11Entity> entry : resetParentOnTheseEntities.entrySet()) {
                if (!deleteTheseEntities.containsKey(entry.getKey())) {
                    final Sb11Entity entity = entry.getValue();
                    final String collectionName = Introspector
                            .decapitalize(entity.getEntityType().getEntityClass().getSimpleName());
                    sb.append("db.").append(collectionName).append(".update(").append("\n");
                    sb.append("  { _id: { $in: [ObjectId('" + entity.getId() + "'), ]}},").append("\n");
                    sb.append("  { $set: {").append("\n");
                    sb.append("    parentId: '" + clientEntity.getId() + "',").append("\n");
                    sb.append("    parentEntityId: '" + clientEntity.getEntityId() + "',").append("\n");
                    sb.append("    parentEntityType: '" + HierarchyLevel.CLIENT.name() + "',").append("\n");
                    sb.append("  }},").append("\n");
                    sb.append("  { upsert: false, multi: true }").append("\n");
                    sb.append(")").append("\n");
                }
            }

            // loop thru and construct the remove command for these invalid entity objects
            for (final Map.Entry<String, Sb11Entity> entry : deleteTheseEntities.entrySet()) {
                final Sb11Entity entity = entry.getValue();
                final String collectionName = Introspector
                        .decapitalize(entity.getEntityType().getEntityClass().getSimpleName());
                sb.append("db.").append(collectionName).append(".remove(").append("\n");
                sb.append("  { _id: { $in: [ObjectId('" + entity.getId() + "'), ]}}").append("\n");
                sb.append(")").append("\n");
            }

            // ensure we have them also clear their persisted cache map
            if (sb.length() > 0) {
                sb.append("db.cacheMap.remove()").append("\n");
                LOGGER.error(
                        "FIX BAD DATA: use these mongo commands to update (and possibly delete) invalid/corrupt entity objects:\n"
                                + sb.toString());
            }
        }
        // ==================================================================================================
        // ==================================================================================================
        // ==================================================================================================

        // collect results
        LOGGER.debug("collecting results from testRegUberEntityBottomUpPathwayBuilder (threads)...");
        for (final TestRegUberEntityBottomUpPathwayBuilder testRegUberEntityBottomUpPathwayBuilder : testRegUberEntityBottomUpPathwayBuilders) {
            pathwaysFromBottomUp.add(testRegUberEntityBottomUpPathwayBuilder.getPathway());
        }

        LOGGER.debug(
                "completed startup buildPathwaysFromBottomUp: took " + (System.currentTimeMillis() - start) + "ms");
        return pathwaysFromBottomUp;
    }

    // create a map with parent as key, and value being a HashSet of all child nodes (no order)
    private Map<String, Set<String>> buildChildLookup(final List<List<String>> pathwaysFromBottomUp) {
        final long start = System.currentTimeMillis();
        LOGGER.debug(
                "starting to build an exhaustive lookup table to so we can quickly determine if one Sb11Entity is a parent to another...");
        final Map<String, Set<String>> childLookup = new HashMap<>();
        for (final List<String> pathwayFromBottomUp : pathwaysFromBottomUp) {
            final List<String> pathwayFromTopDown = Lists.reverse(pathwayFromBottomUp);
            for (int i = 0; i < pathwayFromTopDown.size(); i++) {
                if (!childLookup.containsKey(pathwayFromTopDown.get(i))) {
                    childLookup.put(pathwayFromTopDown.get(i), new HashSet<String>());
                }
                childLookup.get(pathwayFromTopDown.get(i))
                        .addAll(pathwayFromTopDown.subList(i + 1, pathwayFromTopDown.size()));
            }
        }
        final long dur = System.currentTimeMillis() - start;
        LOGGER.debug(
                "done building our exhaustive lookup table so we can quickly lookup if one Sb11Entity is a parent to another. Duration: "
                        + dur + "ms");
        return childLookup;
    }

    // --------------------------------------------------------------------
    // --------------------------------------------------------------------
    // for debug purposes only: print out the hierarchies to log
    // --------------------------------------------------------------------
    // --------------------------------------------------------------------
    private void logHierarchies(final List<List<String>> pathwaysFromBottomUp) {
        final List<String> pathways = new ArrayList<>();
        for (final List<String> pathwayFromBottomUp : pathwaysFromBottomUp) {
            String path = "";
            int levelsFound = 0;
            for (final FormatType formatType : getOrderedSb11EntityFormatTypes()) {
                final String formatTypeMarker = "(" + formatType.name() + ")";
                boolean entityTypeFound = false;
                final List<String> pathwayFromTopDown = Lists.reverse(pathwayFromBottomUp);
                for (final String entity : pathwayFromBottomUp) {
                    if (entity.startsWith(formatTypeMarker)) {
                        path += entity;
                        entityTypeFound = true;
                        levelsFound++;
                        break;
                    }
                }
                if (!entityTypeFound) {
                    path += formatTypeMarker + " ------------------------";
                }
                path += " > ";
                if (levelsFound >= pathwayFromTopDown.size()) {
                    break;
                }

            }
            pathways.add(path.substring(0, path.length() - 2));
            // paths += path.substring(0, path.length() - 2) + "\n";
        }

        Collections.sort(pathways);
        String paths = "\n\n";
        for (final String pathway : pathways) {
            paths += pathway + "\n";
        }
        LOGGER.debug(paths + "\n\n");
    }

    private static List<FormatType> getOrderedSb11EntityFormatTypes() {
        final List<FormatType> formatTypes = new ArrayList<>();
        formatTypes.add(FormatType.CLIENT);
        formatTypes.add(FormatType.GROUPOFSTATES);
        formatTypes.add(FormatType.STATE);
        formatTypes.add(FormatType.GROUPOFDISTRICTS);
        formatTypes.add(FormatType.DISTRICT);
        formatTypes.add(FormatType.GROUPOFINSTITUTIONS);
        formatTypes.add(FormatType.INSTITUTION);
        return formatTypes;
    }

    private void loadInstitutionNcesIdCache() {
        // invalidate cache
        final Cache ncesidCache = this.cacheManager.getCache("institution.ncesid");

        if (ncesidCache != null) {
            ncesidCache.clear();
        }

        final List<String> ncesids = this.institutionRepository.findAllNcesIds();
        final HashSet<String> ncesIdSet = new HashSet<String>(ncesids);
        ncesidCache.put("ncesids", ncesIdSet);
    }
}