Java tutorial
/******************************************************************************* * Educational Online Test Delivery System * Copyright (c) 2013 American Institutes for Research * * Distributed under the AIR Open Source License, Version 1.0 * See accompanying file AIR-License-1_0.txt or at * http://www.smarterapp.org/documents/American_Institutes_for_Research_Open_Source_Software_License.pdf ******************************************************************************/ package org.opentestsystem.delivery.testreg.service.impl; import com.google.common.collect.Lists; import org.apache.commons.lang.StringUtils; import org.opentestsystem.delivery.testreg.domain.CacheMap; import org.opentestsystem.delivery.testreg.domain.ClientEntity; import org.opentestsystem.delivery.testreg.domain.FormatType; import org.opentestsystem.delivery.testreg.domain.HierarchyLevel; import org.opentestsystem.delivery.testreg.domain.Sb11Entity; import org.opentestsystem.delivery.testreg.domain.Student; import org.opentestsystem.delivery.testreg.domain.TestRegistrationBase; import org.opentestsystem.delivery.testreg.persistence.ClientEntityRepository; import org.opentestsystem.delivery.testreg.persistence.InstitutionEntityRepository; import org.opentestsystem.delivery.testreg.persistence.StudentRepository; import org.opentestsystem.delivery.testreg.service.CacheMapService; import org.opentestsystem.delivery.testreg.service.Sb11EntityRepositoryService; import org.opentestsystem.delivery.testreg.service.TestRegUberEntityRelationshipService; import org.opentestsystem.delivery.testreg.service.TestRegUserDetailsService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; import org.springframework.cache.Cache; import org.springframework.cache.CacheManager; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; import javax.annotation.PostConstruct; import javax.annotation.Resource; import java.beans.Introspector; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; /** * This class contains all the logic to hold all the Sb11Entity hierarchy in memory using spring and ehcache. It exposes * methods to build, get, and evict this cached map. */ @Service public class TestRegUberEntityRelationshipServiceImpl implements TestRegUberEntityRelationshipService { private static final Logger LOGGER = LoggerFactory.getLogger(TestRegUberEntityRelationshipService.class); private static final String CACHE_NAME = "UberEntityRelationshipMap"; @Autowired private CacheMapService cacheMapService; @Autowired private TestRegPersisterImpl testRegPersister; @Autowired private Sb11EntityRepositoryService sb11EntityService; @Qualifier("userDetailService") @Autowired private TestRegUserDetailsService testRegUserDetailsService; @Resource private CacheManager cacheManager; @Autowired private InstitutionEntityRepository institutionRepository; @Autowired private StudentRepository studentRepository; @Autowired private ClientEntityRepository clientRepository; @Value("${student.startup.validation:false}") private boolean enableStudentStartupValidation; @Override public Map<String, Set<String>> getUberEntityRelationshipMap() { return getUberEntityRelationshipMap(false); } @Override @Async public Map<String, Set<String>> buildUberEntityRelationshipMapAsync(final boolean evictAndRebuild) { // kick off an async method to build the map and cache it return getUberEntityRelationshipMap(evictAndRebuild); } @Override @PostConstruct public void initialize() { // kick off the building of the cache on startup buildUberEntityRelationshipMapAsync(false); loadInstitutionNcesIdCache(); } @Override public void updateUberEntityRelationshipMap(final Sb11Entity entity) { final CacheMap cacheMap = this.cacheMapService.getCacheMap(CACHE_NAME); if (cacheMap == null) { // can't do it, it was never loaded yet... return; } final Map<String, Set<String>> map = cacheMap.getMap(); if (!map.containsKey(entity.getId())) { map.put(entity.getId(), new HashSet<String>()); } if (entity.getParentId() != null && map.containsKey(entity.getParentId())) { map.get(entity.getParentId()).add(entity.getId()); } saveCacheMap(map); } @Override public void updateUberEntityRelationshipMapDeletion(final Sb11Entity entity) { if (entity != null) { updateUberEntityRelationshipMapDeletion(entity.getId()); } } @Override public void updateUberEntityRelationshipMapDeletion(final String entityMongoId) { final CacheMap cacheMap = this.cacheMapService.getCacheMap(CACHE_NAME); if (cacheMap == null) { // can't do it, it was never loaded yet... return; } final Map<String, Set<String>> map = cacheMap.getMap(); map.remove(entityMongoId); for (final Set<String> children : map.values()) { children.remove(entityMongoId); } } // ============================================================================ private Map<String, Set<String>> getUberEntityRelationshipMap(final boolean evictAndRebuild) { if (!evictAndRebuild) { final CacheMap cacheMap = this.cacheMapService.getCacheMap(CACHE_NAME); if (cacheMap != null) { return cacheMap.getMap(); } } LOGGER.debug((evictAndRebuild ? "***************** evict request; rebuilding the " : "*********************** (cache miss) starting to build ") + CACHE_NAME + "... *******************"); final long start = System.currentTimeMillis(); LOGGER.debug( "starting to build all Sb11Entity paths from hierarchy tree, from bottom upwards (i.e. INSTITUTION up to CLIENT...)"); final Map<String, Set<String>> childLookup = buildChildLookup(buildPathwaysFromBottomUp(false, true)); final long dur = System.currentTimeMillis() - start; LOGGER.debug( "done building all Sb11Entity paths from hierarchy tree, from bottom upwards (i.e. INSTITUTION up to CLIENT). Duration: " + dur + "ms"); LOGGER.debug("done building the " + CACHE_NAME + ". Duration: " + dur + "ms"); LOGGER.debug(CACHE_NAME + " has " + childLookup.keySet().size() + " keys."); if (!enableStudentStartupValidation) { LOGGER.info("Student startup validation disabled. Skipping."); } else { LOGGER.info("Student startup validation enabled. Starting..."); logHierarchies(buildPathwaysFromBottomUp(true, false)); // check all student errors for (final Student student : this.studentRepository.findAll()) { final List<String> errorMessages = new ArrayList<>(); final String stateAbbreviation = student.getStateAbbreviation(); final String institutionMongoId = student.getInstitutionEntityMongoId(); final String institutionEntityId = student.getInstitutionIdentifier(); final String districtMongoId = student.getDistrictEntityMongoId(); final String districtEntityId = student.getDistrictIdentifier(); // check the associated institution if (StringUtils.isBlank(institutionMongoId)) { errorMessages.add("missing institution: missing institutionMongoId"); } else if (!childLookup.containsKey(institutionMongoId)) { errorMessages.add("invalid institution: " + institutionMongoId); } if (StringUtils.isBlank(institutionEntityId)) { errorMessages.add("invalid institution: missing institutionEntityId"); } else { final Sb11Entity institutionEntity = this.sb11EntityService.findByEntityIdAndStateAbbreviation( institutionEntityId, stateAbbreviation, HierarchyLevel.INSTITUTION.getEntityClass()); if (institutionEntity == null) { errorMessages.add("invalid institution: EntityId: " + institutionEntityId + ", State: " + stateAbbreviation); } else if (!institutionEntity.getId().equals(institutionMongoId)) { errorMessages.add("institution discrepancy: Explicit: " + institutionMongoId + ", Implicit: " + institutionEntity.getId()); } } // check the district (if one is associated) if (districtMongoId != null && districtMongoId.length() > 0 && !childLookup.containsKey(districtMongoId)) { errorMessages.add("invalid district: " + districtMongoId); } if (districtEntityId != null && districtEntityId.length() > 0) { final Sb11Entity districtEntity = this.sb11EntityService.findByEntityIdAndStateAbbreviation( districtEntityId, stateAbbreviation, HierarchyLevel.DISTRICT.getEntityClass()); if (districtEntity == null) { errorMessages.add("invalid district: EntityId: " + districtEntityId + ", State: " + stateAbbreviation); } else if (!districtEntity.getId().equals(districtMongoId)) { errorMessages.add("district discrepancy: Explicit: " + districtMongoId + ", Implicit: " + districtEntity.getId()); } } if (errorMessages.size() > 0) { String combinedErrorMessage = "student error (invalid/corrupt): student (" + student.getId() + ")"; for (final String errorMessage : errorMessages) { combinedErrorMessage += "\n student error --> " + errorMessage; } LOGGER.warn(combinedErrorMessage); } } LOGGER.info("Student startup validation complete."); } saveCacheMap(childLookup); return childLookup; } private void saveCacheMap(final Map<String, Set<String>> childLookup) { if (childLookup != null && childLookup.size() > 0) { this.cacheMapService.saveCacheMap(new CacheMap(CACHE_NAME, childLookup), true); loadInstitutionNcesIdCache(); // reload institution ncesid cache LOGGER.debug( "***************** cache evict request, evicting user level entity to role cache ***********"); this.testRegUserDetailsService.evictRoleToEntityCache(); this.testRegUserDetailsService.evictAccessibleUsersCache(); } } // create all the pathways (as lists) from bottom up in the entire tree private List<List<String>> buildPathwaysFromBottomUp(final boolean includeEntityTypeWithId, final boolean logEntityErrors) { final long start = System.currentTimeMillis(); LOGGER.debug("running startup buildPathwaysFromBottomUp..."); final List<TestRegUberEntityBottomUpPathwayBuilder> testRegUberEntityBottomUpPathwayBuilders = new ArrayList<>(); final List<List<String>> pathwaysFromBottomUp = new ArrayList<>(); for (final FormatType sb11EntityFormatType : Lists.reverse(getOrderedSb11EntityFormatTypes())) { for (final TestRegistrationBase testRegistrationBase : this.testRegPersister .findAll(sb11EntityFormatType)) { LOGGER.trace( "creating a testRegUberEntityBottomUpPathwayBuilder (thread) to handle building a pathway: " + System.currentTimeMillis()); final TestRegUberEntityBottomUpPathwayBuilder testRegUberEntityBottomUpPathwayBuilder = new TestRegUberEntityBottomUpPathwayBuilder( (Sb11Entity) testRegistrationBase, includeEntityTypeWithId, this.testRegPersister, this.sb11EntityService); testRegUberEntityBottomUpPathwayBuilders.add(testRegUberEntityBottomUpPathwayBuilder); testRegUberEntityBottomUpPathwayBuilder.run(); } } if (logEntityErrors) { // collecting errors LOGGER.debug("collecting errors from testRegUberEntityBottomUpPathwayBuilder (threads)..."); final Map<String, String> errorMessageMap = new HashMap<>(); for (final TestRegUberEntityBottomUpPathwayBuilder testRegUberEntityBottomUpPathwayBuilder : testRegUberEntityBottomUpPathwayBuilders) { for (final Map.Entry<String, String> entry : testRegUberEntityBottomUpPathwayBuilder .getErrorMessageMap().entrySet()) { errorMessageMap.put(entry.getKey(), entry.getValue()); } } // loop over errors and log them... for (final String entityError : errorMessageMap.values()) { LOGGER.warn(entityError); } } // ================================================================================================== // log proposed mongo commands to fix bad entity data // ================================================================================================== final boolean fixBadData = true; // there actually may not be a client yet final List<ClientEntity> clientEntities = this.clientRepository.findAll(); if (!CollectionUtils.isEmpty(clientEntities) && fixBadData) { final Map<String, Sb11Entity> deleteTheseEntities = new HashMap<>(); final Map<String, Sb11Entity> resetParentOnTheseEntities = new HashMap<>(); LOGGER.debug( "collecting invalid entity information results from testRegUberEntityBottomUpPathwayBuilder (threads)..."); for (final TestRegUberEntityBottomUpPathwayBuilder testRegUberEntityBottomUpPathwayBuilder : testRegUberEntityBottomUpPathwayBuilders) { deleteTheseEntities.putAll(testRegUberEntityBottomUpPathwayBuilder.getDeleteTheseEntities()); resetParentOnTheseEntities .putAll(testRegUberEntityBottomUpPathwayBuilder.getResetParentOnTheseEntities()); } final ClientEntity clientEntity = clientEntities.get(0); final StringBuilder sb = new StringBuilder(); // loop thru and construct the update command to set all these entities to the CLIENT for (final Map.Entry<String, Sb11Entity> entry : resetParentOnTheseEntities.entrySet()) { if (!deleteTheseEntities.containsKey(entry.getKey())) { final Sb11Entity entity = entry.getValue(); final String collectionName = Introspector .decapitalize(entity.getEntityType().getEntityClass().getSimpleName()); sb.append("db.").append(collectionName).append(".update(").append("\n"); sb.append(" { _id: { $in: [ObjectId('" + entity.getId() + "'), ]}},").append("\n"); sb.append(" { $set: {").append("\n"); sb.append(" parentId: '" + clientEntity.getId() + "',").append("\n"); sb.append(" parentEntityId: '" + clientEntity.getEntityId() + "',").append("\n"); sb.append(" parentEntityType: '" + HierarchyLevel.CLIENT.name() + "',").append("\n"); sb.append(" }},").append("\n"); sb.append(" { upsert: false, multi: true }").append("\n"); sb.append(")").append("\n"); } } // loop thru and construct the remove command for these invalid entity objects for (final Map.Entry<String, Sb11Entity> entry : deleteTheseEntities.entrySet()) { final Sb11Entity entity = entry.getValue(); final String collectionName = Introspector .decapitalize(entity.getEntityType().getEntityClass().getSimpleName()); sb.append("db.").append(collectionName).append(".remove(").append("\n"); sb.append(" { _id: { $in: [ObjectId('" + entity.getId() + "'), ]}}").append("\n"); sb.append(")").append("\n"); } // ensure we have them also clear their persisted cache map if (sb.length() > 0) { sb.append("db.cacheMap.remove()").append("\n"); LOGGER.error( "FIX BAD DATA: use these mongo commands to update (and possibly delete) invalid/corrupt entity objects:\n" + sb.toString()); } } // ================================================================================================== // ================================================================================================== // ================================================================================================== // collect results LOGGER.debug("collecting results from testRegUberEntityBottomUpPathwayBuilder (threads)..."); for (final TestRegUberEntityBottomUpPathwayBuilder testRegUberEntityBottomUpPathwayBuilder : testRegUberEntityBottomUpPathwayBuilders) { pathwaysFromBottomUp.add(testRegUberEntityBottomUpPathwayBuilder.getPathway()); } LOGGER.debug( "completed startup buildPathwaysFromBottomUp: took " + (System.currentTimeMillis() - start) + "ms"); return pathwaysFromBottomUp; } // create a map with parent as key, and value being a HashSet of all child nodes (no order) private Map<String, Set<String>> buildChildLookup(final List<List<String>> pathwaysFromBottomUp) { final long start = System.currentTimeMillis(); LOGGER.debug( "starting to build an exhaustive lookup table to so we can quickly determine if one Sb11Entity is a parent to another..."); final Map<String, Set<String>> childLookup = new HashMap<>(); for (final List<String> pathwayFromBottomUp : pathwaysFromBottomUp) { final List<String> pathwayFromTopDown = Lists.reverse(pathwayFromBottomUp); for (int i = 0; i < pathwayFromTopDown.size(); i++) { if (!childLookup.containsKey(pathwayFromTopDown.get(i))) { childLookup.put(pathwayFromTopDown.get(i), new HashSet<String>()); } childLookup.get(pathwayFromTopDown.get(i)) .addAll(pathwayFromTopDown.subList(i + 1, pathwayFromTopDown.size())); } } final long dur = System.currentTimeMillis() - start; LOGGER.debug( "done building our exhaustive lookup table so we can quickly lookup if one Sb11Entity is a parent to another. Duration: " + dur + "ms"); return childLookup; } // -------------------------------------------------------------------- // -------------------------------------------------------------------- // for debug purposes only: print out the hierarchies to log // -------------------------------------------------------------------- // -------------------------------------------------------------------- private void logHierarchies(final List<List<String>> pathwaysFromBottomUp) { final List<String> pathways = new ArrayList<>(); for (final List<String> pathwayFromBottomUp : pathwaysFromBottomUp) { String path = ""; int levelsFound = 0; for (final FormatType formatType : getOrderedSb11EntityFormatTypes()) { final String formatTypeMarker = "(" + formatType.name() + ")"; boolean entityTypeFound = false; final List<String> pathwayFromTopDown = Lists.reverse(pathwayFromBottomUp); for (final String entity : pathwayFromBottomUp) { if (entity.startsWith(formatTypeMarker)) { path += entity; entityTypeFound = true; levelsFound++; break; } } if (!entityTypeFound) { path += formatTypeMarker + " ------------------------"; } path += " > "; if (levelsFound >= pathwayFromTopDown.size()) { break; } } pathways.add(path.substring(0, path.length() - 2)); // paths += path.substring(0, path.length() - 2) + "\n"; } Collections.sort(pathways); String paths = "\n\n"; for (final String pathway : pathways) { paths += pathway + "\n"; } LOGGER.debug(paths + "\n\n"); } private static List<FormatType> getOrderedSb11EntityFormatTypes() { final List<FormatType> formatTypes = new ArrayList<>(); formatTypes.add(FormatType.CLIENT); formatTypes.add(FormatType.GROUPOFSTATES); formatTypes.add(FormatType.STATE); formatTypes.add(FormatType.GROUPOFDISTRICTS); formatTypes.add(FormatType.DISTRICT); formatTypes.add(FormatType.GROUPOFINSTITUTIONS); formatTypes.add(FormatType.INSTITUTION); return formatTypes; } private void loadInstitutionNcesIdCache() { // invalidate cache final Cache ncesidCache = this.cacheManager.getCache("institution.ncesid"); if (ncesidCache != null) { ncesidCache.clear(); } final List<String> ncesids = this.institutionRepository.findAllNcesIds(); final HashSet<String> ncesIdSet = new HashSet<String>(ncesids); ncesidCache.put("ncesids", ncesIdSet); } }