Java tutorial
/** * The contents of this file are subject to the license and copyright * detailed in the LICENSE file at the root of the source * tree and available online at * * https://github.com/keeps/roda */ package org.roda.core.migration; import java.io.IOException; import java.lang.reflect.Modifier; import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Optional; import java.util.Set; import org.apache.commons.lang3.StringUtils; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.request.schema.SchemaRequest; import org.apache.solr.common.util.NamedList; import org.reflections.Reflections; import org.roda.core.RodaCoreFactory; import org.roda.core.common.XMLUtility; import org.roda.core.data.common.RodaConstants; import org.roda.core.data.exceptions.GenericException; import org.roda.core.data.exceptions.RODAException; import org.roda.core.data.utils.JsonUtils; import org.roda.core.data.v2.IsModelObject; import org.roda.core.data.v2.ModelInfo; import org.roda.core.data.v2.common.Pair; import org.roda.core.data.v2.formats.Format; import org.roda.core.data.v2.ip.metadata.PreservationMetadata; import org.roda.core.data.v2.risks.Risk; import org.roda.core.migration.model.FormatToVersion2; import org.roda.core.migration.model.PreservationMetadataFileToVersion2; import org.roda.core.migration.model.RiskToVersion2; import org.roda.core.storage.fs.FSUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class MigrationManager { private static final Logger LOGGER = LoggerFactory.getLogger(MigrationManager.class); private Path modelInfoFile; // map<model class, workflow> private Map<String, MigrationWorkflow> modelMigrations = new HashMap<>(); public MigrationManager(Path dataFolder) { super(); this.modelInfoFile = dataFolder.resolve("model.json"); } // 20161031 hsilva: this method is not invoked in the constructor as it might // get very big & therefore should be done in a lazy fashion public void setupModelMigrations() throws GenericException { addModelMigration(Risk.class, 2, RiskToVersion2.class); addModelMigration(Format.class, 2, FormatToVersion2.class); addModelMigration(PreservationMetadata.class, 2, PreservationMetadataFileToVersion2.class); } private <T extends IsModelObject> void addModelMigration(final Class<T> clazz, final int toVersion, final Class<? extends MigrationAction<T>> migrationClass) throws GenericException { String className = clazz.getName(); MigrationWorkflow classMigrations = modelMigrations.getOrDefault(className, new MigrationWorkflow()); // at the very last I'm updating pointers modelMigrations.put(className, classMigrations); try { MigrationAction<T> migrationAction = migrationClass.newInstance(); if (migrationAction.isToVersionValid(toVersion)) { classMigrations.addMigration(toVersion, migrationClass); } else { LOGGER.error( "Trying to configure migration for model class '{}', setting toVersion to '{}' using action class '{}' but this class says the toVersion is not valid", className, toVersion, migrationClass.getName()); throw new GenericException("Trying to configure migration for model class '" + className + "' with the wrong toVersion"); } } catch (InstantiationException | IllegalAccessException e) { LOGGER.error("Error instantiating migration action class '{}' (which migrates to version {})", migrationClass.getName(), toVersion, e); throw new GenericException("Error instantiating migration action class '" + migrationClass.getName() + "' (which migrates to version '" + toVersion + "')"); } } public boolean isNecessaryToPerformMigration(final SolrClient solrClient, final Optional<Path> tempIndexConfigsPath) throws GenericException { boolean migrationIsNecessary; // check if model migration is necessary migrationIsNecessary = isModelMigrationNecessary(); // check if index migration is necessary migrationIsNecessary = migrationIsNecessary || isIndexMigrationNecessary(solrClient, tempIndexConfigsPath); return migrationIsNecessary; } private boolean isModelMigrationNecessary() throws GenericException { boolean migrationIsNecessary = false; Map<String, Integer> modelClassesVersionsFromCode = getModelClassesVersionsFromCode(true, "Indexed"); Map<String, Integer> modelClassesVersionsInstalled = new HashMap<>(); if (FSUtils.exists(modelInfoFile)) { modelClassesVersionsInstalled = JsonUtils.getObjectFromJson(modelInfoFile, ModelInfo.class) .getInstalledClassesVersions(); } if (modelClassesVersionsInstalled.isEmpty()) { // no information, lets assume first RODA execution LOGGER.info("No model info. available. Writing initial model info. to file {}", modelInfoFile); JsonUtils.writeObjectToFile(new ModelInfo().setInstalledClassesVersions(modelClassesVersionsFromCode), modelInfoFile); } else { // information exists in file, lets see if any migration is needed Map<String, Integer> newModelClassesVersionsToBeWritten = new HashMap<>(); for (Entry<String, Integer> classVersionFromCode : modelClassesVersionsFromCode.entrySet()) { String classFromCode = classVersionFromCode.getKey(); int versionFromCode = classVersionFromCode.getValue(); LOGGER.debug("Checking if model class '{}' requires to do a migration...", classFromCode); // previous information about a class already exists if (modelClassesVersionsInstalled.containsKey(classFromCode)) { int versionInstalled = modelClassesVersionsInstalled.get(classFromCode); if (versionInstalled != versionFromCode) { LOGGER.warn( "A migration may be needed! Model class '{}' version is set to {} in code & installed version is set to {}", classFromCode, versionFromCode, versionInstalled); migrationIsNecessary = true; } } else { // class information does not exists, probably is new & therefore no // model migration is needed LOGGER.info( "No migration is needed as no previous information about model class '{}' exists. Will write its information as it is new!", classFromCode); newModelClassesVersionsToBeWritten.put(classFromCode, versionFromCode); } } // write (just) new classes if (!newModelClassesVersionsToBeWritten.isEmpty()) { modelClassesVersionsInstalled.putAll(newModelClassesVersionsToBeWritten); LOGGER.info("Updating model info. with new classes. to file {}", modelInfoFile); JsonUtils.writeObjectToFile( new ModelInfo().setInstalledClassesVersions(modelClassesVersionsInstalled), modelInfoFile); } if (migrationIsNecessary) { LOGGER.warn( "A migration might be needed. But if you know what you're doing & realize that no migration is needed, write the following content in file '{}':{}{}{}{}", modelInfoFile, System.lineSeparator(), System.lineSeparator(), JsonUtils.getJsonFromObject( new ModelInfo().setInstalledClassesVersions(modelClassesVersionsFromCode)), System.lineSeparator()); } } return migrationIsNecessary; } private Map<String, Integer> getModelClassesVersionsFromCode(final boolean avoidClassesByNamePrefix, final String avoidByNamePrefix) { Map<String, Integer> ret = new HashMap<>(); Reflections reflections = new Reflections("org.roda.core.data.v2"); Set<Class<? extends IsModelObject>> modelClasses = reflections.getSubTypesOf(IsModelObject.class); for (Class<? extends IsModelObject> clazz : modelClasses) { if (Modifier.isAbstract(clazz.getModifiers()) || (avoidClassesByNamePrefix && clazz.getSimpleName().startsWith(avoidByNamePrefix))) { continue; } try { ret.put(clazz.getName(), clazz.newInstance().getClassVersion()); } catch (InstantiationException | IllegalAccessException e) { LOGGER.error("Unable to determine class '{}' model version", clazz.getName(), e); } } return ret; } public void performModelMigrations() throws GenericException { ModelInfo modelInfo = JsonUtils.getObjectFromJson(modelInfoFile, ModelInfo.class); // perform migrations for (Entry<String, MigrationWorkflow> classMigrations : modelMigrations.entrySet()) { String className = classMigrations.getKey(); MigrationWorkflow migrationWorkflow = classMigrations.getValue(); int installedVersion = modelInfo.getInstalledClassesVersions().getOrDefault(className, Integer.MAX_VALUE); // see if there is no need to continue processing this particular class // based on installed version (if any) if (installedVersion >= migrationWorkflow.getLastToVersion()) { continue; } LOGGER.info("Performing migration for class '{}'", className); for (Pair<Integer, Class<? extends MigrationAction>> classMigration : migrationWorkflow .getMigrations()) { Integer toVersion = classMigration.getFirst(); Class<? extends MigrationAction> migrationClass = classMigration.getSecond(); // see if there is no need to perform this particular migration if (installedVersion >= toVersion) { continue; } LOGGER.info("Migrating to version {} using class '{}'", toVersion, migrationClass.getName()); try { // migrate migrationClass.newInstance().migrate(RodaCoreFactory.getStorageService()); LOGGER.info("Migrated with success to version {}", toVersion); // update class specific version after successful migration modelInfo.getInstalledClassesVersions().put(className, toVersion); } catch (InstantiationException | IllegalAccessException e) { LOGGER.error("Error instantiating migration action class '{}' (which migrates to version {})", migrationClass.getName(), toVersion, e); break; } catch (RODAException e) { LOGGER.error("Error executing migration action '{}'. Stopping migrations for class '{}'.", migrationClass.getName(), className); break; } } LOGGER.info("Done migrating class '{}'", className); } // update model info. file JsonUtils.writeObjectToFile(modelInfo, modelInfoFile); } private boolean isIndexMigrationNecessary(SolrClient solrClient, Optional<Path> tempIndexConfigsPath) throws GenericException { boolean migrationIsNecessary = false; if (tempIndexConfigsPath.isPresent()) { Path indexConfigsFolder = tempIndexConfigsPath.get().resolve(RodaConstants.CORE_CONFIG_FOLDER) .resolve(RodaConstants.CORE_INDEX_FOLDER); List<String> solrCollections = getSolrCollections(indexConfigsFolder); Map<String, Integer> indexVersionsFromCode = getIndexVersionsFromCode(indexConfigsFolder, solrCollections); Map<String, Integer> indexVersionsInstalled = getIndexVersionsFromSolr(solrClient, solrCollections); if (indexVersionsFromCode.isEmpty() || indexVersionsInstalled.isEmpty()) { LOGGER.error("Unable to determine if index migration/migrations is/are needed"); throw new GenericException("Unable to determine if index migration/migrations is/are needed"); } else { for (Entry<String, Integer> indexFromCode : indexVersionsFromCode.entrySet()) { String collection = indexFromCode.getKey(); Integer collectionVersionFromCode = indexFromCode.getValue(); if (indexVersionsInstalled.containsKey(collection)) { Integer collectionVersionInstalled = indexVersionsInstalled.get(collection); if (!collectionVersionFromCode.equals(collectionVersionInstalled)) { LOGGER.warn( "A migration is needed! Collection '{}' version is set to {} in code schema.xml & installed version (Solr deployed) is set to {}", collection, collectionVersionFromCode, collectionVersionInstalled); migrationIsNecessary = true; } } else { LOGGER.warn( "A new collection called '{}' exists & needs to be installed before being able to start RODA properly", collection); migrationIsNecessary = true; } } } } else { LOGGER.error("Unable to determine Solr collections via folder with index configs"); throw new GenericException("Unable to determine Solr collections via folder with index configs"); } return migrationIsNecessary; } private List<String> getSolrCollections(Path indexConfigsFolder) { List<String> solrCollections = new ArrayList<>(); try (DirectoryStream<Path> stream = Files.newDirectoryStream(indexConfigsFolder)) { stream.forEach(e -> { if (FSUtils.isDirectory(e)) { solrCollections.add(e.getFileName().toString()); } }); } catch (IOException e) { // do nothing } return solrCollections; } private Map<String, Integer> getIndexVersionsFromCode(Path indexConfigsFolder, List<String> collections) { Map<String, Integer> ret = new HashMap<>(); for (String collection : collections) { Path schemaFile = indexConfigsFolder.resolve(collection).resolve("conf").resolve("schema.xml"); String version = XMLUtility.getStringFromFile(schemaFile, "/schema/@name").replaceFirst(".*-", ""); try { ret.put(collection, Integer.parseInt(version)); } catch (NumberFormatException e) { // do nothing } } return ret; } private Map<String, Integer> getIndexVersionsFromSolr(SolrClient solrClient, List<String> collections) { Map<String, Integer> ret = new HashMap<>(); SolrRequest request = new SchemaRequest.SchemaName(); try { for (String collection : collections) { NamedList<Object> response = solrClient.request(request, collection); for (Entry<String, Object> entry : response) { String value = entry.getValue().toString(); if ("name".equals(entry.getKey()) && StringUtils.isNotBlank(value)) { String version = value.replaceFirst(".*-", ""); try { ret.put(collection, Integer.parseInt(version)); } catch (NumberFormatException e) { // do nothing } break; } } } } catch (SolrServerException | IOException e) { // do nothing } return ret; } private class MigrationWorkflow { private int lastToVersion = Integer.MIN_VALUE; private List<Pair<Integer, Class<? extends MigrationAction>>> migrations = new ArrayList<>(); public void addMigration(int toVersion, Class<? extends MigrationAction> migrationActionClazz) { if (lastToVersion < toVersion) { lastToVersion = toVersion; migrations.add(Pair.of(toVersion, migrationActionClazz)); } else { LOGGER.error( "Error trying to add a migration action class out of order (last toVersion added: {}; toVersion to be added: {})", lastToVersion, toVersion); throw new RuntimeException("Error trying to add a migration action class out of order"); } } public List<Pair<Integer, Class<? extends MigrationAction>>> getMigrations() { return migrations; } public int getLastToVersion() { return lastToVersion; } } }