Java tutorial
/** * The contents of this file are subject to the license and copyright * detailed in the LICENSE file at the root of the source * tree and available online at * * https://github.com/keeps/roda */ package org.roda.core.index.utils; import java.io.IOException; import java.io.Reader; import java.io.Serializable; import java.text.ParseException; import java.util.ArrayList; import java.util.Arrays; import java.util.Date; import java.util.EnumMap; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.stream.Collectors; import javax.xml.parsers.ParserConfigurationException; import javax.xml.stream.FactoryConfigurationError; import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLStreamConstants; import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; import javax.xml.xpath.XPathExpressionException; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrQuery.ORDER; import org.apache.solr.client.solrj.SolrQuery.SortClause; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.response.FacetField; import org.apache.solr.client.solrj.response.FacetField.Count; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.SolrInputField; import org.apache.solr.common.params.FacetParams; import org.apache.solr.common.util.DateUtil; import org.apache.solr.handler.loader.XMLLoader; import org.roda.core.common.MetadataFileUtils; import org.roda.core.common.PremisV3Utils; import org.roda.core.common.RodaUtils; import org.roda.core.common.dips.DIPUtils; import org.roda.core.data.common.RodaConstants; import org.roda.core.data.common.RodaConstants.DateGranularity; import org.roda.core.data.exceptions.AuthorizationDeniedException; import org.roda.core.data.exceptions.GenericException; import org.roda.core.data.exceptions.NotFoundException; import org.roda.core.data.exceptions.NotSupportedException; import org.roda.core.data.exceptions.RequestNotValidException; import org.roda.core.data.utils.JsonUtils; import org.roda.core.data.v2.LiteRODAObject; import org.roda.core.data.v2.common.OptionalWithCause; import org.roda.core.data.v2.formats.Format; import org.roda.core.data.v2.index.IndexResult; import org.roda.core.data.v2.index.IndexRunnable; import org.roda.core.data.v2.index.IsIndexed; import org.roda.core.data.v2.index.facet.FacetFieldResult; import org.roda.core.data.v2.index.facet.FacetParameter; import org.roda.core.data.v2.index.facet.FacetParameter.SORT; import org.roda.core.data.v2.index.facet.Facets; import org.roda.core.data.v2.index.facet.SimpleFacetParameter; import org.roda.core.data.v2.index.filter.AndFiltersParameters; import org.roda.core.data.v2.index.filter.BasicSearchFilterParameter; import org.roda.core.data.v2.index.filter.DateIntervalFilterParameter; import org.roda.core.data.v2.index.filter.DateRangeFilterParameter; import org.roda.core.data.v2.index.filter.EmptyKeyFilterParameter; import org.roda.core.data.v2.index.filter.Filter; import org.roda.core.data.v2.index.filter.FilterParameter; import org.roda.core.data.v2.index.filter.FiltersParameters; import org.roda.core.data.v2.index.filter.LongRangeFilterParameter; import org.roda.core.data.v2.index.filter.NotSimpleFilterParameter; import org.roda.core.data.v2.index.filter.OneOfManyFilterParameter; import org.roda.core.data.v2.index.filter.OrFiltersParameters; import org.roda.core.data.v2.index.filter.SimpleFilterParameter; import org.roda.core.data.v2.index.select.SelectedItems; import org.roda.core.data.v2.index.sort.SortParameter; import org.roda.core.data.v2.index.sort.Sorter; import org.roda.core.data.v2.index.sublist.Sublist; import org.roda.core.data.v2.ip.AIP; import org.roda.core.data.v2.ip.AIPLink; import org.roda.core.data.v2.ip.AIPState; import org.roda.core.data.v2.ip.DIP; import org.roda.core.data.v2.ip.DIPFile; import org.roda.core.data.v2.ip.File; import org.roda.core.data.v2.ip.FileLink; import org.roda.core.data.v2.ip.HasPermissionFilters; import org.roda.core.data.v2.ip.IndexedAIP; import org.roda.core.data.v2.ip.IndexedDIP; import org.roda.core.data.v2.ip.IndexedFile; import org.roda.core.data.v2.ip.IndexedRepresentation; import org.roda.core.data.v2.ip.Permissions; import org.roda.core.data.v2.ip.Permissions.PermissionType; import org.roda.core.data.v2.ip.Representation; import org.roda.core.data.v2.ip.RepresentationLink; import org.roda.core.data.v2.ip.StoragePath; import org.roda.core.data.v2.ip.TransferredResource; import org.roda.core.data.v2.ip.metadata.DescriptiveMetadata; import org.roda.core.data.v2.ip.metadata.FileFormat; import org.roda.core.data.v2.ip.metadata.IndexedPreservationAgent; import org.roda.core.data.v2.ip.metadata.IndexedPreservationEvent; import org.roda.core.data.v2.ip.metadata.IndexedPreservationEvent.PreservationMetadataEventClass; import org.roda.core.data.v2.ip.metadata.LinkingIdentifier; import org.roda.core.data.v2.ip.metadata.OtherMetadata; import org.roda.core.data.v2.ip.metadata.PreservationMetadata.PreservationMetadataType; import org.roda.core.data.v2.jobs.IndexedReport; import org.roda.core.data.v2.jobs.Job; import org.roda.core.data.v2.jobs.Job.JOB_STATE; import org.roda.core.data.v2.jobs.JobStats; import org.roda.core.data.v2.jobs.PluginType; import org.roda.core.data.v2.jobs.Report; import org.roda.core.data.v2.jobs.Report.PluginState; import org.roda.core.data.v2.log.LogEntry; import org.roda.core.data.v2.log.LogEntry.LOG_ENTRY_STATE; import org.roda.core.data.v2.log.LogEntryParameter; import org.roda.core.data.v2.notifications.Notification; import org.roda.core.data.v2.risks.IndexedRisk; import org.roda.core.data.v2.risks.Risk; import org.roda.core.data.v2.risks.RiskIncidence; import org.roda.core.data.v2.user.Group; import org.roda.core.data.v2.user.RODAMember; import org.roda.core.data.v2.user.User; import org.roda.core.model.ModelService; import org.roda.core.model.utils.ModelUtils; import org.roda.core.storage.Binary; import org.roda.core.storage.Directory; import org.roda.core.storage.StorageService; import org.roda.core.storage.fs.FSUtils; import org.roda.core.util.IdUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.xml.sax.SAXException; /** * Utilities class related to Apache Solr * * @author Hlder Silva <hsilva@keep.pt> * @author Lus Faria <lfaria@keep.pt> * @author Sbastien Leroux <sleroux@keep.pt> */ public class SolrUtils { private static final Logger LOGGER = LoggerFactory.getLogger(SolrUtils.class); private static final String DEFAULT_QUERY_PARSER_OPERATOR = "AND"; private static final Set<String> NON_REPEATABLE_FIELDS = new HashSet<>(Arrays.asList(RodaConstants.AIP_TITLE, RodaConstants.AIP_LEVEL, RodaConstants.AIP_DATE_INITIAL, RodaConstants.AIP_DATE_FINAL)); private static Map<String, List<String>> liteFieldsForEachClass = new HashMap<>(); /** Private empty constructor */ private SolrUtils() { // do nothing } /* * Search & Retrieval * ____________________________________________________________________________________________________________________ */ public static <T extends IsIndexed> Long count(SolrClient index, Class<T> classToRetrieve, Filter filter) throws GenericException, RequestNotValidException { return find(index, classToRetrieve, filter, null, new Sublist(0, 0), new ArrayList<>()).getTotalCount(); } public static <T extends IsIndexed> Long count(SolrClient index, Class<T> classToRetrieve, Filter filter, User user, boolean justActive) throws GenericException, RequestNotValidException { return find(index, classToRetrieve, filter, null, new Sublist(0, 0), null, user, justActive, new ArrayList<>()).getTotalCount(); } public static <T extends IsIndexed> T retrieve(SolrClient index, Class<T> classToRetrieve, String id, List<String> fieldsToReturn) throws NotFoundException, GenericException { if (id == null) { throw new GenericException("Could not retrieve object from a null id"); } T ret; try { SolrDocument doc = index.getById(getIndexName(classToRetrieve).get(0), id); if (doc != null) { ret = solrDocumentTo(classToRetrieve, doc, fieldsToReturn); } else { throw new NotFoundException("Could not find document " + id); } } catch (SolrServerException | IOException e) { throw new GenericException("Could not retrieve object from index", e); } return ret; } public static <T extends IsIndexed> List<T> retrieve(SolrClient index, Class<T> classToRetrieve, List<String> id, List<String> fieldsToReturn) throws NotFoundException, GenericException { List<T> ret = new ArrayList<>(); try { int block = RodaConstants.DEFAULT_PAGINATION_VALUE; for (int i = 0; i < id.size(); i += block) { List<String> subList = id.subList(i, (i + block <= id.size() ? i + block : id.size())); SolrDocumentList docs = index.getById(getIndexName(classToRetrieve).get(0), subList); for (SolrDocument doc : docs) { ret.add(solrDocumentTo(classToRetrieve, doc, fieldsToReturn)); } } } catch (SolrServerException | IOException e) { throw new GenericException("Could not retrieve object from index", e); } return ret; } public static <T extends IsIndexed> IndexResult<T> find(SolrClient index, Class<T> classToRetrieve, Filter filter, Sorter sorter, Sublist sublist, List<String> fieldsToReturn) throws GenericException, RequestNotValidException { return find(index, classToRetrieve, filter, sorter, sublist, null, fieldsToReturn); } public static <T extends IsIndexed> IndexResult<T> find(SolrClient index, Class<T> classToRetrieve, Filter filter, Sorter sorter, Sublist sublist, Facets facets, List<String> fieldsToReturn) throws GenericException, RequestNotValidException { IndexResult<T> ret; SolrQuery query = new SolrQuery(); query.setParam("q.op", DEFAULT_QUERY_PARSER_OPERATOR); query.setQuery(parseFilter(filter)); query.setSorts(parseSorter(sorter)); query.setStart(sublist.getFirstElementIndex()); query.setRows(sublist.getMaximumElementCount()); if (!fieldsToReturn.isEmpty()) { query.setFields(fieldsToReturn.toArray(new String[fieldsToReturn.size()])); } parseAndConfigureFacets(facets, query); try { QueryResponse response = index.query(getIndexName(classToRetrieve).get(0), query); ret = queryResponseToIndexResult(response, classToRetrieve, facets, fieldsToReturn); } catch (SolrServerException | IOException e) { throw new GenericException("Could not query index", e); } catch (SolrException e) { throw new RequestNotValidException(e); } catch (RuntimeException e) { throw new GenericException("Unexpected exception while querying index", e); } return ret; } public static <T extends IsIndexed> List<String> getClassLiteFields(Class<T> classToRetrieve) { List<String> ret; if (liteFieldsForEachClass.containsKey(classToRetrieve.getName())) { ret = liteFieldsForEachClass.get(classToRetrieve.getName()); } else { try { ret = classToRetrieve.newInstance().liteFields(); liteFieldsForEachClass.put(classToRetrieve.getName(), ret); } catch (InstantiationException | IllegalAccessException e) { LOGGER.error("Error instantiating object of type {}", classToRetrieve.getName(), e); ret = new ArrayList<>(); } } return ret; } public static <T extends IsIndexed> IndexResult<T> find(SolrClient index, Class<T> classToRetrieve, Filter filter, Sorter sorter, Sublist sublist, Facets facets, User user, boolean justActive, List<String> fieldsToReturn) throws GenericException, RequestNotValidException { IndexResult<T> ret; SolrQuery query = new SolrQuery(); query.setParam("q.op", DEFAULT_QUERY_PARSER_OPERATOR); query.setQuery(parseFilter(filter)); query.setSorts(parseSorter(sorter)); query.setStart(sublist.getFirstElementIndex()); query.setRows(sublist.getMaximumElementCount()); query.setFields(fieldsToReturn.toArray(new String[fieldsToReturn.size()])); parseAndConfigureFacets(facets, query); if (hasPermissionFilters(classToRetrieve)) { query.addFilterQuery(getFilterQueries(user, justActive, classToRetrieve)); } try { QueryResponse response = index.query(getIndexName(classToRetrieve).get(0), query); ret = queryResponseToIndexResult(response, classToRetrieve, facets, fieldsToReturn); } catch (SolrServerException | IOException e) { throw new GenericException("Could not query index", e); } catch (SolrException e) { throw new RequestNotValidException(e.getMessage()); } catch (RuntimeException e) { throw new GenericException("Unexpected exception while querying index", e); } return ret; } /* * "Internal" helper methods * ____________________________________________________________________________________________________________________ */ private static <T> T solrDocumentTo(Class<T> resultClass, SolrDocument doc, List<String> fieldsToReturn) throws GenericException { T ret; if (resultClass.equals(IndexedAIP.class)) { ret = resultClass.cast(solrDocumentToIndexedAIP(doc)); } else if (resultClass.equals(IndexedRepresentation.class) || resultClass.equals(Representation.class)) { ret = resultClass.cast(solrDocumentToRepresentation(doc)); } else if (resultClass.equals(LogEntry.class)) { ret = resultClass.cast(solrDocumentToLogEntry(doc, fieldsToReturn)); } else if (resultClass.equals(Report.class) || resultClass.equals(IndexedReport.class)) { ret = resultClass.cast(solrDocumentToJobReport(doc, fieldsToReturn)); } else if (resultClass.equals(RODAMember.class) || resultClass.equals(User.class) || resultClass.equals(Group.class)) { ret = resultClass.cast(solrDocumentToRodaMember(doc)); } else if (resultClass.equals(TransferredResource.class)) { ret = resultClass.cast(solrDocumentToTransferredResource(doc)); } else if (resultClass.equals(Job.class)) { ret = resultClass.cast(solrDocumentToJob(doc, fieldsToReturn)); } else if (resultClass.equals(Risk.class) || resultClass.equals(IndexedRisk.class)) { ret = resultClass.cast(solrDocumentToRisk(doc)); } else if (resultClass.equals(Format.class)) { ret = resultClass.cast(solrDocumentToFormat(doc)); } else if (resultClass.equals(Notification.class)) { ret = resultClass.cast(solrDocumentToNotification(doc, fieldsToReturn)); } else if (resultClass.equals(RiskIncidence.class)) { ret = resultClass.cast(solrDocumentToRiskIncidence(doc)); } else if (resultClass.equals(DIP.class) || resultClass.equals(IndexedDIP.class)) { ret = resultClass.cast(solrDocumentToIndexedDIP(doc, fieldsToReturn)); } else if (resultClass.equals(DIPFile.class)) { ret = resultClass.cast(solrDocumentToDIPFile(doc)); } else if (resultClass.equals(IndexedFile.class)) { ret = resultClass.cast(solrDocumentToIndexedFile(doc)); } else if (resultClass.equals(IndexedPreservationEvent.class)) { ret = resultClass.cast(solrDocumentToIndexedPreservationEvent(doc)); } else if (resultClass.equals(IndexedPreservationAgent.class)) { ret = resultClass.cast(solrDocumentToIndexedPreservationAgent(doc)); } else { throw new GenericException("Cannot find class index name: " + resultClass.getName()); } return ret; } private static <T> SolrInputDocument toSolrDocument(Class<T> resultClass, T object) throws GenericException, NotSupportedException { SolrInputDocument ret; if (resultClass.equals(IndexedAIP.class)) { throw new NotSupportedException(); } else if (resultClass.equals(IndexedRepresentation.class) || resultClass.equals(Representation.class)) { throw new NotSupportedException(); } else if (resultClass.equals(LogEntry.class)) { ret = logEntryToSolrDocument((LogEntry) object); } else if (resultClass.equals(Report.class) || resultClass.equals(IndexedReport.class)) { throw new NotSupportedException(); } else if (resultClass.equals(RODAMember.class) || resultClass.equals(User.class) || resultClass.equals(Group.class)) { ret = rodaMemberToSolrDocument((RODAMember) object); } else if (resultClass.equals(TransferredResource.class)) { ret = transferredResourceToSolrDocument((TransferredResource) object); } else if (resultClass.equals(Job.class)) { ret = jobToSolrDocument((Job) object); } else if (resultClass.equals(Risk.class) || resultClass.equals(IndexedRisk.class)) { ret = riskToSolrDocument((Risk) object, 0); } else if (resultClass.equals(Format.class)) { ret = formatToSolrDocument((Format) object); } else if (resultClass.equals(Notification.class)) { ret = notificationToSolrDocument((Notification) object); } else if (resultClass.equals(RiskIncidence.class)) { ret = riskIncidenceToSolrDocument((RiskIncidence) object); } else if (resultClass.equals(DIP.class) || resultClass.equals(IndexedDIP.class)) { ret = dipToSolrDocument((DIP) object); } else if (resultClass.equals(IndexedFile.class) || resultClass.equals(DIPFile.class)) { throw new NotSupportedException(); } else if (resultClass.equals(IndexedPreservationEvent.class)) { throw new NotSupportedException(); } else if (resultClass.equals(IndexedPreservationAgent.class)) { throw new NotSupportedException(); } else { throw new GenericException("Cannot find class index name: " + resultClass.getName()); } return ret; } public static <T extends Serializable> List<String> getIndexName(Class<T> resultClass) throws GenericException { List<String> indexNames = new ArrayList<>(); // INFO 201608 nvieira: the first index name must be the "main" one if (resultClass.equals(AIP.class) || resultClass.equals(IndexedAIP.class)) { indexNames.add(RodaConstants.INDEX_AIP); indexNames.add(RodaConstants.INDEX_REPRESENTATION); indexNames.add(RodaConstants.INDEX_FILE); // INFO nvieira 20170207 events should be cleared on specific plugin } else if (resultClass.equals(Representation.class) || resultClass.equals(IndexedRepresentation.class)) { indexNames.add(RodaConstants.INDEX_REPRESENTATION); } else if (resultClass.equals(IndexedPreservationEvent.class)) { indexNames.add(RodaConstants.INDEX_PRESERVATION_EVENTS); } else if (resultClass.equals(IndexedPreservationAgent.class)) { indexNames.add(RodaConstants.INDEX_PRESERVATION_AGENTS); } else if (resultClass.equals(LogEntry.class)) { indexNames.add(RodaConstants.INDEX_ACTION_LOG); } else if (resultClass.equals(Report.class) || resultClass.equals(IndexedReport.class)) { indexNames.add(RodaConstants.INDEX_JOB_REPORT); } else if (resultClass.equals(User.class) || resultClass.equals(Group.class) || resultClass.equals(RODAMember.class)) { indexNames.add(RodaConstants.INDEX_MEMBERS); } else if (resultClass.equals(TransferredResource.class)) { indexNames.add(RodaConstants.INDEX_TRANSFERRED_RESOURCE); } else if (resultClass.equals(Job.class)) { indexNames.add(RodaConstants.INDEX_JOB); indexNames.add(RodaConstants.INDEX_JOB_REPORT); } else if (resultClass.equals(IndexedFile.class) || resultClass.equals(File.class)) { indexNames.add(RodaConstants.INDEX_FILE); } else if (resultClass.equals(Risk.class) || resultClass.equals(IndexedRisk.class)) { indexNames.add(RodaConstants.INDEX_RISK); } else if (resultClass.equals(Format.class)) { indexNames.add(RodaConstants.INDEX_FORMAT); } else if (resultClass.equals(Notification.class)) { indexNames.add(RodaConstants.INDEX_NOTIFICATION); } else if (resultClass.equals(RiskIncidence.class)) { indexNames.add(RodaConstants.INDEX_RISK_INCIDENCE); } else if (resultClass.equals(DIP.class) || resultClass.equals(IndexedDIP.class)) { indexNames.add(RodaConstants.INDEX_DIP); indexNames.add(RodaConstants.INDEX_DIP_FILE); } else if (resultClass.equals(DIPFile.class)) { indexNames.add(RodaConstants.INDEX_DIP_FILE); } else { throw new GenericException("Cannot find class index name: " + resultClass.getName()); } return indexNames; } private static <T> boolean hasPermissionFilters(Class<T> resultClass) throws GenericException { return HasPermissionFilters.class.isAssignableFrom(resultClass); } /** * Method that knows how to escape characters for Solr * <p> * <code>+ - && || ! ( ) { } [ ] ^ " ~ * ? : /</code> * </p> * <p> * Note: chars <code>'*'</code> are not being escaped on purpose * </p> * * @return a string with special characters escaped */ // FIXME perhaps && and || are not being properly escaped: see how to do it public static String escapeSolrSpecialChars(String string) { return string.replaceAll("([+&|!(){}\\[\\-\\]\\^\\\\~?:\"/])", "\\\\$1"); } private static List<String> objectToListString(Object object) { List<String> ret; if (object == null) { ret = new ArrayList<>(); } else if (object instanceof String) { List<String> l = new ArrayList<>(); l.add((String) object); return l; } else if (object instanceof List<?>) { List<?> l = (List<?>) object; ret = new ArrayList<>(); for (Object o : l) { ret.add(o.toString()); } } else { LOGGER.error("Could not convert Solr object to List<String> ({})", object.getClass().getName()); ret = new ArrayList<>(); } return ret; } public static Integer objectToInteger(Object object, Integer defaultValue) { Integer ret = defaultValue; if (object != null) { if (object instanceof Integer) { ret = (Integer) object; } else if (object instanceof String) { try { ret = Integer.parseInt((String) object); } catch (NumberFormatException e) { LOGGER.error("Could not convert Solr object to integer", e); } } else { LOGGER.error("Could not convert Solr object to integer ({})", object.getClass().getName()); } } return ret; } public static Long objectToLong(Object object, Long defaultValue) { Long ret = defaultValue; if (object != null) { if (object instanceof Long) { ret = (Long) object; } else if (object instanceof String) { try { ret = Long.parseLong((String) object); } catch (NumberFormatException e) { LOGGER.error("Could not convert Solr object to long", e); } } else { LOGGER.error("Could not convert Solr object to long ({})", object.getClass().getName()); } } return ret; } @SuppressWarnings("unused") private static Float objectToFloat(Object object) { Float ret; if (object instanceof Float) { ret = (Float) object; } else if (object instanceof String) { try { ret = Float.parseFloat((String) object); } catch (NumberFormatException e) { LOGGER.error("Could not convert Solr object to float", e); ret = null; } } else { LOGGER.error("Could not convert Solr object to float ({})", object.getClass().getName()); ret = null; } return ret; } private static Date objectToDate(Object object) { Date ret; if (object == null) { ret = null; } else if (object instanceof Date) { ret = (Date) object; } else if (object instanceof String) { try { LOGGER.trace("Parsing date ({}) from string", object); ret = RodaUtils.parseDate((String) object); } catch (ParseException e) { LOGGER.error("Could not convert Solr object to date", e); ret = null; } } else { LOGGER.error("Could not convert Solr object to date, unsupported class: {}", object.getClass().getName()); ret = null; } return ret; } private static Boolean objectToBoolean(Object object, Boolean defaultValue) { Boolean ret = defaultValue; if (object != null) { if (object instanceof Boolean) { ret = (Boolean) object; } else if (object instanceof String) { ret = Boolean.parseBoolean((String) object); } else { LOGGER.error("Could not convert Solr object to Boolean ({})", object.getClass().getName()); } } return ret; } private static String objectToString(Object object, String defaultValue) { String ret = defaultValue; if (object != null) { if (object instanceof String) { ret = (String) object; } else { LOGGER.warn("Could not convert Solr object to string, unsupported class: {}", object.getClass().getName()); } } return ret; } private static <E extends Enum<E>> E objectToEnum(Object object, Class<E> enumeration, E defaultValue) { E ret = defaultValue; if (object != null) { if (object instanceof String) { String name = (String) object; try { ret = Enum.valueOf(enumeration, name); } catch (IllegalArgumentException e) { LOGGER.warn("Invalid name for enumeration: {}, name: {}", enumeration.getName(), name); } catch (NullPointerException e) { LOGGER.warn("Error parsing enumeration: {}, name: {}", enumeration.getName(), name); } } else { LOGGER.warn("Could not convert Solr object to enumeration: {}, unsupported class: {}", enumeration.getName(), object.getClass().getName()); } } return ret; } /** * @deprecated use {@link #objectToString(Object, String)} instead */ @Deprecated private static String objectToString(Object object) { String ret; if (object == null) { ret = null; } else if (object instanceof String) { ret = (String) object; } else { LOGGER.warn("Could not convert Solr object to string, unsupported class: {}", object.getClass().getName()); ret = object.toString(); } return ret; } private static <T extends Serializable> IndexResult<T> queryResponseToIndexResult(QueryResponse response, Class<T> responseClass, Facets facets, List<String> liteFields) throws GenericException { final SolrDocumentList docList = response.getResults(); final List<FacetFieldResult> facetResults = processFacetFields(facets, response.getFacetFields()); final long offset = docList.getStart(); final long limit = docList.size(); final long totalCount = docList.getNumFound(); final List<T> docs = new ArrayList<>(); for (SolrDocument doc : docList) { T result = solrDocumentTo(responseClass, doc, liteFields); docs.add(result); } return new IndexResult<>(offset, limit, totalCount, docs, facetResults); } private static List<FacetFieldResult> processFacetFields(Facets facets, List<FacetField> facetFields) { List<FacetFieldResult> ret = new ArrayList<>(); FacetFieldResult facetResult; if (facetFields != null) { for (FacetField facet : facetFields) { LOGGER.trace("facet:{} count:{}", facet.getName(), facet.getValueCount()); facetResult = new FacetFieldResult(facet.getName(), facet.getValueCount(), facets.getParameters().get(facet.getName()).getValues()); for (Count count : facet.getValues()) { LOGGER.trace(" value:{} value:{}", count.getName(), count.getCount()); facetResult.addFacetValue(count.getName(), count.getName(), count.getCount()); } ret.add(facetResult); } } return ret; } public static SolrInputDocument getDescriptiveMetadataFields(Binary binary, String metadataType, String metadataVersion) throws GenericException { SolrInputDocument doc; Map<String, String> parameters = new HashMap<>(); parameters.put("prefix", RodaConstants.INDEX_OTHER_DESCRIPTIVE_DATA_PREFIX); Reader transformationResult = RodaUtils.applyMetadataStylesheet(binary, RodaConstants.CORE_CROSSWALKS_INGEST, metadataType, metadataVersion, parameters); try { XMLLoader loader = new XMLLoader(); XMLStreamReader parser = XMLInputFactory.newInstance().createXMLStreamReader(transformationResult); boolean parsing = true; doc = null; while (parsing) { int event = parser.next(); if (event == XMLStreamConstants.END_DOCUMENT) { parser.close(); parsing = false; } else if (event == XMLStreamConstants.START_ELEMENT) { String currTag = parser.getLocalName(); if ("doc".equals(currTag)) { doc = loader.readDoc(parser); } } } } catch (XMLStreamException | FactoryConfigurationError e) { throw new GenericException("Could not process descriptive metadata binary " + binary.getStoragePath(), e); } finally { IOUtils.closeQuietly(transformationResult); } return doc == null ? new SolrInputDocument() : validateDescriptiveMetadataFields(doc); } private static SolrInputDocument validateDescriptiveMetadataFields(SolrInputDocument doc) { if (doc.get(RodaConstants.AIP_DATE_INITIAL) != null) { Object value = doc.get(RodaConstants.AIP_DATE_INITIAL).getValue(); if (value instanceof String) { try { Date d = DateUtil.parseDate((String) value); doc.setField(RodaConstants.AIP_DATE_INITIAL, d); } catch (ParseException pe) { doc.remove(RodaConstants.AIP_DATE_INITIAL); doc.setField(RodaConstants.AIP_DATE_INITIAL + "_txt", value); } } } if (doc.get(RodaConstants.AIP_DATE_FINAL) != null) { Object value = doc.get(RodaConstants.AIP_DATE_FINAL).getValue(); if (value instanceof String) { try { Date d = DateUtil.parseDate((String) value); doc.setField(RodaConstants.AIP_DATE_FINAL, d); } catch (ParseException pe) { doc.remove(RodaConstants.AIP_DATE_FINAL); doc.setField(RodaConstants.AIP_DATE_FINAL + "_txt", value); } } } return doc; } /* * Roda Filter > Apache Solr query * ____________________________________________________________________________________________________________________ */ public static String parseFilter(Filter filter) throws RequestNotValidException { StringBuilder ret = new StringBuilder(); if (filter == null || filter.getParameters().isEmpty()) { ret.append("*:*"); } else { for (FilterParameter parameter : filter.getParameters()) { parseFilterParameter(ret, parameter, true); } if (ret.length() == 0) { ret.append("*:*"); } } LOGGER.trace("Converting filter {} to query {}", filter, ret); return ret.toString(); } private static void parseFilterParameter(StringBuilder ret, FilterParameter parameter, boolean prefixWithANDOperatorIfBuilderNotEmpty) throws RequestNotValidException { if (parameter instanceof SimpleFilterParameter) { SimpleFilterParameter simplePar = (SimpleFilterParameter) parameter; appendExactMatch(ret, simplePar.getName(), simplePar.getValue(), true, prefixWithANDOperatorIfBuilderNotEmpty); } else if (parameter instanceof OneOfManyFilterParameter) { OneOfManyFilterParameter param = (OneOfManyFilterParameter) parameter; appendValuesUsingOROperator(ret, param.getName(), param.getValues(), prefixWithANDOperatorIfBuilderNotEmpty); } else if (parameter instanceof BasicSearchFilterParameter) { BasicSearchFilterParameter param = (BasicSearchFilterParameter) parameter; appendBasicSearch(ret, param.getName(), param.getValue(), "AND", prefixWithANDOperatorIfBuilderNotEmpty); } else if (parameter instanceof EmptyKeyFilterParameter) { EmptyKeyFilterParameter param = (EmptyKeyFilterParameter) parameter; appendANDOperator(ret, true); ret.append("(*:* NOT " + param.getName() + ":*)"); } else if (parameter instanceof DateRangeFilterParameter) { DateRangeFilterParameter param = (DateRangeFilterParameter) parameter; appendRange(ret, param.getName(), Date.class, param.getFromValue(), String.class, processToDate(param.getToValue(), param.getGranularity(), false), prefixWithANDOperatorIfBuilderNotEmpty); } else if (parameter instanceof DateIntervalFilterParameter) { DateIntervalFilterParameter param = (DateIntervalFilterParameter) parameter; appendRangeInterval(ret, param.getFromName(), param.getToName(), param.getFromValue(), param.getToValue(), param.getGranularity(), prefixWithANDOperatorIfBuilderNotEmpty); } else if (parameter instanceof LongRangeFilterParameter) { LongRangeFilterParameter param = (LongRangeFilterParameter) parameter; appendRange(ret, param.getName(), Long.class, param.getFromValue(), Long.class, param.getToValue(), prefixWithANDOperatorIfBuilderNotEmpty); } else if (parameter instanceof NotSimpleFilterParameter) { NotSimpleFilterParameter notSimplePar = (NotSimpleFilterParameter) parameter; appendNotExactMatch(ret, notSimplePar.getName(), notSimplePar.getValue(), true, prefixWithANDOperatorIfBuilderNotEmpty); } else if (parameter instanceof OrFiltersParameters || parameter instanceof AndFiltersParameters) { FiltersParameters filters = (FiltersParameters) parameter; appendFiltersWithOperator(ret, parameter instanceof OrFiltersParameters ? "OR" : "AND", filters.getValues(), prefixWithANDOperatorIfBuilderNotEmpty); } else { LOGGER.error("Unsupported filter parameter class: {}", parameter.getClass().getName()); throw new RequestNotValidException( "Unsupported filter parameter class: " + parameter.getClass().getName()); } } private static void appendANDOperator(StringBuilder ret, boolean prefixWithANDOperatorIfBuilderNotEmpty) { if (prefixWithANDOperatorIfBuilderNotEmpty && ret.length() > 0) { ret.append(" AND "); } } private static void appendOROperator(StringBuilder ret, boolean prefixWithOROperatorIfBuilderNotEmpty) { if (prefixWithOROperatorIfBuilderNotEmpty && ret.length() > 0) { ret.append(" OR "); } } private static void appendExactMatch(StringBuilder ret, String key, String value, boolean appendDoubleQuotes, boolean prefixWithANDOperatorIfBuilderNotEmpty) { appendANDOperator(ret, prefixWithANDOperatorIfBuilderNotEmpty); ret.append("(").append(key).append(": "); if (appendDoubleQuotes) { ret.append("\""); } ret.append(value.replaceAll("(\")", "\\\\$1")); if (appendDoubleQuotes) { ret.append("\""); } ret.append(")"); } private static void appendValuesUsingOROperator(StringBuilder ret, String key, List<String> values, boolean prefixWithANDOperatorIfBuilderNotEmpty) { if (!values.isEmpty()) { appendANDOperator(ret, prefixWithANDOperatorIfBuilderNotEmpty); ret.append("("); for (int i = 0; i < values.size(); i++) { if (i != 0) { ret.append(" OR "); } appendExactMatch(ret, key, values.get(i), true, false); } ret.append(")"); } } private static void appendBasicSearch(StringBuilder ret, String key, String value, String operator, boolean prefixWithANDOperatorIfBuilderNotEmpty) { if (StringUtils.isBlank(value)) { appendExactMatch(ret, key, "*", false, prefixWithANDOperatorIfBuilderNotEmpty); } else if (value.matches("^\".+\"$")) { appendExactMatch(ret, key, value.substring(1, value.length() - 1), true, prefixWithANDOperatorIfBuilderNotEmpty); } else { appendWhiteSpaceTokenizedString(ret, key, value, operator, prefixWithANDOperatorIfBuilderNotEmpty); } } private static void appendKeyValue(StringBuilder ret, String key, String value) { ret.append(key).append(":").append("(").append(value).append(")"); } private static void appendFiltersWithOperator(StringBuilder ret, String operator, List<FilterParameter> values, boolean prefixWithANDOperatorIfBuilderNotEmpty) throws RequestNotValidException { if (!values.isEmpty()) { appendANDOperator(ret, prefixWithANDOperatorIfBuilderNotEmpty); ret.append("("); for (int i = 0; i < values.size(); i++) { if (i != 0) { ret.append(" ").append(operator).append(" "); } parseFilterParameter(ret, values.get(i), false); } ret.append(")"); } } private static void appendWhiteSpaceTokenizedString(StringBuilder ret, String key, String value, String operator, boolean prefixWithANDOperatorIfBuilderNotEmpty) { appendANDOperator(ret, prefixWithANDOperatorIfBuilderNotEmpty); String[] split = value.trim().split("\\s+"); ret.append("("); for (int i = 0; i < split.length; i++) { if (i != 0 && operator != null) { ret.append(" " + operator + " "); } if (split[i].matches("(AND|OR|NOT)")) { ret.append(key).append(": \"").append(split[i]).append("\""); } else { ret.append(key).append(": (").append(escapeSolrSpecialChars(split[i])).append(")"); } } ret.append(")"); } private static <T extends Serializable, T1 extends Serializable> void appendRange(StringBuilder ret, String key, Class<T> fromClass, T fromValue, Class<T1> toClass, T1 toValue, boolean prefixWithANDOperatorIfBuilderNotEmpty) { if (fromValue != null || toValue != null) { appendANDOperator(ret, prefixWithANDOperatorIfBuilderNotEmpty); ret.append("(").append(key).append(":["); generateRangeValue(ret, fromClass, fromValue); ret.append(" TO "); generateRangeValue(ret, toClass, toValue); ret.append("])"); } } private static <T extends Serializable> void generateRangeValue(StringBuilder ret, Class<T> valueClass, T value) { if (value != null) { if (valueClass.equals(Date.class)) { String date = DateUtil.getThreadLocalDateFormat().format(Date.class.cast(value)); LOGGER.trace("Appending date value \"{}\" to range", date); ret.append(date); } else if (valueClass.equals(Long.class)) { ret.append(Long.class.cast(value)); } else if (valueClass.equals(String.class)) { ret.append(String.class.cast(value)); } else { LOGGER.error("Cannot process range of the type {}", valueClass); } } else { ret.append("*"); } } private static void appendRangeInterval(StringBuilder ret, String fromKey, String toKey, Date fromValue, Date toValue, DateGranularity granularity, boolean prefixWithANDOperatorIfBuilderNotEmpty) { if (fromValue != null || toValue != null) { appendANDOperator(ret, prefixWithANDOperatorIfBuilderNotEmpty); ret.append("("); ret.append(fromKey).append(":["); ret.append(processFromDate(fromValue)); ret.append(" TO "); ret.append(processToDate(toValue, granularity)); ret.append("]").append(" OR "); ret.append(toKey).append(":["); ret.append(processFromDate(fromValue)); ret.append(" TO "); ret.append(processToDate(toValue, granularity)); ret.append("]"); if (fromValue != null && toValue != null) { ret.append(" OR ").append("(").append(fromKey).append(":[* TO ") .append(processToDate(fromValue, granularity)).append("]"); ret.append(" AND ").append(toKey).append(":[").append(processFromDate(toValue)).append(" TO *]") .append(")"); } ret.append(")"); } } private static String processFromDate(Date fromValue) { final String ret; if (fromValue != null) { return DateUtil.getThreadLocalDateFormat().format(fromValue); } else { ret = "*"; } return ret; } private static String processToDate(Date toValue, DateGranularity granularity) { return processToDate(toValue, granularity, true); } private static String processToDate(Date toValue, DateGranularity granularity, boolean returnAsteriskOnNull) { final String ret; StringBuilder sb = new StringBuilder(); if (toValue != null) { sb.append(DateUtil.getThreadLocalDateFormat().format(toValue)); switch (granularity) { case YEAR: sb.append("+1YEAR-1MILLISECOND"); break; case MONTH: sb.append("+1MONTH-1MILLISECOND"); break; case DAY: sb.append("+1DAY-1MILLISECOND"); break; case HOUR: sb.append("+1HOUR-1MILLISECOND"); break; case MINUTE: sb.append("+1MINUTE-1MILLISECOND"); break; case SECOND: sb.append("+1SECOND-1MILLISECOND"); break; default: // do nothing break; } ret = sb.toString(); } else { ret = returnAsteriskOnNull ? "*" : null; } return ret; } private static void appendNotExactMatch(StringBuilder ret, String key, String value, boolean appendDoubleQuotes, boolean prefixWithANDOperatorIfBuilderNotEmpty) { appendExactMatch(ret, "*:* -" + key, value, appendDoubleQuotes, prefixWithANDOperatorIfBuilderNotEmpty); } public static String getLastScanDate(Date scanDate) { return DateUtil.getThreadLocalDateFormat().format(scanDate); } /* * Roda Sorter > Apache Solr Sort clauses * ____________________________________________________________________________________________________________________ */ public static List<SortClause> parseSorter(Sorter sorter) { List<SortClause> ret = new ArrayList<>(); if (sorter != null) { for (SortParameter sortParameter : sorter.getParameters()) { ret.add(new SortClause(sortParameter.getName(), sortParameter.isDescending() ? ORDER.desc : ORDER.asc)); } } return ret; } /* * Roda Facets > Apache Solr Facets * ____________________________________________________________________________________________________________________ */ private static void parseAndConfigureFacets(Facets facets, SolrQuery query) { if (facets != null) { query.setFacetSort(getSolrFacetParameterSortValue(FacetParameter.DEFAULT_SORT)); if (!"".equals(facets.getQuery())) { query.addFacetQuery(facets.getQuery()); } StringBuilder filterQuery = new StringBuilder(); for (Entry<String, FacetParameter> parameter : facets.getParameters().entrySet()) { FacetParameter facetParameter = parameter.getValue(); setSolrFacetParameterSort(query, facetParameter); if (facetParameter instanceof SimpleFacetParameter) { setQueryFacetParameter(query, (SimpleFacetParameter) facetParameter); appendValuesUsingOROperator(filterQuery, facetParameter.getName(), ((SimpleFacetParameter) facetParameter).getValues(), true); } else { LOGGER.error("Unsupported facet parameter class: {}", facetParameter.getClass().getName()); } } if (filterQuery.length() > 0) { query.addFilterQuery(filterQuery.toString()); LOGGER.trace("Query after defining facets: {}", query); } } } private static void setSolrFacetParameterSort(SolrQuery query, FacetParameter facetParameter) { if (FacetParameter.DEFAULT_SORT != facetParameter.getSort()) { query.add(String.format("f.%s.facet.sort", facetParameter.getName()), getSolrFacetParameterSortValue(facetParameter.getSort())); } } private static String getSolrFacetParameterSortValue(SORT facetSort) { return facetSort == SORT.INDEX ? FacetParams.FACET_SORT_INDEX : FacetParams.FACET_SORT_COUNT; } private static void setQueryFacetParameter(SolrQuery query, SimpleFacetParameter facetParameter) { query.addFacetField(facetParameter.getName()); query.add(String.format("f.%s.facet.mincount", facetParameter.getName()), String.valueOf(facetParameter.getMinCount())); query.add(String.format("f.%s.facet.limit", facetParameter.getName()), String.valueOf(facetParameter.getLimit())); } /* * Roda user > Apache Solr filter query * ____________________________________________________________________________________________________________________ */ private static <T extends IsIndexed> String getFilterQueries(User user, boolean justActive, Class<T> classToRetrieve) { StringBuilder fq = new StringBuilder(); // TODO find a better way to define admin super powers if (user != null && !RodaConstants.ADMIN.equals(user.getName())) { fq.append("("); String usersKey = RodaConstants.INDEX_PERMISSION_USERS_PREFIX + PermissionType.READ; appendExactMatch(fq, usersKey, user.getId(), true, false); String groupsKey = RodaConstants.INDEX_PERMISSION_GROUPS_PREFIX + PermissionType.READ; appendValuesUsingOROperatorForQuery(fq, groupsKey, new ArrayList<>(user.getGroups()), true); fq.append(")"); } if (justActive && !IndexedDIP.class.equals(classToRetrieve) && !DIPFile.class.equals(classToRetrieve)) { appendExactMatch(fq, RodaConstants.STATE, AIPState.ACTIVE.toString(), true, true); } return fq.toString(); } private static void appendValuesUsingOROperatorForQuery(StringBuilder ret, String key, List<String> values, boolean prependWithOrIfNeeded) { if (!values.isEmpty()) { if (prependWithOrIfNeeded) { appendOROperator(ret, true); } else { appendANDOperator(ret, true); } ret.append("("); for (int i = 0; i < values.size(); i++) { if (i != 0) { ret.append(" OR "); } appendExactMatch(ret, key, values.get(i), true, false); } ret.append(")"); } } /* * Apache Solr helper methods * ____________________________________________________________________________________________________________________ */ private static void commit(SolrClient index, String... collections) { boolean waitFlush = false; boolean waitSearcher = true; boolean softCommit = true; for (String collection : collections) { try { index.commit(collection, waitFlush, waitSearcher, softCommit); } catch (SolrServerException | IOException e) { LOGGER.error("Error commiting into collection: {}", collection, e); } } } public static void commit(SolrClient index, List<Class<? extends IsIndexed>> resultClasses) throws GenericException { List<String> collections = new ArrayList<>(); for (Class<? extends IsIndexed> resultClass : resultClasses) { collections.add(getIndexName(resultClass).get(0)); } commit(index, collections.toArray(new String[] {})); } @SafeVarargs public static void commit(SolrClient index, Class<? extends IsIndexed>... resultClasses) throws GenericException { commit(index, Arrays.asList(resultClasses)); } public static <T extends IsIndexed> void create(SolrClient index, Class<T> classToCreate, T instance) throws GenericException { try { index.add(getIndexName(classToCreate).get(0), toSolrDocument(classToCreate, instance)); } catch (SolrServerException | IOException | NotSupportedException e) { throw new GenericException("Error adding instance to index", e); } } /* * Crosswalks: RODA Objects <-> Apache Solr documents * ____________________________________________________________________________________________________________________ */ public static IndexedAIP solrDocumentToIndexedAIP(SolrDocument doc) { final String id = objectToString(doc.get(RodaConstants.INDEX_UUID), null); AIPState state = null; if (doc.containsKey(RodaConstants.STATE)) { state = AIPState .valueOf(objectToString(doc.get(RodaConstants.STATE), AIPState.getDefault().toString())); } final String parentId = objectToString(doc.get(RodaConstants.AIP_PARENT_ID), null); final List<String> ingestSIPIds = objectToListString(doc.get(RodaConstants.INGEST_SIP_IDS)); final String ingestJobId = objectToString(doc.get(RodaConstants.INGEST_JOB_ID), ""); final List<String> ingestUpdateJobIds = objectToListString(doc.get(RodaConstants.INGEST_UPDATE_JOB_IDS)); final List<String> ancestors = objectToListString(doc.get(RodaConstants.AIP_ANCESTORS)); final List<String> levels = objectToListString(doc.get(RodaConstants.AIP_LEVEL)); final List<String> titles = objectToListString(doc.get(RodaConstants.AIP_TITLE)); final List<String> descriptions = objectToListString(doc.get(RodaConstants.AIP_DESCRIPTION)); final Date dateInitial = objectToDate(doc.get(RodaConstants.AIP_DATE_INITIAL)); final Date dateFinal = objectToDate(doc.get(RodaConstants.AIP_DATE_FINAL)); final Long numberOfSubmissionFiles = objectToLong(doc.get(RodaConstants.AIP_NUMBER_OF_SUBMISSION_FILES), 0L); final Long numberOfDocumentationFiles = objectToLong( doc.get(RodaConstants.AIP_NUMBER_OF_DOCUMENTATION_FILES), 0L); final Long numberOfSchemaFiles = objectToLong(doc.get(RodaConstants.AIP_NUMBER_OF_SCHEMA_FILES), 0L); final Boolean hasRepresentations = objectToBoolean(doc.get(RodaConstants.AIP_HAS_REPRESENTATIONS), Boolean.FALSE); final Boolean ghost = objectToBoolean(doc.get(RodaConstants.AIP_GHOST), Boolean.FALSE); Permissions permissions = getPermissions(doc); final String title = titles.isEmpty() ? null : titles.get(0); final String description = descriptions.isEmpty() ? null : descriptions.get(0); String level; if (ghost) { level = RodaConstants.AIP_GHOST; } else { level = levels.isEmpty() ? null : levels.get(0); } return new IndexedAIP(id, state, level, title, dateInitial, dateFinal, description, parentId, ancestors, permissions, numberOfSubmissionFiles, numberOfDocumentationFiles, numberOfSchemaFiles, hasRepresentations, ghost).setIngestSIPIds(ingestSIPIds).setIngestJobId(ingestJobId) .setIngestUpdateJobIds(ingestUpdateJobIds); } public static SolrInputDocument aipToSolrInputDocument(AIP aip, List<String> ancestors, ModelService model, boolean safemode) throws RequestNotValidException, GenericException, NotFoundException, AuthorizationDeniedException { SolrInputDocument ret = new SolrInputDocument(); ret.addField(RodaConstants.INDEX_UUID, aip.getId()); ret.addField(RodaConstants.AIP_ID, aip.getId()); ret.addField(RodaConstants.AIP_PARENT_ID, aip.getParentId()); ret.addField(RodaConstants.STATE, aip.getState().toString()); ret.addField(RodaConstants.INGEST_SIP_IDS, aip.getIngestSIPIds()); ret.addField(RodaConstants.INGEST_JOB_ID, aip.getIngestJobId()); ret.addField(RodaConstants.INGEST_UPDATE_JOB_IDS, aip.getIngestUpdateJobIds()); ret.addField(RodaConstants.AIP_ANCESTORS, ancestors); List<String> descriptiveMetadataIds = aip.getDescriptiveMetadata().stream().map(dm -> dm.getId()) .collect(Collectors.toList()); ret.addField(RodaConstants.AIP_DESCRIPTIVE_METADATA_ID, descriptiveMetadataIds); List<String> representationIds = aip.getRepresentations().stream().map(r -> r.getId()) .collect(Collectors.toList()); ret.addField(RodaConstants.AIP_REPRESENTATION_ID, representationIds); ret.addField(RodaConstants.AIP_HAS_REPRESENTATIONS, !representationIds.isEmpty()); setPermissions(aip.getPermissions(), ret); ret.addField(RodaConstants.AIP_GHOST, aip.getGhost() != null ? aip.getGhost() : false); if (!safemode) { // guarding against repeated fields Set<String> usedNonRepeatableFields = new HashSet<>(); for (DescriptiveMetadata metadata : aip.getDescriptiveMetadata()) { StoragePath storagePath = ModelUtils.getDescriptiveMetadataStoragePath(aip.getId(), metadata.getId()); Binary binary = model.getStorage().getBinary(storagePath); try { SolrInputDocument fields = getDescriptiveMetadataFields(binary, metadata.getType(), metadata.getVersion()); for (SolrInputField field : fields) { if (NON_REPEATABLE_FIELDS.contains(field.getName())) { boolean added = usedNonRepeatableFields.add(field.getName()); if (added) { ret.addField(field.getName(), field.getValue(), field.getBoost()); } } else { ret.addField(field.getName(), field.getValue(), field.getBoost()); } } } catch (GenericException e) { LOGGER.info("Problem processing descriptive metadata: {}", e.getMessage()); } catch (Exception e) { LOGGER.error("Error processing descriptive metadata: {}", metadata, e); } } } // Calculate number of documentation and schema files StorageService storage = model.getStorage(); Long numberOfSubmissionFiles; try { Directory submissionDirectory = model.getSubmissionDirectory(aip.getId()); numberOfSubmissionFiles = storage.countResourcesUnderDirectory(submissionDirectory.getStoragePath(), true); } catch (NotFoundException e) { numberOfSubmissionFiles = 0L; } Long numberOfDocumentationFiles; try { Directory documentationDirectory = model.getDocumentationDirectory(aip.getId()); numberOfDocumentationFiles = storage .countResourcesUnderDirectory(documentationDirectory.getStoragePath(), true); } catch (NotFoundException e) { numberOfDocumentationFiles = 0L; } Long numberOfSchemaFiles; try { Directory schemasDirectory = model.getSchemasDirectory(aip.getId()); numberOfSchemaFiles = storage.countResourcesUnderDirectory(schemasDirectory.getStoragePath(), true); } catch (NotFoundException e) { numberOfSchemaFiles = 0L; } ret.addField(RodaConstants.AIP_NUMBER_OF_SUBMISSION_FILES, numberOfSubmissionFiles); ret.addField(RodaConstants.AIP_NUMBER_OF_DOCUMENTATION_FILES, numberOfDocumentationFiles); ret.addField(RodaConstants.AIP_NUMBER_OF_SCHEMA_FILES, numberOfSchemaFiles); return ret; } public static IndexedRepresentation solrDocumentToRepresentation(SolrDocument doc) { final String uuid = objectToString(doc.get(RodaConstants.INDEX_UUID), null); final String id = objectToString(doc.get(RodaConstants.REPRESENTATION_ID), null); final String aipId = objectToString(doc.get(RodaConstants.REPRESENTATION_AIP_ID), null); final Boolean original = objectToBoolean(doc.get(RodaConstants.REPRESENTATION_ORIGINAL), Boolean.FALSE); final String type = objectToString(doc.get(RodaConstants.REPRESENTATION_TYPE), null); final Long sizeInBytes = objectToLong(doc.get(RodaConstants.REPRESENTATION_SIZE_IN_BYTES), 0L); final Long totalNumberOfFiles = objectToLong(doc.get(RodaConstants.REPRESENTATION_NUMBER_OF_DATA_FILES), 0L); final Long numberOfDocumentationFiles = objectToLong( doc.get(RodaConstants.REPRESENTATION_NUMBER_OF_DOCUMENTATION_FILES), 0L); final Long numberOfSchemaFiles = objectToLong(doc.get(RodaConstants.REPRESENTATION_NUMBER_OF_SCHEMA_FILES), 0L); final List<String> ancestors = objectToListString(doc.get(RodaConstants.AIP_ANCESTORS)); return new IndexedRepresentation(uuid, id, aipId, Boolean.TRUE.equals(original), type, sizeInBytes, totalNumberOfFiles, numberOfDocumentationFiles, numberOfSchemaFiles, ancestors); } public static SolrInputDocument representationToSolrDocument(AIP aip, Representation rep, Long sizeInBytes, Long numberOfDataFiles, Long numberOfDocumentationFiles, Long numberOfSchemaFiles, List<String> ancestors) { SolrInputDocument doc = new SolrInputDocument(); doc.addField(RodaConstants.INDEX_UUID, IdUtils.getRepresentationId(rep)); doc.addField(RodaConstants.REPRESENTATION_ID, rep.getId()); doc.addField(RodaConstants.REPRESENTATION_AIP_ID, rep.getAipId()); doc.addField(RodaConstants.REPRESENTATION_ORIGINAL, rep.isOriginal()); doc.addField(RodaConstants.REPRESENTATION_TYPE, rep.getType()); doc.addField(RodaConstants.REPRESENTATION_SIZE_IN_BYTES, sizeInBytes); doc.addField(RodaConstants.REPRESENTATION_NUMBER_OF_DATA_FILES, numberOfDataFiles); doc.addField(RodaConstants.REPRESENTATION_NUMBER_OF_DOCUMENTATION_FILES, numberOfDocumentationFiles); doc.addField(RodaConstants.REPRESENTATION_NUMBER_OF_SCHEMA_FILES, numberOfSchemaFiles); // indexing active state and permissions doc.addField(RodaConstants.STATE, aip.getState().toString()); doc.addField(RodaConstants.INGEST_SIP_IDS, aip.getIngestSIPIds()); doc.addField(RodaConstants.INGEST_JOB_ID, aip.getIngestJobId()); doc.addField(RodaConstants.INGEST_UPDATE_JOB_IDS, aip.getIngestUpdateJobIds()); doc.addField(RodaConstants.REPRESENTATION_ANCESTORS, ancestors); setPermissions(aip.getPermissions(), doc); return doc; } public static SolrInputDocument fileToSolrDocument(AIP aip, File file, List<String> ancestors) { SolrInputDocument doc = new SolrInputDocument(); doc.addField(RodaConstants.INDEX_UUID, IdUtils.getFileId(file)); List<String> path = file.getPath(); doc.addField(RodaConstants.FILE_PATH, path); if (path != null && !path.isEmpty()) { List<String> ancestorsPath = getFileAncestorsPath(file.getAipId(), file.getRepresentationId(), path); if (!ancestorsPath.isEmpty()) { doc.addField(RodaConstants.FILE_PARENT_UUID, ancestorsPath.get(ancestorsPath.size() - 1)); doc.addField(RodaConstants.FILE_ANCESTORS_PATH, ancestorsPath); } } doc.addField(RodaConstants.FILE_AIP_ID, file.getAipId()); doc.addField(RodaConstants.FILE_FILE_ID, file.getId()); doc.addField(RodaConstants.FILE_REPRESENTATION_ID, file.getRepresentationId()); doc.addField(RodaConstants.FILE_REPRESENTATION_UUID, IdUtils.getRepresentationId(file.getAipId(), file.getRepresentationId())); doc.addField(RodaConstants.FILE_ISDIRECTORY, file.isDirectory()); // extra-fields try { StoragePath filePath = ModelUtils.getFileStoragePath(file); doc.addField(RodaConstants.FILE_STORAGEPATH, FSUtils.getStoragePathAsString(filePath, false)); } catch (RequestNotValidException e) { LOGGER.warn("Could not index file storage path", e); } String fileId = file.getId(); if (!file.isDirectory() && fileId.contains(".") && !fileId.startsWith(".")) { String extension = fileId.substring(fileId.lastIndexOf('.') + 1); doc.addField(RodaConstants.FILE_EXTENSION, extension); } // indexing AIP inherited info doc.addField(RodaConstants.STATE, aip.getState().toString()); doc.addField(RodaConstants.INGEST_SIP_IDS, aip.getIngestSIPIds()); doc.addField(RodaConstants.INGEST_JOB_ID, aip.getIngestJobId()); doc.addField(RodaConstants.INGEST_UPDATE_JOB_IDS, aip.getIngestUpdateJobIds()); doc.addField(RodaConstants.FILE_ANCESTORS, ancestors); setPermissions(aip.getPermissions(), doc); return doc; } private static List<String> getFileAncestorsPath(String aipId, String representationId, List<String> path) { List<String> parentFileDirectoryPath = new ArrayList<>(); List<String> ancestorsPath = new ArrayList<>(); parentFileDirectoryPath.addAll(path); while (!parentFileDirectoryPath.isEmpty()) { int lastElementIndex = parentFileDirectoryPath.size() - 1; String parentFileId = parentFileDirectoryPath.get(lastElementIndex); parentFileDirectoryPath.remove(lastElementIndex); ancestorsPath.add(0, IdUtils.getFileId(aipId, representationId, parentFileDirectoryPath, parentFileId)); } return ancestorsPath; } public static IndexedFile solrDocumentToIndexedFile(SolrDocument doc) { String uuid = objectToString(doc.get(RodaConstants.INDEX_UUID), null); String aipId = objectToString(doc.get(RodaConstants.FILE_AIP_ID), null); String representationId = objectToString(doc.get(RodaConstants.FILE_REPRESENTATION_ID), null); String fileId = objectToString(doc.get(RodaConstants.FILE_FILE_ID), null); List<String> path = objectToListString(doc.get(RodaConstants.FILE_PATH)); String representationUUID = objectToString(doc.get(RodaConstants.FILE_REPRESENTATION_UUID), null); String parentUUID = objectToString(doc.get(RodaConstants.FILE_PARENT_UUID), null); List<String> ancestorsPath = objectToListString(doc.get(RodaConstants.FILE_ANCESTORS_PATH)); String originalName = objectToString(doc.get(RodaConstants.FILE_ORIGINALNAME), null); List<String> hash = objectToListString(doc.get(RodaConstants.FILE_HASH)); long size = objectToLong(doc.get(RodaConstants.FILE_SIZE), 0L); boolean isDirectory = objectToBoolean(doc.get(RodaConstants.FILE_ISDIRECTORY), Boolean.FALSE); String storagePath = objectToString(doc.get(RodaConstants.FILE_STORAGEPATH), null); // format String formatDesignationName = objectToString(doc.get(RodaConstants.FILE_FILEFORMAT), null); String formatDesignationVersion = objectToString(doc.get(RodaConstants.FILE_FORMAT_VERSION), null); String mimetype = objectToString(doc.get(RodaConstants.FILE_FORMAT_MIMETYPE), null); String pronom = objectToString(doc.get(RodaConstants.FILE_PRONOM), null); String extension = objectToString(doc.get(RodaConstants.FILE_EXTENSION), null); // FIXME how to restore format registries Map<String, String> formatRegistries = new HashMap<>(); // technical features String creatingApplicationName = objectToString(doc.get(RodaConstants.FILE_CREATING_APPLICATION_NAME), null); String creatingApplicationVersion = objectToString(doc.get(RodaConstants.FILE_CREATING_APPLICATION_VERSION), null); String dateCreatedByApplication = objectToString(doc.get(RodaConstants.FILE_DATE_CREATED_BY_APPLICATION), null); final List<String> ancestors = objectToListString(doc.get(RodaConstants.AIP_ANCESTORS)); // handle other properties Map<String, List<String>> otherProperties = new HashMap<>(); for (String fieldName : doc.getFieldNames()) { if (fieldName.endsWith("_txt")) { List<String> otherProperty = objectToListString(doc.get(fieldName)); otherProperties.put(fieldName, otherProperty); } } FileFormat fileFormat = new FileFormat(formatDesignationName, formatDesignationVersion, mimetype, pronom, extension, formatRegistries); return new IndexedFile(uuid, parentUUID, aipId, representationId, representationUUID, path, ancestorsPath, fileId, fileFormat, originalName, size, isDirectory, creatingApplicationName, creatingApplicationVersion, dateCreatedByApplication, hash, storagePath, ancestors, otherProperties); } public static SolrInputDocument addOtherPropertiesToIndexedFile(String prefix, OtherMetadata otherMetadataBinary, ModelService model, SolrClient index) throws RequestNotValidException, GenericException, NotFoundException, AuthorizationDeniedException, ParserConfigurationException, SAXException, IOException, XPathExpressionException, SolrServerException { SolrDocument solrDocument = index.getById(RodaConstants.INDEX_FILE, IdUtils.getFileId(otherMetadataBinary.getAipId(), otherMetadataBinary.getRepresentationId(), otherMetadataBinary.getFileDirectoryPath(), otherMetadataBinary.getFileId())); Binary binary = model.retrieveOtherMetadataBinary(otherMetadataBinary); Map<String, List<String>> otherProperties = MetadataFileUtils.parseBinary(binary); for (Map.Entry<String, List<String>> entry : otherProperties.entrySet()) { solrDocument.setField(prefix + entry.getKey(), entry.getValue()); } return solrDocumentToSolrInputDocument(solrDocument); } private static LogEntry solrDocumentToLogEntry(SolrDocument doc, List<String> fieldsToReturn) { final String id = objectToString(doc.get(RodaConstants.LOG_ID), null); final String actionComponent = objectToString(doc.get(RodaConstants.LOG_ACTION_COMPONENT), null); final String actionMethod = objectToString(doc.get(RodaConstants.LOG_ACTION_METHOD), null); final String address = objectToString(doc.get(RodaConstants.LOG_ADDRESS), null); final Date datetime = objectToDate(doc.get(RodaConstants.LOG_DATETIME)); final long duration = objectToLong(doc.get(RodaConstants.LOG_DURATION), 0L); final String parameters = objectToString(doc.get(RodaConstants.LOG_PARAMETERS), null); final String relatedObjectId = objectToString(doc.get(RodaConstants.LOG_RELATED_OBJECT_ID), null); final String username = objectToString(doc.get(RodaConstants.LOG_USERNAME), null); LOG_ENTRY_STATE state = null; if (doc.containsKey(RodaConstants.LOG_STATE)) { state = LOG_ENTRY_STATE .valueOf(objectToString(doc.get(RodaConstants.LOG_STATE), LOG_ENTRY_STATE.UNKNOWN.toString())); } LogEntry entry = new LogEntry(); entry.setId(id); entry.setActionComponent(actionComponent); entry.setActionMethod(actionMethod); entry.setAddress(address); entry.setDatetime(datetime); entry.setDuration(duration); entry.setState(state); try { if (fieldsToReturn.isEmpty() || fieldsToReturn.contains(RodaConstants.LOG_PARAMETERS)) { entry.setParameters( JsonUtils.getListFromJson(parameters == null ? "" : parameters, LogEntryParameter.class)); } } catch (GenericException e) { LOGGER.error("Error parsing log entry parameters", e); } entry.setRelatedObjectID(relatedObjectId); entry.setUsername(username); return entry; } public static SolrInputDocument logEntryToSolrDocument(LogEntry logEntry) { SolrInputDocument doc = new SolrInputDocument(); doc.addField(RodaConstants.INDEX_UUID, logEntry.getId()); doc.addField(RodaConstants.LOG_ID, logEntry.getId()); doc.addField(RodaConstants.LOG_ACTION_COMPONENT, logEntry.getActionComponent()); doc.addField(RodaConstants.LOG_ACTION_METHOD, logEntry.getActionMethod()); doc.addField(RodaConstants.LOG_ADDRESS, logEntry.getAddress()); doc.addField(RodaConstants.LOG_DATETIME, logEntry.getDatetime()); doc.addField(RodaConstants.LOG_DURATION, logEntry.getDuration()); doc.addField(RodaConstants.LOG_PARAMETERS, JsonUtils.getJsonFromObject(logEntry.getParameters())); doc.addField(RodaConstants.LOG_RELATED_OBJECT_ID, logEntry.getRelatedObjectID()); doc.addField(RodaConstants.LOG_USERNAME, logEntry.getUsername()); doc.addField(RodaConstants.LOG_STATE, logEntry.getState().toString()); return doc; } public static SolrInputDocument rodaMemberToSolrDocument(RODAMember member) { SolrInputDocument doc = new SolrInputDocument(); doc.addField(RodaConstants.INDEX_UUID, member.getId()); doc.addField(RodaConstants.MEMBERS_ID, member.getId()); doc.addField(RodaConstants.MEMBERS_IS_ACTIVE, member.isActive()); doc.addField(RodaConstants.MEMBERS_IS_USER, member.isUser()); doc.addField(RodaConstants.MEMBERS_NAME, member.getName()); if (member.getDirectRoles() != null) { doc.addField(RodaConstants.MEMBERS_ROLES_DIRECT, new ArrayList<String>(member.getDirectRoles())); } if (member.getAllRoles() != null) { doc.addField(RodaConstants.MEMBERS_ROLES_ALL, new ArrayList<String>(member.getAllRoles())); } if (StringUtils.isNotBlank(member.getFullName())) { doc.addField(RodaConstants.MEMBERS_FULLNAME, member.getFullName()); } // Add user specific fields if (member instanceof User) { User user = (User) member; doc.addField(RodaConstants.MEMBERS_EMAIL, user.getEmail()); if (user.getGroups() != null) { doc.addField(RodaConstants.MEMBERS_GROUPS, new ArrayList<String>(user.getGroups())); } } // Add group specific fields if (member instanceof Group) { Group group = (Group) member; if (group.getUsers() != null) { doc.addField(RodaConstants.MEMBERS_USERS, new ArrayList<String>(group.getUsers())); } } return doc; } private static RODAMember solrDocumentToRodaMember(SolrDocument doc) { final String id = objectToString(doc.get(RodaConstants.INDEX_UUID), null); final String name = objectToString(doc.get(RodaConstants.MEMBERS_NAME), null); final boolean isActive = objectToBoolean(doc.get(RodaConstants.MEMBERS_IS_ACTIVE), Boolean.FALSE); final boolean isUser = objectToBoolean(doc.get(RodaConstants.MEMBERS_IS_USER), Boolean.FALSE); final String fullName = objectToString(doc.get(RodaConstants.MEMBERS_FULLNAME), null); final String email = objectToString(doc.get(RodaConstants.MEMBERS_EMAIL), null); final Set<String> groups = new HashSet<>(objectToListString(doc.get(RodaConstants.MEMBERS_GROUPS))); final Set<String> users = new HashSet<>(objectToListString(doc.get(RodaConstants.MEMBERS_USERS))); final Set<String> directRoles = new HashSet<>( objectToListString(doc.get(RodaConstants.MEMBERS_ROLES_DIRECT))); final Set<String> allRoles = new HashSet<>(objectToListString(doc.get(RodaConstants.MEMBERS_ROLES_ALL))); if (isUser) { User user = new User(); user.setId(id); user.setName(name); user.setActive(isActive); user.setFullName(fullName); user.setDirectRoles(directRoles); user.setAllRoles(allRoles); user.setEmail(email); user.setGroups(groups); return user; } else { Group group = new Group(); group.setId(id); group.setName(name); group.setActive(isActive); group.setFullName(fullName); group.setDirectRoles(directRoles); group.setAllRoles(allRoles); group.setUsers(users); return group; } } private static IndexedPreservationEvent solrDocumentToIndexedPreservationEvent(SolrDocument doc) { final String id = objectToString(doc.get(RodaConstants.INDEX_UUID), null); final String aipId = objectToString(doc.get(RodaConstants.PRESERVATION_EVENT_AIP_ID), null); final String representationUUID = objectToString( doc.get(RodaConstants.PRESERVATION_EVENT_REPRESENTATION_UUID), null); final String fileUUID = objectToString(doc.get(RodaConstants.PRESERVATION_EVENT_FILE_UUID), null); IndexedPreservationEvent ipe = new IndexedPreservationEvent(); ipe.setId(id); ipe.setAipID(aipId); ipe.setRepresentationUUID(representationUUID); ipe.setFileUUID(fileUUID); String objectClass = objectToString(doc.get(RodaConstants.PRESERVATION_EVENT_OBJECT_CLASS), null); if (StringUtils.isNotBlank(objectClass)) { ipe.setObjectClass(PreservationMetadataEventClass.valueOf(objectClass.toUpperCase())); } final Date eventDateTime = objectToDate(doc.get(RodaConstants.PRESERVATION_EVENT_DATETIME)); final String eventDetail = objectToString(doc.get(RodaConstants.PRESERVATION_EVENT_DETAIL), ""); final String eventType = objectToString(doc.get(RodaConstants.PRESERVATION_EVENT_TYPE), ""); final String eventOutcome = objectToString(doc.get(RodaConstants.PRESERVATION_EVENT_OUTCOME), ""); final List<String> agents = objectToListString( doc.get(RodaConstants.PRESERVATION_EVENT_LINKING_AGENT_IDENTIFIER)); final List<String> outcomes = objectToListString( doc.get(RodaConstants.PRESERVATION_EVENT_LINKING_OUTCOME_OBJECT_IDENTIFIER)); final List<String> sources = objectToListString( doc.get(RodaConstants.PRESERVATION_EVENT_LINKING_SOURCE_OBJECT_IDENTIFIER)); ipe.setEventDateTime(eventDateTime); ipe.setEventDetail(eventDetail); ipe.setEventType(eventType); ipe.setEventOutcome(eventOutcome); try { List<LinkingIdentifier> ids = new ArrayList<>(); for (String source : sources) { ids.add(JsonUtils.getObjectFromJson(source, LinkingIdentifier.class)); } ipe.setSourcesObjectIds(ids); } catch (GenericException | RuntimeException e) { LOGGER.error("Error setting event linking source", e); } try { List<LinkingIdentifier> ids = new ArrayList<>(); for (String outcome : outcomes) { ids.add(JsonUtils.getObjectFromJson(outcome, LinkingIdentifier.class)); } ipe.setOutcomeObjectIds(ids); } catch (GenericException | RuntimeException e) { LOGGER.error("Error setting event linking outcome", e); } try { List<LinkingIdentifier> ids = new ArrayList<>(); for (String agent : agents) { ids.add(JsonUtils.getObjectFromJson(agent, LinkingIdentifier.class)); } ipe.setLinkingAgentIds(ids); } catch (GenericException | RuntimeException e) { LOGGER.error("Error setting event linking agents", e); } return ipe; } private static IndexedPreservationAgent solrDocumentToIndexedPreservationAgent(SolrDocument doc) { final String id = objectToString(doc.get(RodaConstants.INDEX_UUID), null); final String name = objectToString(doc.get(RodaConstants.PRESERVATION_AGENT_NAME), null); final String type = objectToString(doc.get(RodaConstants.PRESERVATION_AGENT_TYPE), null); final String extension = objectToString(doc.get(RodaConstants.PRESERVATION_AGENT_EXTENSION), null); final String version = objectToString(doc.get(RodaConstants.PRESERVATION_AGENT_VERSION), null); final String note = objectToString(doc.get(RodaConstants.PRESERVATION_AGENT_NOTE), null); final List<String> roles = objectToListString(doc.get(RodaConstants.PRESERVATION_AGENT_ROLES)); IndexedPreservationAgent ipa = new IndexedPreservationAgent(); ipa.setId(id); ipa.setName(name); ipa.setType(type); ipa.setExtension(extension); ipa.setVersion(version); ipa.setNote(note); ipa.setRoles(roles); return ipa; } private static TransferredResource solrDocumentToTransferredResource(SolrDocument doc) { String uuid = objectToString(doc.get(RodaConstants.INDEX_UUID), null); String fullPath = objectToString(doc.get(RodaConstants.TRANSFERRED_RESOURCE_FULLPATH), null); String id = objectToString(doc.get(RodaConstants.TRANSFERRED_RESOURCE_ID), null); String parentId = null; String parentUUID = null; if (doc.containsKey(RodaConstants.TRANSFERRED_RESOURCE_PARENT_ID)) { parentId = objectToString(doc.get(RodaConstants.TRANSFERRED_RESOURCE_PARENT_ID), null); parentUUID = objectToString(doc.get(RodaConstants.TRANSFERRED_RESOURCE_PARENT_UUID), null); } String relativePath = objectToString(doc.get(RodaConstants.TRANSFERRED_RESOURCE_RELATIVEPATH), null); Date d = objectToDate(doc.get(RodaConstants.TRANSFERRED_RESOURCE_DATE)); if (d == null) { LOGGER.warn("Error parsing transferred resource date. Setting date to current date."); d = new Date(); } boolean isFile = objectToBoolean(doc.get(RodaConstants.TRANSFERRED_RESOURCE_ISFILE), Boolean.FALSE); long size = objectToLong(doc.get(RodaConstants.TRANSFERRED_RESOURCE_SIZE), 0L); String name = objectToString(doc.get(RodaConstants.TRANSFERRED_RESOURCE_NAME), null); List<String> ancestorsPath = objectToListString(doc.get(RodaConstants.TRANSFERRED_RESOURCE_ANCESTORS)); Date lastScanDate = objectToDate(doc.get(RodaConstants.TRANSFERRED_RESOURCE_LAST_SCAN_DATE)); TransferredResource tr = new TransferredResource(); tr.setUUID(uuid); tr.setFullPath(fullPath); tr.setId(id); tr.setCreationDate(d); tr.setName(name); tr.setRelativePath(relativePath); tr.setSize(size); tr.setParentId(parentId); tr.setParentUUID(parentUUID); tr.setFile(isFile); tr.setAncestorsPaths(ancestorsPath); tr.setLastScanDate(lastScanDate); return tr; } public static SolrInputDocument transferredResourceToSolrDocument(TransferredResource resource) { SolrInputDocument transferredResource = new SolrInputDocument(); transferredResource.addField(RodaConstants.INDEX_UUID, resource.getUUID()); transferredResource.addField(RodaConstants.TRANSFERRED_RESOURCE_ID, resource.getId()); transferredResource.addField(RodaConstants.TRANSFERRED_RESOURCE_FULLPATH, resource.getFullPath()); if (resource.getParentId() != null) { transferredResource.addField(RodaConstants.TRANSFERRED_RESOURCE_PARENT_ID, resource.getParentId()); transferredResource.addField(RodaConstants.TRANSFERRED_RESOURCE_PARENT_UUID, IdUtils.createUUID(resource.getParentId())); } if (resource.getRelativePath() != null) { transferredResource.addField(RodaConstants.TRANSFERRED_RESOURCE_RELATIVEPATH, resource.getRelativePath()); } transferredResource.addField(RodaConstants.TRANSFERRED_RESOURCE_DATE, resource.getCreationDate()); transferredResource.addField(RodaConstants.TRANSFERRED_RESOURCE_ISFILE, resource.isFile()); transferredResource.addField(RodaConstants.TRANSFERRED_RESOURCE_SIZE, resource.getSize()); transferredResource.addField(RodaConstants.TRANSFERRED_RESOURCE_NAME, resource.getName()); if (resource.getAncestorsPaths() != null && !resource.getAncestorsPaths().isEmpty()) { transferredResource.addField(RodaConstants.TRANSFERRED_RESOURCE_ANCESTORS, resource.getAncestorsPaths()); } transferredResource.addField(RodaConstants.TRANSFERRED_RESOURCE_LAST_SCAN_DATE, resource.getLastScanDate()); return transferredResource; } public static SolrInputDocument jobToSolrDocument(Job job) { SolrInputDocument doc = new SolrInputDocument(); doc.addField(RodaConstants.INDEX_UUID, job.getId()); doc.addField(RodaConstants.JOB_ID, job.getId()); doc.addField(RodaConstants.JOB_NAME, job.getName()); doc.addField(RodaConstants.JOB_USERNAME, job.getUsername()); doc.addField(RodaConstants.JOB_START_DATE, job.getStartDate()); doc.addField(RodaConstants.JOB_END_DATE, job.getEndDate()); doc.addField(RodaConstants.JOB_STATE, job.getState().toString()); doc.addField(RodaConstants.JOB_STATE_DETAILS, job.getStateDetails()); JobStats jobStats = job.getJobStats(); doc.addField(RodaConstants.JOB_COMPLETION_PERCENTAGE, jobStats.getCompletionPercentage()); doc.addField(RodaConstants.JOB_SOURCE_OBJECTS_COUNT, jobStats.getSourceObjectsCount()); doc.addField(RodaConstants.JOB_SOURCE_OBJECTS_WAITING_TO_BE_PROCESSED, jobStats.getSourceObjectsWaitingToBeProcessed()); doc.addField(RodaConstants.JOB_SOURCE_OBJECTS_BEING_PROCESSED, jobStats.getSourceObjectsBeingProcessed()); doc.addField(RodaConstants.JOB_SOURCE_OBJECTS_PROCESSED_WITH_SUCCESS, jobStats.getSourceObjectsProcessedWithSuccess()); doc.addField(RodaConstants.JOB_SOURCE_OBJECTS_PROCESSED_WITH_FAILURE, jobStats.getSourceObjectsProcessedWithFailure()); doc.addField(RodaConstants.JOB_OUTCOME_OBJECTS_WITH_MANUAL_INTERVENTION, jobStats.getOutcomeObjectsWithManualIntervention()); doc.addField(RodaConstants.JOB_PLUGIN_TYPE, job.getPluginType().toString()); doc.addField(RodaConstants.JOB_PLUGIN, job.getPlugin()); doc.addField(RodaConstants.JOB_PLUGIN_PARAMETERS, JsonUtils.getJsonFromObject(job.getPluginParameters())); doc.addField(RodaConstants.JOB_SOURCE_OBJECTS, JsonUtils.getJsonFromObject(job.getSourceObjects())); doc.addField(RodaConstants.JOB_OUTCOME_OBJECTS_CLASS, job.getOutcomeObjectsClass()); return doc; } public static Job solrDocumentToJob(SolrDocument doc, List<String> fieldsToReturn) { Job job = new Job(); job.setId(objectToString(doc.get(RodaConstants.INDEX_UUID), null)); job.setName(objectToString(doc.get(RodaConstants.JOB_NAME), null)); job.setUsername(objectToString(doc.get(RodaConstants.JOB_USERNAME), null)); job.setStartDate(objectToDate(doc.get(RodaConstants.JOB_START_DATE))); job.setEndDate(objectToDate(doc.get(RodaConstants.JOB_END_DATE))); if (doc.containsKey(RodaConstants.JOB_STATE)) { job.setState(JOB_STATE.valueOf(objectToString(doc.get(RodaConstants.JOB_STATE), null))); } job.setStateDetails(objectToString(doc.get(RodaConstants.JOB_STATE_DETAILS), null)); JobStats jobStats = job.getJobStats(); jobStats.setCompletionPercentage(objectToInteger(doc.get(RodaConstants.JOB_COMPLETION_PERCENTAGE), 0)); jobStats.setSourceObjectsCount(objectToInteger(doc.get(RodaConstants.JOB_SOURCE_OBJECTS_COUNT), 0)); jobStats.setSourceObjectsWaitingToBeProcessed( objectToInteger(doc.get(RodaConstants.JOB_SOURCE_OBJECTS_WAITING_TO_BE_PROCESSED), 0)); jobStats.setSourceObjectsBeingProcessed( objectToInteger(doc.get(RodaConstants.JOB_SOURCE_OBJECTS_BEING_PROCESSED), 0)); jobStats.setSourceObjectsProcessedWithSuccess( objectToInteger(doc.get(RodaConstants.JOB_SOURCE_OBJECTS_PROCESSED_WITH_SUCCESS), 0)); jobStats.setSourceObjectsProcessedWithFailure( objectToInteger(doc.get(RodaConstants.JOB_SOURCE_OBJECTS_PROCESSED_WITH_FAILURE), 0)); jobStats.setOutcomeObjectsWithManualIntervention( objectToInteger(doc.get(RodaConstants.JOB_OUTCOME_OBJECTS_WITH_MANUAL_INTERVENTION), 0)); if (doc.containsKey(RodaConstants.JOB_PLUGIN_TYPE)) { job.setPluginType(PluginType.valueOf(objectToString(doc.get(RodaConstants.JOB_PLUGIN_TYPE), null))); } job.setPlugin(objectToString(doc.get(RodaConstants.JOB_PLUGIN), null)); if (fieldsToReturn.isEmpty() || fieldsToReturn.contains(RodaConstants.JOB_PLUGIN_PARAMETERS)) { job.setPluginParameters( JsonUtils.getMapFromJson(objectToString(doc.get(RodaConstants.JOB_PLUGIN_PARAMETERS), ""))); } try { if (fieldsToReturn.isEmpty() || fieldsToReturn.contains(RodaConstants.JOB_SOURCE_OBJECTS)) { job.setSourceObjects(JsonUtils.getObjectFromJson( objectToString(doc.get(RodaConstants.JOB_SOURCE_OBJECTS), ""), SelectedItems.class)); } } catch (GenericException e) { LOGGER.error("Error parsing report in job objects", e); } job.setOutcomeObjectsClass(objectToString(doc.get(RodaConstants.JOB_OUTCOME_OBJECTS_CLASS), null)); return job; } public static <T extends Serializable> SolrInputDocument jobReportToSolrDocument(Report jobReport, Job job, SolrClient index) { SolrInputDocument doc = new SolrInputDocument(); doc.addField(RodaConstants.INDEX_UUID, jobReport.getId()); doc.addField(RodaConstants.JOB_REPORT_ID, jobReport.getId()); doc.addField(RodaConstants.JOB_REPORT_JOB_ID, jobReport.getJobId()); doc.addField(RodaConstants.JOB_REPORT_SOURCE_OBJECT_ID, jobReport.getSourceObjectId()); doc.addField(RodaConstants.JOB_REPORT_SOURCE_OBJECT_ORIGINAL_IDS, jobReport.getSourceObjectOriginalIds()); doc.addField(RodaConstants.JOB_REPORT_SOURCE_OBJECT_ORIGINAL_NAME, jobReport.getSourceObjectOriginalName()); doc.addField(RodaConstants.JOB_REPORT_OUTCOME_OBJECT_ID, jobReport.getOutcomeObjectId()); doc.addField(RodaConstants.JOB_REPORT_OUTCOME_OBJECT_STATE, jobReport.getOutcomeObjectState().toString()); doc.addField(RodaConstants.JOB_REPORT_TITLE, jobReport.getTitle()); doc.addField(RodaConstants.JOB_REPORT_DATE_CREATED, jobReport.getDateCreated()); doc.addField(RodaConstants.JOB_REPORT_DATE_UPDATED, jobReport.getDateUpdated()); doc.addField(RodaConstants.JOB_REPORT_COMPLETION_PERCENTAGE, jobReport.getCompletionPercentage()); doc.addField(RodaConstants.JOB_REPORT_STEPS_COMPLETED, jobReport.getStepsCompleted()); doc.addField(RodaConstants.JOB_REPORT_TOTAL_STEPS, jobReport.getTotalSteps()); doc.addField(RodaConstants.JOB_REPORT_PLUGIN, jobReport.getPlugin()); doc.addField(RodaConstants.JOB_REPORT_PLUGIN_NAME, jobReport.getPluginName()); doc.addField(RodaConstants.JOB_REPORT_PLUGIN_VERSION, jobReport.getPluginVersion()); doc.addField(RodaConstants.JOB_REPORT_PLUGIN_STATE, jobReport.getPluginState().toString()); doc.addField(RodaConstants.JOB_REPORT_PLUGIN_DETAILS, jobReport.getPluginDetails()); doc.addField(RodaConstants.JOB_REPORT_HTML_PLUGIN_DETAILS, jobReport.isHtmlPluginDetails()); doc.addField(RodaConstants.JOB_REPORT_REPORTS, JsonUtils.getJsonFromObject(jobReport.getReports())); doc.addField(RodaConstants.JOB_REPORT_SOURCE_OBJECT_CLASS, jobReport.getSourceObjectClass()); doc.addField(RodaConstants.JOB_REPORT_OUTCOME_OBJECT_CLASS, jobReport.getOutcomeObjectClass()); doc.addField(RodaConstants.JOB_REPORT_JOB_NAME, job.getName()); doc.addField(RodaConstants.JOB_REPORT_SOURCE_OBJECT_LABEL, getObjectLabel(index, jobReport.getSourceObjectClass(), jobReport.getSourceObjectId())); doc.addField(RodaConstants.JOB_REPORT_OUTCOME_OBJECT_LABEL, getObjectLabel(index, jobReport.getOutcomeObjectClass(), jobReport.getOutcomeObjectId())); return doc; } private static <T extends Serializable> String getObjectLabel(SolrClient index, String className, String id) { if (StringUtils.isNotBlank(className) && StringUtils.isNotBlank(id)) { try { Class<T> objectClass = (Class<T>) Class.forName(className); if (objectClass.equals(LiteRODAObject.class)) { return null; } String field = getIndexName(objectClass).get(0); SolrDocument doc = index.getById(field, id); if (doc != null) { if (objectClass.equals(AIP.class)) { return objectToString(doc.get(RodaConstants.AIP_TITLE), null); } else if (objectClass.equals(Representation.class)) { return objectToString(doc.get(RodaConstants.REPRESENTATION_ID), null); } else if (objectClass.equals(File.class)) { return objectToString(doc.get(RodaConstants.FILE_FILE_ID), null); } else if (objectClass.equals(TransferredResource.class)) { return objectToString(doc.get(RodaConstants.TRANSFERRED_RESOURCE_ID), null); } else if (objectClass.equals(DIP.class)) { return objectToString(doc.get(RodaConstants.DIP_TITLE), null); } else if (objectClass.equals(DIPFile.class)) { return objectToString(doc.get(RodaConstants.DIPFILE_ID), null); } else { return objectToString(doc.get(RodaConstants.INDEX_UUID), null); } } } catch (ClassNotFoundException | GenericException | SolrServerException | IOException e) { LOGGER.error("Could not return object label of {} {}", className, id, e); } } return null; } private static IndexedReport solrDocumentToJobReport(SolrDocument doc, List<String> fieldsToReturn) { IndexedReport jobReport = new IndexedReport(); jobReport.setId(objectToString(doc.get(RodaConstants.INDEX_UUID), null)); jobReport.setJobId(objectToString(doc.get(RodaConstants.JOB_REPORT_JOB_ID), null)); jobReport.setSourceObjectId(objectToString(doc.get(RodaConstants.JOB_REPORT_SOURCE_OBJECT_ID), null)); jobReport.setSourceObjectClass(objectToString(doc.get(RodaConstants.JOB_REPORT_SOURCE_OBJECT_CLASS), null)); jobReport.setSourceObjectOriginalIds( objectToListString(doc.get(RodaConstants.JOB_REPORT_SOURCE_OBJECT_ORIGINAL_IDS))); jobReport.setSourceObjectOriginalName( objectToString(doc.get(RodaConstants.JOB_REPORT_SOURCE_OBJECT_ORIGINAL_NAME), null)); jobReport.setOutcomeObjectId(objectToString(doc.get(RodaConstants.JOB_REPORT_OUTCOME_OBJECT_ID), null)); jobReport.setOutcomeObjectClass( objectToString(doc.get(RodaConstants.JOB_REPORT_OUTCOME_OBJECT_CLASS), null)); if (doc.containsKey(RodaConstants.JOB_REPORT_OUTCOME_OBJECT_STATE)) { jobReport.setOutcomeObjectState(AIPState.valueOf(objectToString( doc.get(RodaConstants.JOB_REPORT_OUTCOME_OBJECT_STATE), AIPState.getDefault().toString()))); } jobReport.setTitle(objectToString(doc.get(RodaConstants.JOB_REPORT_TITLE), null)); jobReport.setDateCreated(objectToDate(doc.get(RodaConstants.JOB_REPORT_DATE_CREATED))); jobReport.setDateUpdated(objectToDate(doc.get(RodaConstants.JOB_REPORT_DATE_UPDATED))); jobReport.setCompletionPercentage( objectToInteger(doc.get(RodaConstants.JOB_REPORT_COMPLETION_PERCENTAGE), 0)); jobReport.setStepsCompleted(objectToInteger(doc.get(RodaConstants.JOB_REPORT_STEPS_COMPLETED), 0)); jobReport.setTotalSteps(objectToInteger(doc.get(RodaConstants.JOB_REPORT_TOTAL_STEPS), 0)); jobReport.setPlugin(objectToString(doc.get(RodaConstants.JOB_REPORT_PLUGIN), null)); jobReport.setPluginName(objectToString(doc.get(RodaConstants.JOB_REPORT_PLUGIN_NAME), null)); jobReport.setPluginVersion(objectToString(doc.get(RodaConstants.JOB_REPORT_PLUGIN_VERSION), null)); jobReport.setPluginState( PluginState.valueOf(objectToString(doc.get(RodaConstants.JOB_REPORT_PLUGIN_STATE), null))); jobReport.setPluginDetails(objectToString(doc.get(RodaConstants.JOB_REPORT_PLUGIN_DETAILS), null)); jobReport.setHtmlPluginDetails( objectToBoolean(doc.get(RodaConstants.JOB_REPORT_HTML_PLUGIN_DETAILS), false)); try { if (fieldsToReturn.isEmpty() || fieldsToReturn.contains(RodaConstants.JOB_REPORT_REPORTS)) { jobReport.setReports(JsonUtils.getListFromJson( objectToString(doc.get(RodaConstants.JOB_REPORT_REPORTS), ""), Report.class)); } } catch (GenericException e) { LOGGER.error("Error parsing report in job report", e); } jobReport.setJobName(objectToString(doc.get(RodaConstants.JOB_REPORT_JOB_NAME), null)); jobReport.setSourceObjectLabel(objectToString(doc.get(RodaConstants.JOB_REPORT_SOURCE_OBJECT_LABEL), null)); jobReport.setOutcomeObjectLabel( objectToString(doc.get(RodaConstants.JOB_REPORT_OUTCOME_OBJECT_LABEL), null)); return jobReport; } public static SolrInputDocument riskToSolrDocument(Risk risk, int incidences) { SolrInputDocument doc = new SolrInputDocument(); doc.addField(RodaConstants.INDEX_UUID, risk.getId()); doc.addField(RodaConstants.RISK_ID, risk.getId()); doc.addField(RodaConstants.RISK_NAME, risk.getName()); doc.addField(RodaConstants.RISK_DESCRIPTION, risk.getDescription()); doc.addField(RodaConstants.RISK_IDENTIFIED_ON, risk.getIdentifiedOn()); doc.addField(RodaConstants.RISK_IDENTIFIED_BY, risk.getIdentifiedBy()); doc.addField(RodaConstants.RISK_CATEGORY, risk.getCategory()); doc.addField(RodaConstants.RISK_NOTES, risk.getNotes()); doc.addField(RodaConstants.RISK_PRE_MITIGATION_PROBABILITY, risk.getPreMitigationProbability()); doc.addField(RodaConstants.RISK_PRE_MITIGATION_IMPACT, risk.getPreMitigationImpact()); doc.addField(RodaConstants.RISK_PRE_MITIGATION_SEVERITY, risk.getPreMitigationSeverity()); doc.addField(RodaConstants.RISK_PRE_MITIGATION_SEVERITY_LEVEL, risk.getPreMitigationSeverityLevel().toString()); doc.addField(RodaConstants.RISK_PRE_MITIGATION_NOTES, risk.getPreMitigationNotes()); doc.addField(RodaConstants.RISK_POST_MITIGATION_PROBABILITY, risk.getPostMitigationProbability()); doc.addField(RodaConstants.RISK_POST_MITIGATION_IMPACT, risk.getPostMitigationImpact()); doc.addField(RodaConstants.RISK_POST_MITIGATION_SEVERITY, risk.getPostMitigationSeverity()); if (risk.getPostMitigationSeverityLevel() != null) { doc.addField(RodaConstants.RISK_POST_MITIGATION_SEVERITY_LEVEL, risk.getPostMitigationSeverityLevel().toString()); } doc.addField(RodaConstants.RISK_CURRENT_SEVERITY_LEVEL, risk.getCurrentSeverityLevel()); doc.addField(RodaConstants.RISK_POST_MITIGATION_NOTES, risk.getPostMitigationNotes()); doc.addField(RodaConstants.RISK_MITIGATION_STRATEGY, risk.getMitigationStrategy()); doc.addField(RodaConstants.RISK_MITIGATION_OWNER_TYPE, risk.getMitigationOwnerType()); doc.addField(RodaConstants.RISK_MITIGATION_OWNER, risk.getMitigationOwner()); doc.addField(RodaConstants.RISK_MITIGATION_RELATED_EVENT_IDENTIFIER_TYPE, risk.getMitigationRelatedEventIdentifierType()); doc.addField(RodaConstants.RISK_MITIGATION_RELATED_EVENT_IDENTIFIER_VALUE, risk.getMitigationRelatedEventIdentifierValue()); doc.addField(RodaConstants.RISK_CREATED_ON, risk.getCreatedOn()); doc.addField(RodaConstants.RISK_CREATED_BY, risk.getCreatedBy()); doc.addField(RodaConstants.RISK_UPDATED_ON, risk.getUpdatedOn()); doc.addField(RodaConstants.RISK_UPDATED_BY, risk.getUpdatedBy()); if (risk instanceof IndexedRisk) { doc.addField(RodaConstants.RISK_INCIDENCES_COUNT, ((IndexedRisk) risk).getIncidencesCount()); doc.addField(RodaConstants.RISK_UNMITIGATED_INCIDENCES_COUNT, ((IndexedRisk) risk).getUnmitigatedIncidencesCount()); } else { doc.addField(RodaConstants.RISK_INCIDENCES_COUNT, incidences); doc.addField(RodaConstants.RISK_UNMITIGATED_INCIDENCES_COUNT, incidences); } return doc; } public static IndexedRisk solrDocumentToRisk(SolrDocument doc) { IndexedRisk risk = new IndexedRisk(); risk.setId(objectToString(doc.get(RodaConstants.INDEX_UUID), null)); risk.setName(objectToString(doc.get(RodaConstants.RISK_NAME), null)); risk.setDescription(objectToString(doc.get(RodaConstants.RISK_DESCRIPTION), null)); risk.setIdentifiedOn(objectToDate(doc.get(RodaConstants.RISK_IDENTIFIED_ON))); risk.setIdentifiedBy(objectToString(doc.get(RodaConstants.RISK_IDENTIFIED_BY), null)); risk.setCategory(objectToString(doc.get(RodaConstants.RISK_CATEGORY), null)); risk.setNotes(objectToString(doc.get(RodaConstants.RISK_NOTES), null)); risk.setPreMitigationProbability( objectToInteger(doc.get(RodaConstants.RISK_PRE_MITIGATION_PROBABILITY), 0)); risk.setPreMitigationImpact(objectToInteger(doc.get(RodaConstants.RISK_PRE_MITIGATION_IMPACT), 0)); risk.setPreMitigationSeverity(objectToInteger(doc.get(RodaConstants.RISK_PRE_MITIGATION_SEVERITY), 0)); if (doc.containsKey(RodaConstants.RISK_PRE_MITIGATION_SEVERITY_LEVEL)) { risk.setPreMitigationSeverityLevel(objectToEnum( doc.get(RodaConstants.RISK_PRE_MITIGATION_SEVERITY_LEVEL), Risk.SEVERITY_LEVEL.class, null)); } risk.setPreMitigationNotes(objectToString(doc.get(RodaConstants.RISK_PRE_MITIGATION_NOTES), null)); risk.setPostMitigationProbability( objectToInteger(doc.get(RodaConstants.RISK_POST_MITIGATION_PROBABILITY), 0)); risk.setPostMitigationImpact(objectToInteger(doc.get(RodaConstants.RISK_POST_MITIGATION_IMPACT), 0)); risk.setPostMitigationSeverity(objectToInteger(doc.get(RodaConstants.RISK_POST_MITIGATION_SEVERITY), 0)); if (doc.containsKey(RodaConstants.RISK_POST_MITIGATION_SEVERITY_LEVEL)) { risk.setPostMitigationSeverityLevel(objectToEnum( doc.get(RodaConstants.RISK_POST_MITIGATION_SEVERITY_LEVEL), Risk.SEVERITY_LEVEL.class, null)); } risk.setPostMitigationNotes(objectToString(doc.get(RodaConstants.RISK_POST_MITIGATION_NOTES), null)); risk.setMitigationStrategy(objectToString(doc.get(RodaConstants.RISK_MITIGATION_STRATEGY), null)); risk.setMitigationOwnerType(objectToString(doc.get(RodaConstants.RISK_MITIGATION_OWNER_TYPE), null)); risk.setMitigationOwner(objectToString(doc.get(RodaConstants.RISK_MITIGATION_OWNER), null)); risk.setMitigationRelatedEventIdentifierType( objectToString(doc.get(RodaConstants.RISK_MITIGATION_RELATED_EVENT_IDENTIFIER_TYPE), null)); risk.setMitigationRelatedEventIdentifierValue( objectToString(doc.get(RodaConstants.RISK_MITIGATION_RELATED_EVENT_IDENTIFIER_VALUE), null)); risk.setCreatedOn(objectToDate(doc.get(RodaConstants.RISK_CREATED_ON))); risk.setCreatedBy(objectToString(doc.get(RodaConstants.RISK_CREATED_BY), null)); risk.setUpdatedOn(objectToDate(doc.get(RodaConstants.RISK_UPDATED_ON))); risk.setUpdatedBy(objectToString(doc.get(RodaConstants.RISK_UPDATED_BY), null)); risk.setIncidencesCount(objectToInteger(doc.get(RodaConstants.RISK_INCIDENCES_COUNT), 0)); risk.setUnmitigatedIncidencesCount( objectToInteger(doc.get(RodaConstants.RISK_UNMITIGATED_INCIDENCES_COUNT), 0)); return risk; } public static SolrInputDocument formatToSolrDocument(Format format) { SolrInputDocument doc = new SolrInputDocument(); doc.addField(RodaConstants.INDEX_UUID, format.getId()); doc.addField(RodaConstants.FORMAT_ID, format.getId()); doc.addField(RodaConstants.FORMAT_NAME, format.getName()); doc.addField(RodaConstants.FORMAT_NAME_SORT, format.getName()); doc.addField(RodaConstants.FORMAT_DEFINITION, format.getDefinition()); doc.addField(RodaConstants.FORMAT_CATEGORY, format.getCategories()); doc.addField(RodaConstants.FORMAT_CATEGORY_SORT, (format.getCategories() != null && !format.getCategories().isEmpty()) ? format.getCategories().get(0) : null); doc.addField(RodaConstants.FORMAT_LATEST_VERSION, format.getLatestVersion()); if (format.getPopularity() != null) { doc.addField(RodaConstants.FORMAT_POPULARITY, format.getPopularity()); } doc.addField(RodaConstants.FORMAT_DEVELOPER, format.getDeveloper()); doc.addField(RodaConstants.FORMAT_INITIAL_RELEASE, format.getInitialRelease()); doc.addField(RodaConstants.FORMAT_STANDARD, format.getStandard()); doc.addField(RodaConstants.FORMAT_IS_OPEN_FORMAT, format.isOpenFormat()); doc.addField(RodaConstants.FORMAT_WEBSITE, format.getWebsites()); doc.addField(RodaConstants.FORMAT_PROVENANCE_INFORMATION, format.getProvenanceInformation()); doc.addField(RodaConstants.FORMAT_EXTENSIONS, format.getExtensions()); doc.addField(RodaConstants.FORMAT_MIMETYPES, format.getMimetypes()); doc.addField(RodaConstants.FORMAT_PRONOMS, format.getPronoms()); doc.addField(RodaConstants.FORMAT_UTIS, format.getUtis()); doc.addField(RodaConstants.FORMAT_ALTERNATIVE_DESIGNATIONS, format.getAlternativeDesignations()); doc.addField(RodaConstants.FORMAT_VERSIONS, format.getVersions()); return doc; } public static Format solrDocumentToFormat(SolrDocument doc) { Format format = new Format(); format.setId(objectToString(doc.get(RodaConstants.INDEX_UUID), null)); format.setName(objectToString(doc.get(RodaConstants.FORMAT_NAME), null)); format.setDefinition(objectToString(doc.get(RodaConstants.FORMAT_DEFINITION), null)); format.setCategories(objectToListString(doc.get(RodaConstants.FORMAT_CATEGORY))); format.setLatestVersion(objectToString(doc.get(RodaConstants.FORMAT_LATEST_VERSION), null)); format.setPopularity(objectToInteger(doc.get(RodaConstants.FORMAT_POPULARITY), null)); format.setDeveloper(objectToString(doc.get(RodaConstants.FORMAT_DEVELOPER), null)); format.setInitialRelease(objectToDate(doc.get(RodaConstants.FORMAT_INITIAL_RELEASE))); format.setStandard(objectToString(doc.get(RodaConstants.FORMAT_STANDARD), null)); format.setOpenFormat(objectToBoolean(doc.get(RodaConstants.FORMAT_IS_OPEN_FORMAT), Boolean.FALSE)); format.setWebsites(objectToListString(doc.get(RodaConstants.FORMAT_WEBSITE))); format.setProvenanceInformation(objectToString(doc.get(RodaConstants.FORMAT_PROVENANCE_INFORMATION), null)); format.setExtensions(objectToListString(doc.get(RodaConstants.FORMAT_EXTENSIONS))); format.setMimetypes(objectToListString(doc.get(RodaConstants.FORMAT_MIMETYPES))); format.setPronoms(objectToListString(doc.get(RodaConstants.FORMAT_PRONOMS))); format.setUtis(objectToListString(doc.get(RodaConstants.FORMAT_UTIS))); format.setAlternativeDesignations( objectToListString(doc.get(RodaConstants.FORMAT_ALTERNATIVE_DESIGNATIONS))); format.setVersions(objectToListString(doc.get(RodaConstants.FORMAT_VERSIONS))); return format; } public static SolrInputDocument notificationToSolrDocument(Notification notification) { SolrInputDocument doc = new SolrInputDocument(); doc.addField(RodaConstants.INDEX_UUID, notification.getId()); doc.addField(RodaConstants.NOTIFICATION_ID, notification.getId()); doc.addField(RodaConstants.NOTIFICATION_SUBJECT, notification.getSubject()); doc.addField(RodaConstants.NOTIFICATION_BODY, notification.getBody()); doc.addField(RodaConstants.NOTIFICATION_SENT_ON, notification.getSentOn()); doc.addField(RodaConstants.NOTIFICATION_FROM_USER, notification.getFromUser()); doc.addField(RodaConstants.NOTIFICATION_RECIPIENT_USERS, notification.getRecipientUsers()); doc.addField(RodaConstants.NOTIFICATION_ACKNOWLEDGE_TOKEN, notification.getAcknowledgeToken()); doc.addField(RodaConstants.NOTIFICATION_IS_ACKNOWLEDGED, notification.isAcknowledged()); doc.addField(RodaConstants.NOTIFICATION_ACKNOWLEDGED_USERS, JsonUtils.getJsonFromObject(notification.getAcknowledgedUsers())); doc.addField(RodaConstants.NOTIFICATION_STATE, notification.getState().toString()); return doc; } public static Notification solrDocumentToNotification(SolrDocument doc, List<String> fieldsToReturn) { Notification notification = new Notification(); notification.setId(objectToString(doc.get(RodaConstants.INDEX_UUID), null)); notification.setSubject(objectToString(doc.get(RodaConstants.NOTIFICATION_SUBJECT), null)); notification.setBody(objectToString(doc.get(RodaConstants.NOTIFICATION_BODY), null)); notification.setSentOn(objectToDate(doc.get(RodaConstants.NOTIFICATION_SENT_ON))); notification.setFromUser(objectToString(doc.get(RodaConstants.NOTIFICATION_FROM_USER), null)); notification.setRecipientUsers(objectToListString(doc.get(RodaConstants.NOTIFICATION_RECIPIENT_USERS))); notification .setAcknowledgeToken(objectToString(doc.get(RodaConstants.NOTIFICATION_ACKNOWLEDGE_TOKEN), null)); notification.setAcknowledged( objectToBoolean(doc.get(RodaConstants.NOTIFICATION_IS_ACKNOWLEDGED), Boolean.FALSE)); if (fieldsToReturn.isEmpty() || fieldsToReturn.contains(RodaConstants.NOTIFICATION_ACKNOWLEDGED_USERS)) { notification.setAcknowledgedUsers(JsonUtils .getMapFromJson(objectToString(doc.get(RodaConstants.NOTIFICATION_ACKNOWLEDGED_USERS), ""))); } if (doc.containsKey(RodaConstants.NOTIFICATION_STATE)) { String defaultState = Notification.NOTIFICATION_STATE.COMPLETED.toString(); notification.setState(Notification.NOTIFICATION_STATE .valueOf(objectToString(doc.get(RodaConstants.NOTIFICATION_STATE), defaultState))); } return notification; } public static SolrInputDocument riskIncidenceToSolrDocument(RiskIncidence incidence) { SolrInputDocument doc = new SolrInputDocument(); doc.addField(RodaConstants.INDEX_UUID, incidence.getId()); doc.addField(RodaConstants.RISK_INCIDENCE_ID, incidence.getId()); doc.addField(RodaConstants.RISK_INCIDENCE_AIP_ID, incidence.getAipId()); doc.addField(RodaConstants.RISK_INCIDENCE_REPRESENTATION_ID, incidence.getRepresentationId()); doc.addField(RodaConstants.RISK_INCIDENCE_FILE_PATH, incidence.getFilePath()); doc.addField(RodaConstants.RISK_INCIDENCE_FILE_ID, incidence.getFileId()); doc.addField(RodaConstants.RISK_INCIDENCE_OBJECT_CLASS, incidence.getObjectClass()); doc.addField(RodaConstants.RISK_INCIDENCE_RISK_ID, incidence.getRiskId()); doc.addField(RodaConstants.RISK_INCIDENCE_DESCRIPTION, incidence.getDescription()); doc.addField(RodaConstants.RISK_INCIDENCE_BYPLUGIN, incidence.isByPlugin()); doc.addField(RodaConstants.RISK_INCIDENCE_STATUS, incidence.getStatus().toString()); doc.addField(RodaConstants.RISK_INCIDENCE_SEVERITY, incidence.getSeverity().toString()); doc.addField(RodaConstants.RISK_INCIDENCE_DETECTED_ON, incidence.getDetectedOn()); doc.addField(RodaConstants.RISK_INCIDENCE_DETECTED_BY, incidence.getDetectedBy()); doc.addField(RodaConstants.RISK_INCIDENCE_MITIGATED_ON, incidence.getMitigatedOn()); doc.addField(RodaConstants.RISK_INCIDENCE_MITIGATED_BY, incidence.getMitigatedBy()); doc.addField(RodaConstants.RISK_INCIDENCE_MITIGATED_DESCRIPTION, incidence.getMitigatedDescription()); doc.addField(RodaConstants.RISK_INCIDENCE_FILE_PATH_COMPUTED, StringUtils.join(incidence.getFilePath(), RodaConstants.RISK_INCIDENCE_FILE_PATH_COMPUTED_SEPARATOR)); return doc; } public static RiskIncidence solrDocumentToRiskIncidence(SolrDocument doc) { RiskIncidence incidence = new RiskIncidence(); incidence.setId(objectToString(doc.get(RodaConstants.INDEX_UUID), null)); incidence.setAipId(objectToString(doc.get(RodaConstants.RISK_INCIDENCE_AIP_ID), null)); incidence .setRepresentationId(objectToString(doc.get(RodaConstants.RISK_INCIDENCE_REPRESENTATION_ID), null)); incidence.setFilePath(objectToListString(doc.get(RodaConstants.RISK_INCIDENCE_FILE_PATH))); incidence.setFileId(objectToString(doc.get(RodaConstants.RISK_INCIDENCE_FILE_ID), null)); incidence.setObjectClass(objectToString(doc.get(RodaConstants.RISK_INCIDENCE_OBJECT_CLASS), null)); incidence.setRiskId(objectToString(doc.get(RodaConstants.RISK_INCIDENCE_RISK_ID), null)); incidence.setDescription(objectToString(doc.get(RodaConstants.RISK_INCIDENCE_DESCRIPTION), null)); if (doc.containsKey(RodaConstants.RISK_INCIDENCE_STATUS)) { incidence.setStatus(RiskIncidence.INCIDENCE_STATUS .valueOf(objectToString(doc.get(RodaConstants.RISK_INCIDENCE_STATUS), RiskIncidence.INCIDENCE_STATUS.UNMITIGATED.toString()))); } if (doc.containsKey(RodaConstants.RISK_INCIDENCE_SEVERITY)) { incidence.setSeverity(Risk.SEVERITY_LEVEL.valueOf(objectToString( doc.get(RodaConstants.RISK_INCIDENCE_SEVERITY), Risk.SEVERITY_LEVEL.MODERATE.toString()))); } incidence.setDetectedOn(objectToDate(doc.get(RodaConstants.RISK_INCIDENCE_DETECTED_ON))); incidence.setDetectedBy(objectToString(doc.get(RodaConstants.RISK_INCIDENCE_DETECTED_BY), null)); incidence.setMitigatedOn(objectToDate(doc.get(RodaConstants.RISK_INCIDENCE_MITIGATED_ON))); incidence.setMitigatedBy(objectToString(doc.get(RodaConstants.RISK_INCIDENCE_MITIGATED_BY), null)); incidence.setMitigatedDescription( objectToString(doc.get(RodaConstants.RISK_INCIDENCE_MITIGATED_DESCRIPTION), null)); return incidence; } public static SolrInputDocument dipToSolrDocument(DIP dip) { SolrInputDocument doc = new SolrInputDocument(); doc.addField(RodaConstants.INDEX_UUID, dip.getId()); doc.addField(RodaConstants.DIP_ID, dip.getId()); doc.addField(RodaConstants.DIP_TITLE, dip.getTitle()); doc.addField(RodaConstants.DIP_DESCRIPTION, dip.getDescription()); doc.addField(RodaConstants.DIP_TYPE, dip.getType()); doc.addField(RodaConstants.DIP_DATE_CREATED, dip.getDateCreated()); doc.addField(RodaConstants.DIP_LAST_MODIFIED, dip.getLastModified()); doc.addField(RodaConstants.DIP_IS_PERMANENT, dip.getIsPermanent()); doc.addField(RodaConstants.DIP_PROPERTIES, JsonUtils.getJsonFromObject(dip.getProperties())); doc.addField(RodaConstants.DIP_AIP_IDS, JsonUtils.getJsonFromObject(dip.getAipIds())); doc.addField(RodaConstants.DIP_REPRESENTATION_IDS, JsonUtils.getJsonFromObject(dip.getRepresentationIds())); doc.addField(RodaConstants.DIP_FILE_IDS, JsonUtils.getJsonFromObject(dip.getFileIds())); List<String> allAipUUIDs = new ArrayList<>(); List<String> allRepresentationUUIDs = new ArrayList<>(); List<String> aipUUIDs = new ArrayList<>(); for (AIPLink aip : dip.getAipIds()) { aipUUIDs.add(aip.getAipId()); } allAipUUIDs.addAll(aipUUIDs); List<String> representationUUIDs = new ArrayList<>(); for (RepresentationLink rep : dip.getRepresentationIds()) { representationUUIDs.add(IdUtils.getRepresentationId(rep)); if (!allAipUUIDs.contains(rep.getAipId())) { allAipUUIDs.add(rep.getAipId()); } } allRepresentationUUIDs.addAll(representationUUIDs); List<String> fileUUIDs = new ArrayList<>(); for (FileLink file : dip.getFileIds()) { fileUUIDs.add(IdUtils.getFileId(file)); if (!allAipUUIDs.contains(file.getAipId())) { allAipUUIDs.add(file.getAipId()); } String repUUID = IdUtils.getRepresentationId(file.getAipId(), file.getRepresentationId()); if (!allRepresentationUUIDs.contains(repUUID)) { allRepresentationUUIDs.add(repUUID); } } doc.addField(RodaConstants.DIP_AIP_UUIDS, aipUUIDs); doc.addField(RodaConstants.DIP_REPRESENTATION_UUIDS, representationUUIDs); doc.addField(RodaConstants.DIP_FILE_UUIDS, fileUUIDs); doc.addField(RodaConstants.DIP_ALL_AIP_UUIDS, allAipUUIDs); doc.addField(RodaConstants.DIP_ALL_REPRESENTATION_UUIDS, allRepresentationUUIDs); setPermissions(dip.getPermissions(), doc); OptionalWithCause<String> openURL = DIPUtils.getCompleteOpenExternalURL(dip); if (openURL.isPresent()) { doc.addField(RodaConstants.DIP_OPEN_EXTERNAL_URL, openURL.get()); } else if (openURL.getCause() != null) { LOGGER.error("Error indexing DIP open external URL", openURL.getCause()); } return doc; } public static IndexedDIP solrDocumentToIndexedDIP(SolrDocument doc, List<String> fieldsToReturn) { IndexedDIP dip = new IndexedDIP(); dip.setId(objectToString(doc.get(RodaConstants.INDEX_UUID), null)); dip.setTitle(objectToString(doc.get(RodaConstants.DIP_TITLE), null)); dip.setDescription(objectToString(doc.get(RodaConstants.DIP_DESCRIPTION), null)); dip.setType(objectToString(doc.get(RodaConstants.DIP_TYPE), null)); dip.setDateCreated(objectToDate(doc.get(RodaConstants.DIP_DATE_CREATED))); dip.setLastModified(objectToDate(doc.get(RodaConstants.DIP_LAST_MODIFIED))); dip.setIsPermanent(objectToBoolean(doc.get(RodaConstants.DIP_IS_PERMANENT), Boolean.FALSE)); boolean emptyFields = fieldsToReturn.isEmpty(); if (emptyFields || fieldsToReturn.contains(RodaConstants.DIP_PROPERTIES)) { dip.setProperties(JsonUtils.getMapFromJson(objectToString(doc.get(RodaConstants.DIP_PROPERTIES), ""))); } try { if (emptyFields || fieldsToReturn.contains(RodaConstants.DIP_AIP_IDS)) { String aipIds = objectToString(doc.get(RodaConstants.DIP_AIP_IDS), null); dip.setAipIds(JsonUtils.getListFromJson(aipIds == null ? "" : aipIds, AIPLink.class)); } if (emptyFields || fieldsToReturn.contains(RodaConstants.DIP_REPRESENTATION_IDS)) { String representationIds = objectToString(doc.get(RodaConstants.DIP_REPRESENTATION_IDS), null); dip.setRepresentationIds(JsonUtils.getListFromJson( representationIds == null ? "" : representationIds, RepresentationLink.class)); } if (emptyFields || fieldsToReturn.contains(RodaConstants.DIP_FILE_IDS)) { String fileIds = objectToString(doc.get(RodaConstants.DIP_FILE_IDS), null); dip.setFileIds(JsonUtils.getListFromJson(fileIds == null ? "" : fileIds, FileLink.class)); } } catch (GenericException e) { LOGGER.error("Error getting related ids from DIP index"); } dip.setPermissions(getPermissions(doc)); dip.setOpenExternalURL(objectToString(doc.get(RodaConstants.DIP_OPEN_EXTERNAL_URL), null)); return dip; } public static SolrInputDocument dipFileToSolrDocument(DIP dip, DIPFile file) { SolrInputDocument doc = new SolrInputDocument(); doc.addField(RodaConstants.INDEX_UUID, IdUtils.getDIPFileId(file)); List<String> path = file.getPath(); doc.addField(RodaConstants.DIPFILE_PATH, path); if (path != null && !path.isEmpty()) { List<String> ancestorsPath = getDIPFileAncestorsPath(file.getDipId(), path); if (!ancestorsPath.isEmpty()) { doc.addField(RodaConstants.DIPFILE_ANCESTORS_UUIDS, ancestorsPath); } doc.addField(RodaConstants.DIPFILE_PARENT_UUID, IdUtils.getDIPFileId(file.getDipId(), path.subList(0, path.size() - 1), path.get(path.size() - 1))); } doc.addField(RodaConstants.DIPFILE_DIP_ID, file.getDipId()); doc.addField(RodaConstants.DIPFILE_ID, file.getId()); doc.addField(RodaConstants.DIPFILE_IS_DIRECTORY, file.isDirectory()); doc.addField(RodaConstants.DIPFILE_SIZE, Long.toString(file.getSize())); // extra-fields try { StoragePath filePath = ModelUtils.getDIPFileStoragePath(file); doc.addField(RodaConstants.DIPFILE_STORAGE_PATH, FSUtils.getStoragePathAsString(filePath, false)); } catch (RequestNotValidException e) { LOGGER.warn("Could not index DIP file storage path", e); } setPermissions(dip.getPermissions(), doc); return doc; } private static List<String> getDIPFileAncestorsPath(String dipId, List<String> path) { List<String> parentFileDirectoryPath = new ArrayList<>(); List<String> ancestorsPath = new ArrayList<>(); parentFileDirectoryPath.addAll(path); while (!parentFileDirectoryPath.isEmpty()) { int lastElementIndex = parentFileDirectoryPath.size() - 1; String parentFileId = parentFileDirectoryPath.get(lastElementIndex); parentFileDirectoryPath.remove(lastElementIndex); ancestorsPath.add(0, IdUtils.getDIPFileId(dipId, parentFileDirectoryPath, parentFileId)); } return ancestorsPath; } public static DIPFile solrDocumentToDIPFile(SolrDocument doc) { DIPFile file = new DIPFile(); file.setUUID(objectToString(doc.get(RodaConstants.INDEX_UUID), null)); file.setId(objectToString(doc.get(RodaConstants.DIPFILE_ID), null)); file.setDipId(objectToString(doc.get(RodaConstants.DIPFILE_DIP_ID), null)); file.setPath(objectToListString(doc.get(RodaConstants.DIPFILE_PATH))); file.setAncestorsUUIDs(objectToListString(doc.get(RodaConstants.DIPFILE_ANCESTORS_UUIDS))); file.setDirectory(objectToBoolean(doc.get(RodaConstants.DIPFILE_IS_DIRECTORY), Boolean.FALSE)); file.setStoragePath(objectToString(doc.get(RodaConstants.DIPFILE_STORAGE_PATH), null)); file.setSize(objectToLong(doc.get(RodaConstants.DIPFILE_SIZE), 0L)); return file; } /* * Partial updates of RODA objects * ____________________________________________________________________________________________________________________ */ public static SolrInputDocument aipStateUpdateToSolrDocument(AIP aip) { return stateUpdateToSolrDocument(aip.getId(), aip.getState()); } public static SolrInputDocument representationStateUpdateToSolrDocument(Representation representation, AIPState state) { return stateUpdateToSolrDocument(IdUtils.getRepresentationId(representation), state); } public static SolrInputDocument fileStateUpdateToSolrDocument(File file, AIPState state) { return stateUpdateToSolrDocument(IdUtils.getFileId(file), state); } public static SolrInputDocument preservationEventStateUpdateToSolrDocument(String preservationEventID, String preservationEventAipId, AIPState state) { SolrInputDocument document = stateUpdateToSolrDocument(preservationEventID, state); document.addField(RodaConstants.PRESERVATION_EVENT_AIP_ID, preservationEventAipId); return document; } private static SolrInputDocument stateUpdateToSolrDocument(String uuid, AIPState state) { SolrInputDocument doc = new SolrInputDocument(); doc.addField(RodaConstants.INDEX_UUID, uuid); doc.addField(RodaConstants.STATE, set(state.toString())); return doc; } public static SolrInputDocument aipPermissionsUpdateToSolrDocument(AIP aip) { SolrInputDocument document = new SolrInputDocument(); document.addField(RodaConstants.INDEX_UUID, aip.getId()); return permissionsUpdateToSolrDocument(document, aip.getPermissions()); } public static SolrInputDocument dipPermissionsUpdateToSolrDocument(DIP dip) { SolrInputDocument document = new SolrInputDocument(); document.addField(RodaConstants.INDEX_UUID, dip.getId()); return permissionsUpdateToSolrDocument(document, dip.getPermissions()); } public static SolrInputDocument representationPermissionsUpdateToSolrDocument(Representation representation, Permissions permissions) { SolrInputDocument document = new SolrInputDocument(); document.addField(RodaConstants.INDEX_UUID, IdUtils.getRepresentationId(representation)); return permissionsUpdateToSolrDocument(document, permissions); } public static SolrInputDocument filePermissionsUpdateToSolrDocument(File file, Permissions permissions) { SolrInputDocument document = new SolrInputDocument(); document.addField(RodaConstants.INDEX_UUID, IdUtils.getFileId(file)); return permissionsUpdateToSolrDocument(document, permissions); } public static SolrInputDocument preservationEventPermissionsUpdateToSolrDocument(String preservationEventID, String preservationEventAipId, Permissions permissions, AIPState state) { SolrInputDocument document = new SolrInputDocument(); document.addField(RodaConstants.INDEX_UUID, preservationEventID); document = permissionsUpdateToSolrDocument(document, permissions); document.addField(RodaConstants.STATE, state.toString()); document.addField(RodaConstants.PRESERVATION_EVENT_AIP_ID, preservationEventAipId); return document; } private static SolrInputDocument permissionsUpdateToSolrDocument(SolrInputDocument doc, Permissions permissions) { for (Entry<PermissionType, Set<String>> entry : permissions.getUsers().entrySet()) { String key = RodaConstants.INDEX_PERMISSION_USERS_PREFIX + entry.getKey(); List<String> value = new ArrayList<>(entry.getValue()); doc.addField(key, set(value)); } for (Entry<PermissionType, Set<String>> entry : permissions.getGroups().entrySet()) { String key = RodaConstants.INDEX_PERMISSION_GROUPS_PREFIX + entry.getKey(); List<String> value = new ArrayList<>(entry.getValue()); doc.addField(key, set(value)); } return doc; } public static SolrInputDocument updateAIPParentId(String aipId, String parentId, List<String> ancestors) throws RequestNotValidException, GenericException, AuthorizationDeniedException { SolrInputDocument doc = new SolrInputDocument(); doc.addField(RodaConstants.INDEX_UUID, aipId); doc.addField(RodaConstants.AIP_PARENT_ID, set(parentId)); doc.addField(RodaConstants.AIP_ANCESTORS, set(ancestors)); return doc; } public static SolrInputDocument updateAIPAncestors(String aipId, List<String> ancestors) throws RequestNotValidException, GenericException, AuthorizationDeniedException { SolrInputDocument doc = new SolrInputDocument(); doc.addField(RodaConstants.INDEX_UUID, aipId); doc.addField(RodaConstants.AIP_ANCESTORS, set(ancestors)); return doc; } public static SolrInputDocument updateRepresentationAncestors(String representationUUID, List<String> ancestors) throws RequestNotValidException, GenericException, AuthorizationDeniedException { SolrInputDocument doc = new SolrInputDocument(); doc.addField(RodaConstants.INDEX_UUID, representationUUID); doc.addField(RodaConstants.REPRESENTATION_ANCESTORS, set(ancestors)); return doc; } public static SolrInputDocument updateFileAncestors(String fileUUID, List<String> ancestors) throws RequestNotValidException, GenericException, AuthorizationDeniedException { SolrInputDocument doc = new SolrInputDocument(); doc.addField(RodaConstants.INDEX_UUID, fileUUID); doc.addField(RodaConstants.FILE_ANCESTORS, set(ancestors)); return doc; } public static Map<String, Object> set(Object value) { Map<String, Object> fieldModifier = new HashMap<>(1); // 20160511 this workaround fixes solr wrong behaviour with partial update // of empty lists if (value instanceof List && ((List<?>) value).isEmpty()) { fieldModifier.put("set", null); } else { fieldModifier.put("set", value); } return fieldModifier; } /* * Crosswalks auxiliary methods: RODA Objects <-> Apache Solr documents * ____________________________________________________________________________________________________________________ */ private static Permissions getPermissions(SolrDocument doc) { Permissions permissions = new Permissions(); EnumMap<PermissionType, Set<String>> userPermissions = new EnumMap<>(PermissionType.class); for (PermissionType type : PermissionType.values()) { String key = RodaConstants.INDEX_PERMISSION_USERS_PREFIX + type; Set<String> users = new HashSet<>(); users.addAll(objectToListString(doc.get(key))); userPermissions.put(type, users); } EnumMap<PermissionType, Set<String>> groupPermissions = new EnumMap<>(PermissionType.class); for (PermissionType type : PermissionType.values()) { String key = RodaConstants.INDEX_PERMISSION_GROUPS_PREFIX + type; Set<String> groups = new HashSet<>(); groups.addAll(objectToListString(doc.get(key))); groupPermissions.put(type, groups); } permissions.setUsers(userPermissions); permissions.setGroups(groupPermissions); return permissions; } private static void setPermissions(Permissions permissions, final SolrInputDocument ret) { for (Entry<PermissionType, Set<String>> entry : permissions.getUsers().entrySet()) { String key = RodaConstants.INDEX_PERMISSION_USERS_PREFIX + entry.getKey(); List<String> value = new ArrayList<>(entry.getValue()); ret.addField(key, value); } for (Entry<PermissionType, Set<String>> entry : permissions.getGroups().entrySet()) { String key = RodaConstants.INDEX_PERMISSION_GROUPS_PREFIX + entry.getKey(); List<String> value = new ArrayList<>(entry.getValue()); ret.addField(key, value); } } public static List<String> getAncestors(String parentId, ModelService model) throws RequestNotValidException, GenericException, AuthorizationDeniedException { List<String> ancestors = new ArrayList<>(); String nextAncestorId = parentId; while (nextAncestorId != null) { try { AIP nextAncestor = model.retrieveAIP(nextAncestorId); if (ancestors.contains(nextAncestorId)) { break; } ancestors.add(nextAncestorId); nextAncestorId = nextAncestor.getParentId(); } catch (NotFoundException e) { LOGGER.warn("Could not find one AIP ancestor. Ancestor id: {}", nextAncestorId); nextAncestorId = null; } } return ancestors; } /** * WARNING: this should only be used to debug/tests only * * @return * @throws IOException * @throws SolrServerException */ public static QueryResponse executeSolrQuery(SolrClient index, String collection, String solrQueryString) throws SolrServerException, IOException { LOGGER.trace("query string: {}", solrQueryString); SolrQuery query = new SolrQuery(); for (String string : solrQueryString.split("&")) { String[] split = string.split("="); query.add(split[0], split[1]); } LOGGER.trace("executeSolrQuery: {}", query); return index.query(collection, query); } public static SolrInputDocument premisToSolr(PreservationMetadataType preservationMetadataType, AIP aip, String representationUUID, String fileUUID, Binary binary) throws GenericException { SolrInputDocument doc; Map<String, String> stylesheetOpt = new HashMap<>(); stylesheetOpt.put(RodaConstants.PRESERVATION_EVENT_OBJECT_CLASS, PreservationMetadataEventClass.REPOSITORY.toString()); if (aip != null) { stylesheetOpt.put(RodaConstants.PRESERVATION_EVENT_OBJECT_CLASS, PreservationMetadataEventClass.AIP.toString()); stylesheetOpt.put(RodaConstants.PRESERVATION_EVENT_AIP_ID, aip.getId()); if (representationUUID != null) { stylesheetOpt.put(RodaConstants.PRESERVATION_EVENT_REPRESENTATION_UUID, representationUUID); stylesheetOpt.put(RodaConstants.PRESERVATION_EVENT_OBJECT_CLASS, PreservationMetadataEventClass.REPRESENTATION.toString()); } if (fileUUID != null) { stylesheetOpt.put(RodaConstants.PRESERVATION_EVENT_FILE_UUID, fileUUID); stylesheetOpt.put(RodaConstants.PRESERVATION_EVENT_OBJECT_CLASS, PreservationMetadataEventClass.FILE.toString()); } } Reader reader = null; try { reader = RodaUtils.applyMetadataStylesheet(binary, RodaConstants.CORE_CROSSWALKS_INGEST_OTHER, RodaConstants.PREMIS_METADATA_TYPE, RodaConstants.PREMIS_METADATA_VERSION, stylesheetOpt); XMLLoader loader = new XMLLoader(); XMLStreamReader parser = XMLInputFactory.newInstance().createXMLStreamReader(reader); boolean parsing = true; doc = null; while (parsing) { int event = parser.next(); if (event == XMLStreamConstants.END_DOCUMENT) { parser.close(); parsing = false; } else if (event == XMLStreamConstants.START_ELEMENT && "doc".equals(parser.getLocalName())) { doc = loader.readDoc(parser); } } } catch (XMLStreamException | FactoryConfigurationError e) { throw new GenericException("Could not process PREMIS " + binary.getStoragePath(), e); } finally { IOUtils.closeQuietly(reader); } if (preservationMetadataType == PreservationMetadataType.EVENT && doc != null) { try { List<LinkingIdentifier> agents = PremisV3Utils.extractAgentsFromEvent(binary); for (LinkingIdentifier id : agents) { doc.addField(RodaConstants.PRESERVATION_EVENT_LINKING_AGENT_IDENTIFIER, JsonUtils.getJsonFromObject(id)); } } catch (org.roda.core.data.v2.validation.ValidationException e) { LOGGER.warn("Error setting linking agent field: {}", e.getMessage()); } try { List<LinkingIdentifier> sources = PremisV3Utils.extractObjectFromEvent(binary); for (LinkingIdentifier id : sources) { doc.addField(RodaConstants.PRESERVATION_EVENT_LINKING_SOURCE_OBJECT_IDENTIFIER, JsonUtils.getJsonFromObject(id)); } } catch (org.roda.core.data.v2.validation.ValidationException e) { LOGGER.warn("Error setting linking source field: {}", e.getMessage()); } try { List<LinkingIdentifier> outcomes = PremisV3Utils.extractObjectFromEvent(binary); for (LinkingIdentifier id : outcomes) { doc.addField(RodaConstants.PRESERVATION_EVENT_LINKING_OUTCOME_OBJECT_IDENTIFIER, JsonUtils.getJsonFromObject(id)); } } catch (org.roda.core.data.v2.validation.ValidationException e) { LOGGER.warn("Error setting linking outcome field: {}", e.getMessage()); } // indexing active state and permissions if (aip != null) { doc.addField(RodaConstants.STATE, aip.getState().toString()); setPermissions(aip.getPermissions(), doc); } else { doc.addField(RodaConstants.STATE, AIPState.ACTIVE); } } // set uuid from id defined in xslt if (doc != null) { doc.addField(RodaConstants.INDEX_UUID, doc.getFieldValue(RodaConstants.PRESERVATION_EVENT_ID)); } else { doc = new SolrInputDocument(); } return doc; } public static <T extends IsIndexed> List<String> suggest(SolrClient index, Class<T> classToRetrieve, String field, String queryString, boolean justActive, User user, boolean allowPartial) throws GenericException { StringBuilder queryBuilder = new StringBuilder(); appendKeyValue(queryBuilder, field + RodaConstants.INDEX_SEARCH_SUFFIX, queryString + "*"); SolrQuery query = new SolrQuery(); query.setParam("q.op", DEFAULT_QUERY_PARSER_OPERATOR); query.setQuery(queryBuilder.toString()); if (hasPermissionFilters(classToRetrieve)) { query.addFilterQuery(getFilterQueries(user, justActive, classToRetrieve)); } parseAndConfigureFacets(new Facets(new SimpleFacetParameter(field)), query); List<String> suggestions = new ArrayList<>(); try { QueryResponse response = index.query(getIndexName(classToRetrieve).get(0), query); response.getFacetField(field).getValues().forEach(count -> suggestions.add(count.getName())); } catch (SolrServerException | IOException | SolrException e) { throw new GenericException("Could not get suggestions", e); } catch (RuntimeException e) { throw new GenericException("Unexpected exception while querying index", e); } return suggestions; } public static <T extends IsIndexed> void execute(SolrClient index, Class<T> classToRetrieve, Filter filter, List<String> fieldsToReturn, IndexRunnable<T> indexRunnable) throws GenericException, RequestNotValidException, AuthorizationDeniedException { Sorter sorter = null; int offset = 0; int pagesize = RodaConstants.DEFAULT_PAGINATION_VALUE; boolean done; do { Sublist sublist = new Sublist(offset, pagesize); IndexResult<T> find = SolrUtils.find(index, classToRetrieve, filter, sorter, sublist, fieldsToReturn); for (T target : find.getResults()) { indexRunnable.run(target); } done = find.getResults().isEmpty(); offset += pagesize; } while (!done); } private static SolrInputDocument solrDocumentToSolrInputDocument(SolrDocument d) { SolrInputDocument doc = new SolrInputDocument(); for (String name : d.getFieldNames()) { doc.addField(name, d.getFieldValue(name)); } return doc; } public static <T extends IsIndexed> void delete(SolrClient index, Class<T> classToDelete, List<String> ids) throws GenericException { try { index.deleteById(getIndexName(classToDelete).get(0), ids); } catch (SolrServerException | IOException e) { throw new GenericException("Could not delete items", e); } } public static <T extends IsIndexed> void delete(SolrClient index, Class<T> classToDelete, Filter filter) throws GenericException, RequestNotValidException { try { index.deleteByQuery(getIndexName(classToDelete).get(0), parseFilter(filter)); } catch (SolrServerException | IOException e) { throw new GenericException("Could not delete items", e); } } }