ddf.catalog.cache.solr.impl.SolrCache.java Source code

Java tutorial

Introduction

Here is the source code for ddf.catalog.cache.solr.impl.SolrCache.java

Source

/**
 * Copyright (c) Codice Foundation
 * <p>
 * This is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser
 * General Public License as published by the Free Software Foundation, either version 3 of the
 * License, or any later version.
 * <p>
 * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without
 * even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * Lesser General Public License for more details. A copy of the GNU Lesser General Public License
 * is distributed along with this program and can be found at
 * <http://www.gnu.org/licenses/lgpl.html>.
 */
package ddf.catalog.cache.solr.impl;

import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;

import javax.management.InstanceAlreadyExistsException;
import javax.management.MBeanServer;
import javax.management.MalformedObjectNameException;
import javax.management.ObjectName;
import javax.management.StandardMBean;

import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.UpdateResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.codice.ddf.configuration.SystemBaseUrl;
import org.codice.solr.factory.SolrClientFactory;
import org.opengis.filter.Filter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.collect.Lists;

import ddf.catalog.cache.SolrCacheMBean;
import ddf.catalog.data.Metacard;
import ddf.catalog.data.MetacardCreationException;
import ddf.catalog.data.Result;
import ddf.catalog.data.impl.MetacardImpl;
import ddf.catalog.filter.FilterAdapter;
import ddf.catalog.operation.DeleteRequest;
import ddf.catalog.operation.QueryRequest;
import ddf.catalog.operation.SourceResponse;
import ddf.catalog.operation.impl.QueryImpl;
import ddf.catalog.operation.impl.QueryRequestImpl;
import ddf.catalog.source.UnsupportedQueryException;
import ddf.catalog.source.solr.DynamicSchemaResolver;
import ddf.catalog.source.solr.SchemaFields;
import ddf.catalog.source.solr.SolrFilterDelegate;
import ddf.catalog.source.solr.SolrFilterDelegateFactory;
import ddf.catalog.source.solr.SolrMetacardClient;

/**
 * Catalog cache implementation using Apache Solr
 */
public class SolrCache implements SolrCacheMBean {

    public static final String METACARD_CACHE_CORE_NAME = "metacard_cache";

    public static final String METACARD_SOURCE_NAME = "metacard_source" + SchemaFields.TEXT_SUFFIX;

    public static final String METACARD_ID_NAME = "original_id" + SchemaFields.TEXT_SUFFIX;

    // the unique id field used in the platform solr standalone server
    public static final String METACARD_UNIQUE_ID_NAME = "id" + SchemaFields.TEXT_SUFFIX;

    public static final String CACHED_DATE = "cached" + SchemaFields.DATE_SUFFIX;

    private static final Logger LOGGER = LoggerFactory.getLogger(SolrCache.class);

    private FilterAdapter filterAdapter;

    private ObjectName objectName;

    private String url = SolrClientFactory.getDefaultHttpsAddress();

    private SolrClient server;

    private SolrFilterDelegateFactory solrFilterDelegateFactory;

    private SolrMetacardClient client;

    private AtomicBoolean dirty = new AtomicBoolean(false);

    private ScheduledExecutorService scheduler;

    private long expirationIntervalInMinutes = 10;

    private long expirationAgeInMinutes = TimeUnit.DAYS.toMinutes(7);

    private MBeanServer mbeanServer;

    /**
     * Convenience constructor that creates a the Solr server
     *
     * @param adapter injected implementation of FilterAdapter
     */
    public SolrCache(FilterAdapter adapter, SolrFilterDelegateFactory solrFilterDelegateFactory) {
        this.filterAdapter = adapter;
        this.solrFilterDelegateFactory = solrFilterDelegateFactory;
        this.url = SystemBaseUrl.constructUrl("/solr");
        this.updateServer(this.url);
        configureCacheExpirationScheduler();

        try {
            objectName = new ObjectName(SolrCacheMBean.OBJECTNAME);
        } catch (MalformedObjectNameException e) {
            LOGGER.info("Could not create object name", e);
        }

        configureMBean();
    }

    public SourceResponse query(QueryRequest request) throws UnsupportedQueryException {
        return client.query(request);
    }

    public void create(Collection<Metacard> metacards) {
        if (metacards == null || metacards.size() == 0) {
            return;
        }

        List<Metacard> updatedMetacards = new ArrayList<>();
        for (Metacard metacard : metacards) {
            if (metacard != null) {
                if (StringUtils.isNotBlank(metacard.getSourceId()) && StringUtils.isNotBlank(metacard.getId())) {
                    updatedMetacards.add(metacard);
                }
            } else {
                LOGGER.debug("metacard in result was null");
            }
        }

        try {
            client.add(updatedMetacards, false);
            dirty.set(true);
        } catch (SolrServerException | SolrException | IOException | MetacardCreationException e) {
            LOGGER.warn("Solr server exception caching metacard(s)", e);
        }
    }

    public void delete(DeleteRequest deleteRequest) {
        if (deleteRequest == null) {
            return;
        }

        String attributeName = deleteRequest.getAttributeName();
        if (StringUtils.isBlank(attributeName)) {
            LOGGER.warn("Attribute name cannot be empty. Cannot delete from cache.");
            return;
        }

        String fieldName = attributeName + SchemaFields.TEXT_SUFFIX;
        if (fieldName.equals(METACARD_UNIQUE_ID_NAME)) {
            fieldName = METACARD_ID_NAME;
        }

        try {
            client.deleteByIds(fieldName, deleteRequest.getAttributeValues(), false);
            dirty.set(true);
        } catch (SolrServerException | IOException e) {
            LOGGER.error("Solr server exception while deleting from cache", e);
        }
    }

    public void setExpirationIntervalInMinutes(long expirationInterval) {
        this.expirationIntervalInMinutes = expirationInterval;
        configureCacheExpirationScheduler();
    }

    public void setExpirationAgeInMinutes(long expirationAgeInMinutes) {
        this.expirationAgeInMinutes = expirationAgeInMinutes;
    }

    private void configureCacheExpirationScheduler() {
        shutdownCacheExpirationScheduler();
        LOGGER.info("Configuring cache expiration scheduler with an expiration interval of {} minute(s).",
                expirationIntervalInMinutes);
        scheduler = Executors.newSingleThreadScheduledExecutor();
        scheduler.scheduleAtFixedRate(new ExpirationRunner(), 0, expirationIntervalInMinutes, TimeUnit.MINUTES);
    }

    private void configureMBean() {
        LOGGER.info("Registering Cache Manager Service MBean");
        mbeanServer = ManagementFactory.getPlatformMBeanServer();

        try {
            try {

                mbeanServer.registerMBean(new StandardMBean(this, SolrCacheMBean.class), objectName);
            } catch (InstanceAlreadyExistsException e) {
                LOGGER.info("Re-registering Cache Manager MBean");
                mbeanServer.unregisterMBean(objectName);
                mbeanServer.registerMBean(new StandardMBean(this, SolrCacheMBean.class), objectName);
            }
        } catch (Exception e) {
            LOGGER.warn("Could not register MBean.", e);
        }
    }

    private void shutdownCacheExpirationScheduler() {
        if (scheduler != null && !scheduler.isShutdown()) {
            LOGGER.debug("Shutting down cache expiration scheduler.");
            scheduler.shutdown();
            try {
                // Wait up to 60 seconds existing tasks to terminate
                if (!scheduler.awaitTermination(60, TimeUnit.SECONDS)) {
                    scheduler.shutdownNow();
                    // Wait up to 60 seconds for tasks to respond to being cancelled
                    if (!scheduler.awaitTermination(60, TimeUnit.SECONDS)) {
                        LOGGER.warn("Cache expiration scheduler did not terminate.");
                    }
                }
            } catch (InterruptedException e) {
                // (Recancel/cancel if current thread also interrupted
                scheduler.shutdownNow();
                // Preserve interrupt status
                Thread.currentThread().interrupt();
            }
        } else {
            LOGGER.debug("Cache expiration scheduler already shutdown.");
        }
    }

    public void updateServer(String newUrl) {
        LOGGER.info("New url {}", newUrl);

        if (newUrl != null) {
            if (!StringUtils.equalsIgnoreCase(newUrl.trim(), url) || server == null) {

                this.url = newUrl.trim();

                if (server != null) {
                    LOGGER.info(
                            "Shutting down the connection manager to the Solr Server and releasing allocated resources.");
                    try {
                        server.close();
                        LOGGER.info("Shutdown complete.");
                    } catch (IOException e) {
                        LOGGER.info("Failed to shutdown Solr server.", e);
                    }
                }

                try {
                    server = SolrClientFactory.getHttpSolrClient(url, METACARD_CACHE_CORE_NAME).get();
                } catch (InterruptedException | ExecutionException e) {
                    LOGGER.error("Failed to get solr server from future", e);
                }
                client = new CacheSolrMetacardClient(this.server, filterAdapter, solrFilterDelegateFactory);
            }
        } else {
            this.url = null;
        }
    }

    public void forceCommit() {
        try {
            if (dirty.compareAndSet(true, false)) {
                server.commit();
            }
        } catch (SolrServerException | IOException e) {
            LOGGER.warn("Unable to commit changes to cache.", e);
        }
    }

    public String getMetacardId(SolrDocument doc) {
        return doc.getFirstValue(METACARD_ID_NAME).toString();
    }

    public String getMetacardSource(SolrDocument doc) {
        return doc.getFirstValue(METACARD_SOURCE_NAME).toString();
    }

    public void shutdown() {
        LOGGER.info("Shutting down cache expiration scheduler.");
        shutdownCacheExpirationScheduler();
        LOGGER.info("Shutting down Solr server.");
        try {
            server.close();
        } catch (IOException e) {
            LOGGER.info("Failed to shutdown Solr server.", e);
        }
    }

    @Override
    public void removeAll() throws IOException, SolrServerException {
        client.deleteByQuery("*:*");
    }

    @Override
    public void removeById(String[] ids) throws IOException, SolrServerException {
        List<String> idList = Arrays.asList(ids);
        client.deleteByIds(METACARD_ID_NAME, idList, false);
    }

    @Override
    public List<Metacard> query(Filter filter) throws UnsupportedQueryException {
        QueryRequest queryRequest = new QueryRequestImpl(new QueryImpl(filter), true);

        SourceResponse response = client.query(queryRequest);
        return getMetacardsFromResponse(response);
    }

    private List<Metacard> getMetacardsFromResponse(SourceResponse sourceResponse) {
        if (CollectionUtils.isNotEmpty(sourceResponse.getResults())) {
            List<Metacard> metacards = new ArrayList<>(sourceResponse.getResults().size());
            for (Result result : sourceResponse.getResults()) {
                metacards.add(result.getMetacard());
            }
            return metacards;
        }
        return Collections.emptyList();
    }

    private class ExpirationRunner implements Runnable {

        @Override
        public void run() {
            try {
                LOGGER.debug("Expiring cache.");
                server.deleteByQuery(CACHED_DATE + ":[* TO NOW-" + expirationAgeInMinutes + "MINUTES]");
            } catch (SolrServerException | IOException e) {
                LOGGER.warn("Unable to expire cache.", e);
            }
        }
    }

    private class CacheSolrMetacardClient extends SolrMetacardClient {

        public CacheSolrMetacardClient(SolrClient client, FilterAdapter catalogFilterAdapter,
                SolrFilterDelegateFactory solrFilterDelegateFactory) {
            super(client, catalogFilterAdapter, solrFilterDelegateFactory, new DynamicSchemaResolver());
        }

        @Override
        public MetacardImpl createMetacard(SolrDocument doc) throws MetacardCreationException {
            MetacardImpl metacard = super.createMetacard(doc);

            metacard.setSourceId(getMetacardSource(doc));
            metacard.setId(getMetacardId(doc));

            return metacard;
        }

        public UpdateResponse delete(String query) throws IOException, SolrServerException {
            return server.deleteByQuery(query);
        }

        @Override
        protected SolrQuery postAdapt(QueryRequest request, SolrFilterDelegate filterDelegate, SolrQuery query)
                throws UnsupportedQueryException {
            if (!request.isEnterprise()) {
                List<SolrQuery> sourceQueries = new ArrayList<>();
                for (String source : request.getSourceIds()) {
                    sourceQueries.add(filterDelegate.propertyIsEqualTo(
                            StringUtils.removeEnd(METACARD_SOURCE_NAME, SchemaFields.TEXT_SUFFIX), source, true));
                }
                if (sourceQueries.size() > 0) {
                    SolrQuery allSourcesQuery;
                    if (sourceQueries.size() > 1) {
                        allSourcesQuery = filterDelegate.or(sourceQueries);
                    } else {
                        allSourcesQuery = sourceQueries.get(0);
                    }
                    query = filterDelegate.and(Lists.newArrayList(query, allSourcesQuery));
                }
            }

            return super.postAdapt(request, filterDelegate, query);
        }

        @Override
        protected SolrInputDocument getSolrInputDocument(Metacard metacard) throws MetacardCreationException {
            SolrInputDocument solrInputDocument = super.getSolrInputDocument(metacard);

            solrInputDocument.addField(CACHED_DATE, new Date());

            if (StringUtils.isNotBlank(metacard.getSourceId())) {
                solrInputDocument.addField(METACARD_SOURCE_NAME, metacard.getSourceId());
                solrInputDocument.setField(METACARD_UNIQUE_ID_NAME, metacard.getSourceId() + metacard.getId());
                solrInputDocument.addField(METACARD_ID_NAME, metacard.getId());
            }

            return solrInputDocument;
        }
    }
}