Java tutorial
/** * Copyright (c) 2003-2017 The Apereo Foundation * * Licensed under the Educational Community License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://opensource.org/licenses/ecl2 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.sakaiproject.search.elasticsearch; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.FilterBuilders.missingFilter; import static org.elasticsearch.index.query.FilterBuilders.orFilter; import static org.elasticsearch.index.query.FilterBuilders.termFilter; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; import static org.elasticsearch.index.query.QueryBuilders.filteredQuery; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.index.query.QueryBuilders.matchQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; import static org.elasticsearch.index.query.QueryBuilders.termsQuery; import static org.elasticsearch.search.facet.FacetBuilders.termsFacet; import java.io.IOException; import java.io.StringWriter; import java.text.DecimalFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.Timer; import java.util.TimerTask; import java.util.function.Predicate; import java.util.stream.Collectors; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest; import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; import org.elasticsearch.action.admin.indices.status.IndexStatus; import org.elasticsearch.action.admin.indices.status.IndicesStatusRequest; import org.elasticsearch.action.admin.indices.status.IndicesStatusResponse; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.count.CountResponse; import org.elasticsearch.action.delete.DeleteRequestBuilder; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.client.Client; import org.elasticsearch.common.collect.Sets; import org.elasticsearch.common.lang3.ArrayUtils; import org.elasticsearch.common.lang3.tuple.ImmutablePair; import org.elasticsearch.common.lang3.tuple.Pair; import org.elasticsearch.common.settings.loader.JsonSettingsLoader; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.indices.IndexAlreadyExistsException; import org.elasticsearch.search.SearchHit; import org.sakaiproject.authz.api.SecurityAdvisor; import org.sakaiproject.authz.api.SecurityService; import org.sakaiproject.component.api.ServerConfigurationService; import org.sakaiproject.event.api.Event; import org.sakaiproject.event.api.EventTrackingService; import org.sakaiproject.event.api.Notification; import org.sakaiproject.event.api.NotificationService; import org.sakaiproject.search.api.EntityContentProducer; import org.sakaiproject.search.api.EntityContentProducerEvents; import org.sakaiproject.search.api.SearchIndexBuilder; import org.sakaiproject.search.api.SearchService; import org.sakaiproject.search.api.SearchStatus; import org.sakaiproject.search.elasticsearch.filter.SearchItemFilter; import org.sakaiproject.search.model.SearchBuilderItem; import org.slf4j.Logger; import org.springframework.util.Assert; import com.google.common.collect.Lists; import com.google.common.collect.Maps; /** * */ public abstract class BaseElasticSearchIndexBuilder implements ElasticSearchIndexBuilder { protected static final String DEFAULT_FACET_NAME = "tag"; protected static final String DEFAULT_SUGGESTION_MATCHING_FIELD_NAME = SearchService.FIELD_TITLE; /** * Key in the "validation map" built up by {@link #validateAddResourceEvent(Event)} (specifically * {@link #validateResourceName(Event, Map)}. */ protected static final String ADD_RESOURCE_VALIDATION_KEY_RESOURCE_NAME = "RESOURCE_NAME"; protected static final String ADD_RESOURCE_VALIDATION_KEY_CONTENT_PRODUCER = "CONTENT_PRODUCER"; protected static final String ADD_RESOURCE_VALIDATION_KEY_INDEX_ACTION = "INDEX_ACTION"; protected static final String ADD_RESOURCE_VALIDATION_KEY_ENTITY_ID = "ENTITY_ID"; protected static final String DELETE_RESOURCE_KEY_DOCUMENT_ID = "DOCUMENT_ID"; protected static final String DELETE_RESOURCE_KEY_ENTITY_REFERENCE = "ENTITY_REFERENCE"; protected final static SecurityAdvisor allowAllAdvisor = (userId, function, reference) -> SecurityAdvisor.SecurityAdvice.ALLOWED; protected SecurityService securityService; protected ServerConfigurationService serverConfigurationService; protected EventTrackingService eventTrackingService; /** * ES Client, with access to any indexes in the cluster. */ protected Client client; /** * the ES indexname */ protected String indexName; /** * Logical, well-known name, possibly distinct from the physical ES {@link #indexName} */ protected String name; /** * set to true to force an index rebuild at startup time, defaults to false. This is probably something * you never want to use, other than in development or testing */ protected boolean rebuildIndexOnStartup = false; protected boolean useSuggestions = true; /** * max number of suggestions to return when looking for suggestions (this populates the autocomplete drop down in the UI) */ protected int maxNumberOfSuggestions = 10; protected String suggestionMatchingFieldName = DEFAULT_SUGGESTION_MATCHING_FIELD_NAME; protected String[] suggestionResultFieldNames; protected String[] searchResultFieldNames; protected SearchItemFilter filter; protected boolean useFacetting = true; protected String facetName = DEFAULT_FACET_NAME; /** * N most frequent terms */ protected int facetTermSize = 10; /** * Number of documents to index at a time for each run of the context indexing task (defaults to 500). * Setting this too low will slow things down, setting it to high won't allow all nodes in the cluster * to share the load. */ protected int contentIndexBatchSize = 500; /** * Number of actions to send in one elasticsearch bulk index call * defaults to 10. Setting this * to too high a number will have memory implications as you'll be keeping * more content in memory until the request is executed. */ protected int bulkRequestSize = 10; /** * number seconds of wait after startup before starting the BulkContentIndexerTask (defaults to 3 minutes) */ protected int delay = 180; /** * how often the BulkContentIndexerTask runs in seconds (defaults to 1 minute) */ protected int period = 60; protected long startTime; protected long lastLoad; /** * this turns off the threads and does indexing inline. DO NOT enable this in prod. * It is meant for testing, especially unit tests only. */ protected boolean testMode = false; protected String defaultMappingResource = "/org/sakaiproject/search/elastic/bundle/mapping.json"; /** * by default the mapping in configured in the mapping.json file. This can be overridden by injecting * json into this property. * * See {@link <a href="http://www.elasticsearch.org/guide/reference/mapping/">elasticsearch mapping reference</a> } for * more information on configuration that is available. For example, if you want to change the analyzer config for * a particular field this is the place to do it. */ protected String mapping = null; /** * Combination of {@link #mapping}, its fallback read from {@link #defaultMappingResource}, and any overrides * implemented in {@link #initializeElasticSearchMapping(String)}. (Currently there are no such overrides in that * method... just the fallback resource lookup. And historically the mapping config was stored exclusively in * {@link #mapping}. This new {@code mappingMerged} field was added for symmetry with {@link #indexSettingsMerged}). */ protected String mappingMerged = null; protected String defaultIndexSettingsResource = "/org/sakaiproject/search/elastic/bundle/indexSettings.json"; /** * Expects a JSON string of ElasticSearch index settings. You can set this in your * sakai.properties files and inject a value using the indexSettings@org.sakaiproject.search.api.SearchIndexBuilder * property. By default this value is configured by the indexSettings.json files. * * See {@link <a href="http://www.elasticsearch.org/guide/reference/index-modules/">elasticsearch index modules</a>} * for more information on configuration that is available. */ protected String indexSettings = null; /** * Combination of {@link #indexSettings}, its fallback read from {@link #defaultIndexSettingsResource} and * overrides read in from {@code ServerConfigurationService}. This is the operational set of index configs. * {@link #indexSettings} is just preserved for reference. */ protected Map<String, String> indexSettingsMerged = new HashMap(); protected String indexedDocumentType = null; /** * indexing thread that performs loading the actual content into the index. */ protected Timer backgroundScheduler = null; protected Set<EntityContentProducer> producers = Sets.newConcurrentHashSet(); protected Set<String> triggerFunctions = Sets.newHashSet(); protected ElasticSearchIndexBuilderEventRegistrar eventRegistrar; public boolean isEnabled() { return serverConfigurationService.getBoolean("search.enable", false); } @Override public void destroy() { this.client = null; this.eventRegistrar = null; } @Override public void initialize(ElasticSearchIndexBuilderEventRegistrar eventRegistrar, Client client) { if (!isEnabled()) { getLog().debug("ElasticSearch is not enabled. Skipping initialization of index builder [" + getName() + "]. Set search.enable=true to change that."); return; } if (testMode) { getLog().warn( "IN TEST MODE for index builder [" + getName() + "]. DO NOT enable this in production !!!"); } getLog().info("Initializing ElasticSearch index builder [" + getName() + "]..."); String indexNamespace = serverConfigurationService.getString("search.indexNamespace", null); if (StringUtils.isNotBlank(indexNamespace)) { this.indexName = indexNamespace + "_" + this.indexName; } this.eventRegistrar = eventRegistrar; this.client = client; beforeElasticSearchConfigInitialization(); requireConfiguration(); this.mappingMerged = initializeElasticSearchMapping(this.mapping); this.indexSettingsMerged = initializeElasticSearchIndexSettings(this.indexSettings); beforeBackgroundSchedulerInitialization(); this.backgroundScheduler = initializeBackgroundScheduler(); backgroundScheduler.schedule(initializeContentQueueProcessingTask(), (delay * 1000), (period * 1000)); initializeIndex(); this.eventRegistrar.updateEventsFor(this); } /** * Gives subclasses a chance to initialize configuration prior to reading/processing any * ES configs. May be important for setting up defaults, for example, or for ensuring * subclass-specific configs are in place before any background tasks are in place. * (Though the latter would be better factored into {@link #beforeBackgroundSchedulerInitialization()} */ protected abstract void beforeElasticSearchConfigInitialization(); protected void requireConfiguration() { Assert.hasText(name, "Must specify a logical name for this index builder"); Assert.hasText(indexName, "Must specify a physical index name for this index builder"); Assert.hasText(indexedDocumentType, "Must specify an indexed document type for this index builder"); } /** * Called after all ES config has been processed but before the background scheduler has been set up * and before any index startup ops have been invoked ({@link #initializeIndex()}. I.e. this is a * subclass's last chance to set up any configs on which background jobs and/or index maintenance * in general might depend. */ protected abstract void beforeBackgroundSchedulerInitialization(); protected String initializeElasticSearchMapping(String injectedConfig) { // if there is a value here its been overridden by injection, we will use the overridden configuration String mappingConfig = injectedConfig; if (org.apache.commons.lang3.StringUtils.isEmpty(injectedConfig)) { try { StringWriter writer = new StringWriter(); IOUtils.copy(getClass().getResourceAsStream(this.defaultMappingResource), writer, "UTF-8"); mappingConfig = writer.toString(); } catch (Exception ex) { getLog().error("Failed to load mapping config: " + ex.getMessage(), ex); } } getLog().debug("ElasticSearch mapping will be configured as follows for index builder [" + getName() + "]:" + mappingConfig); return mappingConfig; } protected Map<String, String> initializeElasticSearchIndexSettings(String injectedConfig) { String defaultConfig = injectedConfig; if (org.apache.commons.lang3.StringUtils.isEmpty(injectedConfig)) { try { StringWriter writer = new StringWriter(); IOUtils.copy(getClass().getResourceAsStream(this.defaultIndexSettingsResource), writer, "UTF-8"); defaultConfig = writer.toString(); } catch (Exception ex) { getLog().error("Failed to load indexSettings config from [" + this.defaultIndexSettingsResource + "] for index builder [" + getName() + "]", ex); } } JsonSettingsLoader loader = new JsonSettingsLoader(); Map<String, String> mergedConfig = null; try { mergedConfig = loader.load(defaultConfig); } catch (IOException e) { getLog().error("Problem loading indexSettings for index builder [" + getName() + "]", e); } // Set these here so we don't have to do this string concatenation // and comparison every time through the upcoming 'for' loop final boolean IS_DEFAULT = SearchIndexBuilder.DEFAULT_INDEX_BUILDER_NAME.equals(getName()); final String DEFAULT_INDEX = ElasticSearchConstants.CONFIG_PROPERTY_PREFIX + "index."; final String LOCAL_INDEX = String.format("%s%s%s.", ElasticSearchConstants.CONFIG_PROPERTY_PREFIX, "index_", getName(), "."); // load anything set into the ServerConfigurationService that starts with "elasticsearch.index." this will // override anything set in the indexSettings config for (ServerConfigurationService.ConfigItem configItem : serverConfigurationService.getConfigData() .getItems()) { String propertyName = configItem.getName(); if (IS_DEFAULT && (propertyName.startsWith(DEFAULT_INDEX))) { propertyName = propertyName.replaceFirst(DEFAULT_INDEX, "index."); mergedConfig.put(propertyName, (String) configItem.getValue()); } else if (propertyName.startsWith(LOCAL_INDEX)) { propertyName = propertyName.replaceFirst(LOCAL_INDEX, "index."); mergedConfig.put(propertyName, (String) configItem.getValue()); } } if (getLog().isDebugEnabled()) { for (Map.Entry<String, String> entry : mergedConfig.entrySet()) { getLog().debug("Index property '" + entry.getKey() + "' set to: " + entry.getValue() + "' for index builder '" + getName() + "'"); } } return mergedConfig; } protected Timer initializeBackgroundScheduler() { // name is historical return new Timer("[elasticsearch content indexer " + getName() + "]", true); } protected TimerTask initializeContentQueueProcessingTask() { return testMode ? new NoOpTask() : newBulkContentIndexerTask(); } protected static class NoOpTask extends TimerTask { @Override public void run() { // nothing to do } } /** * This is the task that searches for any docs in the search index that do not have content yet, * digests the content and loads it into the index. Any docs with empty content will be removed from * the index. This timer task is run by the timer thread based on the period configured elsewhere */ protected class BulkContentIndexerTask extends TimerTask { @Override public void run() { try { getLog().debug("Running content indexing task for index builder [" + getName() + "]"); enableAzgSecurityAdvisor(); processContentQueue(); } catch (Exception e) { getLog().error("Content indexing failure for index builder [" + getName() + "]", e); } finally { disableAzgSecurityAdvisor(); } } } protected TimerTask newBulkContentIndexerTask() { return new BulkContentIndexerTask(); } protected class RebuildIndexTask extends TimerTask { /** * Rebuild the index from the entities own stored state {@inheritDoc} */ @Override public void run() { // let's not hog the whole CPU just in case you have lots of sites with lots of data this could take a bit Thread.currentThread().setPriority(Thread.NORM_PRIORITY - 1); rebuildIndexImmediately(); } } protected TimerTask newRebuildIndexTask() { return new RebuildIndexTask(); } /** * Searches for any docs in the search index that have not been indexed yet, * digests the content and loads it into the index. Any docs with empty content will be removed from * the index. */ protected void processContentQueue() { startTime = System.currentTimeMillis(); // If there are a lot of docs queued up this could take awhile we don't want // to eat up all the CPU cycles. Thread.currentThread().setPriority(Thread.NORM_PRIORITY - 1); if (getPendingDocuments() == 0) { getLog().trace("No pending docs for index builder [" + getName() + "]"); return; } SearchResponse response = findContentQueue(); SearchHit[] hits = response.getHits().hits(); List<NoContentException> noContentExceptions = new ArrayList(); getLog().debug(getPendingDocuments() + " pending docs for index builder [" + getName() + "]"); BulkRequestBuilder bulkRequest = newContentQueueBulkUpdateRequestBuilder(); for (SearchHit hit : hits) { if (bulkRequest.numberOfActions() < bulkRequestSize) { try { processContentQueueEntry(hit, bulkRequest); } catch (NoContentException e) { noContentExceptions.add(e); } } else { executeBulkRequest(bulkRequest); bulkRequest = newContentQueueBulkUpdateRequestBuilder(); } } // execute any remaining bulks requests not executed yet if (bulkRequest.numberOfActions() > 0) { executeBulkRequest(bulkRequest); } // remove any docs without content, so we don't try to index them again if (!noContentExceptions.isEmpty()) { for (NoContentException noContentException : noContentExceptions) { deleteDocument(noContentException); } } lastLoad = System.currentTimeMillis(); if (hits.length > 0) { getLog().info("Finished indexing " + hits.length + " docs in " + ((lastLoad - startTime)) + " ms for index builder " + getName()); } } protected void processContentQueueEntry(SearchHit hit, BulkRequestBuilder bulkRequest) throws NoContentException { String reference = getFieldFromSearchHit(SearchService.FIELD_REFERENCE, hit); EntityContentProducer ecp = newEntityContentProducer(reference); if (ecp != null) { //updating was causing issues without a _source, so doing delete and re-add try { deleteDocument(hit); bulkRequest.add(prepareIndex(reference, ecp, true)); } catch (NoContentException e) { throw e; } catch (Exception e) { getLog().error("Failed to process content queue entry with id [" + hit.getId() + "] in index builder [" + getName() + "]", e); } } else { noContentProducerForContentQueueEntry(hit, reference); } } protected void executeBulkRequest(BulkRequestBuilder bulkRequest) { BulkResponse bulkResponse = bulkRequest.execute().actionGet(); getLog().info("Bulk request of batch size: " + bulkRequest.numberOfActions() + " took " + bulkResponse.getTookInMillis() + " ms in index builder: " + getName()); for (BulkItemResponse response : bulkResponse.getItems()) { if (response.getResponse() instanceof DeleteResponse) { DeleteResponse deleteResponse = response.getResponse(); if (response.isFailed()) { getLog().error("Problem deleting doc: " + response.getId() + " in index builder: " + getName() + " error: " + response.getFailureMessage()); } else if (!deleteResponse.isFound()) { getLog().debug("ES could not find a doc with id: " + deleteResponse.getId() + " to delete in index builder: " + getName()); } else { getLog().debug("ES deleted a doc with id: " + deleteResponse.getId() + " in index builder: " + getName()); } } else if (response.getResponse() instanceof IndexResponse) { IndexResponse indexResponse = response.getResponse(); if (response.isFailed()) { getLog().error("Problem updating content for doc: " + response.getId() + " in index builder: " + getName() + " error: " + response.getFailureMessage()); } else { getLog().debug("ES indexed content for doc with id: " + indexResponse.getId() + " in index builder: " + getName()); } } } } protected void noContentProducerForContentQueueEntry(SearchHit hit, String reference) throws NoContentException { // if there is no content to index remove the doc, its pointless to have it included in the index // and we will just waste cycles looking at it again everytime this thread runs, and will probably // never finish because of it. throw new NoContentException(hit.getId(), reference, null); } protected BulkRequestBuilder newContentQueueBulkUpdateRequestBuilder() { return client.prepareBulk(); } protected SearchResponse findContentQueue() { SearchRequestBuilder searchRequestBuilder = prepareFindContentQueue(); return findContentQueueWithRequest(searchRequestBuilder); } protected SearchRequestBuilder prepareFindContentQueue() { SearchRequestBuilder searchRequestBuilder = newFindContentQueueRequestBuilder(); searchRequestBuilder = addFindContentQueueRequestParams(searchRequestBuilder); searchRequestBuilder = completeFindContentQueueRequestBuilder(searchRequestBuilder); return searchRequestBuilder; } protected SearchRequestBuilder newFindContentQueueRequestBuilder() { return client.prepareSearch(indexName); } protected SearchRequestBuilder addFindContentQueueRequestParams(SearchRequestBuilder searchRequestBuilder) { return searchRequestBuilder.setQuery(matchAllQuery()).setTypes(indexedDocumentType) .setPostFilter(orFilter(missingFilter(SearchService.FIELD_INDEXED), termFilter(SearchService.FIELD_INDEXED, false))) .setSize(contentIndexBatchSize) .addFields(SearchService.FIELD_REFERENCE, SearchService.FIELD_SITEID); } protected abstract SearchRequestBuilder completeFindContentQueueRequestBuilder( SearchRequestBuilder searchRequestBuilder); protected SearchResponse findContentQueueWithRequest(SearchRequestBuilder searchRequestBuilder) { return searchRequestBuilder.execute().actionGet(); } protected void deleteDocument(SearchHit searchHit) { final Map<String, Object> deleteParams = extractDeleteDocumentParams(searchHit); deleteDocumentWithParams(deleteParams); } protected void deleteDocument(NoContentException noContentException) { final Map<String, Object> deleteParams = extractDeleteDocumentParams(noContentException); deleteDocumentWithParams(deleteParams); } protected void deleteDocumentWithParams(Map<String, Object> deleteParams) { final DeleteRequestBuilder deleteRequestBuilder = prepareDeleteDocument(deleteParams); final DeleteResponse deleteResponse = deleteDocumentWithRequest(deleteRequestBuilder); if (getLog().isDebugEnabled()) { if (!deleteResponse.isFound()) { getLog().debug( "Could not delete doc with by id: " + deleteParams.get(DELETE_RESOURCE_KEY_DOCUMENT_ID) + " in index builder [" + getName() + "] because the document wasn't found"); } else { getLog().debug("ES deleted a doc with id: " + deleteResponse.getId() + " in index builder [" + getName() + "]"); } } } protected DeleteRequestBuilder prepareDeleteDocument(Map<String, Object> deleteParams) { DeleteRequestBuilder deleteRequestBuilder = newDeleteRequestBuilder(deleteParams); deleteRequestBuilder = completeDeleteRequestBuilder(deleteRequestBuilder, deleteParams); return deleteRequestBuilder; } protected abstract DeleteRequestBuilder completeDeleteRequestBuilder(DeleteRequestBuilder deleteRequestBuilder, Map<String, Object> deleteParams); protected DeleteResponse deleteDocumentWithRequest(DeleteRequestBuilder deleteRequestBuilder) { return deleteRequestBuilder.execute().actionGet(); } private DeleteRequestBuilder newDeleteRequestBuilder(Map<String, Object> deleteParams) { return client.prepareDelete(indexName, indexedDocumentType, (String) deleteParams.get(DELETE_RESOURCE_KEY_DOCUMENT_ID)); } protected Map<String, Object> extractDeleteDocumentParams(SearchHit searchHit) { final Map<String, Object> params = Maps.newHashMap(); params.put(DELETE_RESOURCE_KEY_DOCUMENT_ID, searchHit.getId()); return params; } protected Map<String, Object> extractDeleteDocumentParams(NoContentException noContentException) { final Map<String, Object> params = Maps.newHashMap(); params.put(DELETE_RESOURCE_KEY_DOCUMENT_ID, noContentException.getId()); params.put(DELETE_RESOURCE_KEY_ENTITY_REFERENCE, noContentException.getReference()); return params; } protected void initializeIndex() { // init index and kick off rebuild if necessary if (rebuildIndexOnStartup) { rebuildIndex(); } else { assureIndex(); } } /** * creates a new index if one does not exist */ protected void assureIndex() { IndicesExistsResponse response = client.admin().indices().exists(new IndicesExistsRequest(indexName)) .actionGet(); if (!response.isExists()) { createIndex(); } } /** * creates a new index, does not check if the exist exists */ protected void createIndex() { try { CreateIndexResponse createResponse = client.admin().indices().create(new CreateIndexRequest(indexName) .settings(indexSettingsMerged).mapping(indexedDocumentType, mappingMerged)).actionGet(); if (!createResponse.isAcknowledged()) { getLog().error("Index wasn't created for index builder [" + getName() + "], can't rebuild"); } } catch (IndexAlreadyExistsException e) { getLog().warn("Index already created for index builder [" + getName() + "]"); } } /** * removes any existing index and creates a new one */ protected void recreateIndex() { IndicesExistsResponse response = client.admin().indices().exists(new IndicesExistsRequest(indexName)) .actionGet(); if (response.isExists()) { client.admin().indices().delete(new DeleteIndexRequest(indexName)).actionGet(); } // create index createIndex(); } /** * Removes any existing index, creates a new index, and rebuilds the index from the entities own stored state {@inheritDoc} */ @Override public void rebuildIndex() { recreateIndex(); if (testMode) { rebuildIndexImmediately(); return; } backgroundScheduler.schedule(newRebuildIndexTask(), 0); } protected abstract void rebuildIndexImmediately(); /** * refresh the index from the current stored state {@inheritDoc} */ public void refreshIndex() { RefreshResponse response = client.admin().indices().refresh(new RefreshRequest(indexName)).actionGet(); } /** * * @param resourceName * @param ecp * @return */ protected IndexRequestBuilder prepareIndex(String resourceName, EntityContentProducer ecp, boolean includeContent) throws IOException, NoContentException { IndexRequestBuilder requestBuilder = newIndexRequestBuilder(resourceName, ecp, includeContent); final XContentBuilder requestContentSource = buildIndexRequestContentSource(resourceName, ecp, includeContent); requestBuilder = requestBuilder.setSource(requestContentSource); return completeIndexRequestBuilder(requestBuilder, resourceName, ecp, includeContent); } protected IndexRequestBuilder newIndexRequestBuilder(String resourceName, EntityContentProducer ecp, boolean includeContent) throws IOException { return client.prepareIndex(indexName, indexedDocumentType, ecp.getId(resourceName)); } protected abstract IndexRequestBuilder completeIndexRequestBuilder(IndexRequestBuilder requestBuilder, String resourceName, EntityContentProducer ecp, boolean includeContent) throws IOException; protected XContentBuilder buildIndexRequestContentSource(String resourceName, EntityContentProducer ecp, boolean includeContent) throws NoContentException, IOException { XContentBuilder requestBuilder = newIndexRequestContentSourceBuilder(resourceName, ecp, includeContent); requestBuilder = addFields(requestBuilder, resourceName, ecp, includeContent); requestBuilder = addCustomProperties(requestBuilder, resourceName, ecp, includeContent); requestBuilder = addContent(requestBuilder, resourceName, ecp, includeContent); return completeIndexRequestContentSourceBuilder(requestBuilder, resourceName, ecp, includeContent); } protected XContentBuilder newIndexRequestContentSourceBuilder(String resourceName, EntityContentProducer ecp, boolean includeContent) throws IOException { return jsonBuilder().startObject(); } protected abstract XContentBuilder addFields(XContentBuilder contentSourceBuilder, String resourceName, EntityContentProducer ecp, boolean includeContent) throws IOException; protected XContentBuilder addCustomProperties(XContentBuilder contentSourceBuilder, String resourceName, EntityContentProducer ecp, boolean includeContent) throws IOException { Map<String, Collection<String>> properties = extractCustomProperties(resourceName, ecp); for (Map.Entry<String, Collection<String>> entry : properties.entrySet()) { contentSourceBuilder = contentSourceBuilder.field(entry.getKey(), entry.getValue()); } return contentSourceBuilder; } protected XContentBuilder addContent(XContentBuilder contentSourceBuilder, String resourceName, EntityContentProducer ecp, boolean includeContent) throws NoContentException, IOException { if (includeContent || testMode) { String content = ecp.getContent(resourceName); // some of the ecp impls produce content with nothing but whitespace, its waste of time to index those if (StringUtils.isNotBlank(content)) { return contentSourceBuilder // cannot rely on ecp for providing something reliable to maintain index state // indexed indicates if the document was indexed .field(SearchService.FIELD_INDEXED, true).field(SearchService.FIELD_CONTENTS, content); } else { return noContentForIndexRequest(contentSourceBuilder, resourceName, ecp, includeContent); } } return contentSourceBuilder; } protected abstract XContentBuilder noContentForIndexRequest(XContentBuilder contentSourceBuilder, String resourceName, EntityContentProducer ecp, boolean includeContent) throws NoContentException; protected XContentBuilder completeIndexRequestContentSourceBuilder(XContentBuilder contentSourceBuilder, String resourceName, EntityContentProducer ecp, boolean includeContent) throws IOException { return contentSourceBuilder.endObject(); } /** * * @param resourceName * @param ecp * @return */ protected void prepareIndexAdd(String resourceName, EntityContentProducer ecp, boolean includeContent) throws NoContentException { try { prepareIndex(resourceName, ecp, includeContent).execute().actionGet(); } catch (NoContentException e) { throw e; } catch (Throwable t) { getLog().error("Error: trying to register resource " + resourceName + " in index builder: " + getName(), t); } } /** * schedules content for indexing. * @param resourceName * @param ecp * @return */ protected void indexAdd(String resourceName, EntityContentProducer ecp) { try { prepareIndexAdd(resourceName, ecp, false); } catch (NoContentException e) { deleteDocument(e); } catch (Exception e) { getLog().error("Problem updating content indexing in index builder: " + getName() + " for entity: " + resourceName, e); } } @Override public int getPendingDocuments() { try { CountResponse response = client.prepareCount(indexName) .setQuery(filteredQuery(matchAllQuery(), orFilter(missingFilter(SearchService.FIELD_INDEXED), termFilter(SearchService.FIELD_INDEXED, false)))) .execute().actionGet(); return (int) response.getCount(); } catch (Exception e) { getLog().error("Problem getting pending docs for index builder [" + getName() + "]", e); } return 0; } @Override public boolean isBuildQueueEmpty() { return getPendingDocuments() == 0; } @Override public void addResource(Notification notification, Event event) { getLog().debug("Add resource " + notification + "::" + event + " in index builder " + getName()); final Map<String, Object> validationContext; try { validationContext = validateAddResourceEvent(event); } catch (Exception e) { // Only debug level b/c almost all runtime validation failures will be uninteresting. // In almost all cases they'll be caused by either the absence of a capable SearchableContentHandler, // and actually that can be expected to happen *a lot* since the historical EntityContentProducer // registration mechanism set events of interest on the search *service*, not on the index builder to // which it (the producer) was actually bound. And that registration does not include a pointer back to the // registering producer. So without modifying all providers in the wild, index builders will just have // to deal with event storms they don't care about. getLog().debug("Skipping index for event " + event + " in index builder [" + getName() + "] because it did not validate", e); return; } dispatchValidatedAddResource(validationContext); } protected Map<String, Object> validateAddResourceEvent(Event event) throws IllegalArgumentException, IllegalStateException { final Map<String, Object> validationContext = Maps.newHashMap(); validateServiceEnabled(event, validationContext); validateResourceName(event, validationContext); validateContentProducer(event, validationContext); validateIndexable(event, validationContext); validateIndexAction(event, validationContext); completeAddResourceEventValidations(event, validationContext); return validationContext; } protected void validateServiceEnabled(Event event, Map<String, Object> validationContext) throws IllegalArgumentException, IllegalStateException { if (!isEnabled()) { throw new IllegalStateException("ElasticSearch is not enabled. Set search.enable=true to change that."); } } protected void validateResourceName(Event event, Map<String, Object> validationContext) throws IllegalArgumentException, IllegalStateException { String resourceName = event.getResource(); if (resourceName == null) { // default if null (historical behavior) resourceName = ""; } if (resourceName.length() > 255) { throw new IllegalArgumentException( "Entity Reference is longer than 255 characters. Reference=" + resourceName); } validationContext.put(ADD_RESOURCE_VALIDATION_KEY_RESOURCE_NAME, resourceName); } protected void validateContentProducer(Event event, Map<String, Object> validationContext) throws IllegalArgumentException, IllegalStateException { final EntityContentProducer ecp = newEntityContentProducer(event); if (ecp == null) { throw new IllegalArgumentException("No registered SearchableContentProducer for event [" + event + "] in indexBuilder [" + getName() + "]"); } validationContext.put(ADD_RESOURCE_VALIDATION_KEY_CONTENT_PRODUCER, ecp); } protected void validateIndexable(Event event, Map<String, Object> validationContext) throws IllegalArgumentException, IllegalStateException { final EntityContentProducer ecp = (EntityContentProducer) validationContext .get(ADD_RESOURCE_VALIDATION_KEY_CONTENT_PRODUCER); final String resourceName = (String) validationContext.get(ADD_RESOURCE_VALIDATION_KEY_RESOURCE_NAME); final String id = ecp.getId(resourceName); if (StringUtils.isEmpty(id)) { throw new IllegalArgumentException("Entity ID could not be derived from resource name [" + resourceName + "] for event [" + event + "] in index builder [" + getName() + "]"); } validationContext.put(ADD_RESOURCE_VALIDATION_KEY_ENTITY_ID, id); } protected void validateIndexAction(Event event, Map<String, Object> validationContext) throws IllegalArgumentException, IllegalStateException, UnsupportedOperationException { final EntityContentProducer ecp = (EntityContentProducer) validationContext .get(ADD_RESOURCE_VALIDATION_KEY_CONTENT_PRODUCER); final IndexAction action = IndexAction.getAction(ecp.getAction(event)); if (!(isSupportedIndexAction(action))) { throw new UnsupportedOperationException("Event [" + event + "] resolved to an unsupported IndexAction [" + action + "] in index builder [" + getName() + "]"); } validationContext.put(ADD_RESOURCE_VALIDATION_KEY_INDEX_ACTION, action); } protected boolean isSupportedIndexAction(IndexAction action) { return IndexAction.ADD.equals(action) || IndexAction.DELETE.equals(action); } protected abstract void completeAddResourceEventValidations(Event event, Map<String, Object> validationContext) throws IllegalArgumentException, IllegalStateException; protected void dispatchValidatedAddResource(Map<String, Object> validationContext) { final IndexAction indexAction = (IndexAction) validationContext .get(ADD_RESOURCE_VALIDATION_KEY_INDEX_ACTION); final String resourceName = (String) validationContext.get(ADD_RESOURCE_VALIDATION_KEY_RESOURCE_NAME); final EntityContentProducer ecp = (EntityContentProducer) validationContext .get(ADD_RESOURCE_VALIDATION_KEY_CONTENT_PRODUCER); getLog().debug("Action on '" + resourceName + "' detected as " + indexAction.name() + " in index builder " + getName()); switch (indexAction) { case ADD: indexAdd(resourceName, ecp); break; case DELETE: deleteDocumentWithParams(extractDeleteDocumentParams(validationContext)); break; default: // Should never happen if validation process was implemented correctly throw new UnsupportedOperationException( indexAction + " is not supported in index builder " + getName()); } } protected Map<String, Object> extractDeleteDocumentParams(Map<String, Object> validationContext) { final Map<String, Object> params = Maps.newHashMap(); params.put(DELETE_RESOURCE_KEY_DOCUMENT_ID, validationContext.get(ADD_RESOURCE_VALIDATION_KEY_ENTITY_ID)); return params; } /** * Extract properties from the given {@link EntityContentProducer} * <p> * The {@link EntityContentProducer#getCustomProperties(String)} method returns a map of different kind of elements. * To avoid casting and calls to {@code instanceof}, extractCustomProperties does all the work and returns a formated * map containing only {@link Collection<String>}. * </p> * * @param resourceName affected resource * @param contentProducer producer providing properties for the given resource * @return a formated map of {@link Collection<String>} */ protected Map<String, Collection<String>> extractCustomProperties(String resourceName, EntityContentProducer contentProducer) { Map<String, ?> m = contentProducer.getCustomProperties(resourceName); if (m == null) return Collections.emptyMap(); Map<String, Collection<String>> properties = new HashMap<String, Collection<String>>(m.size()); for (Map.Entry<String, ?> propertyEntry : m.entrySet()) { String propertyName = propertyEntry.getKey(); Object propertyValue = propertyEntry.getValue(); Collection<String> values; //Check for basic data type that could be provided by the EntityContentProducer //If the data type can't be defined, nothing is stored. The toString method could be called, but some values //could be not meant to be indexed. if (propertyValue instanceof String) values = Collections.singleton((String) propertyValue); else if (propertyValue instanceof String[]) values = Arrays.asList((String[]) propertyValue); else if (propertyValue instanceof Collection) values = (Collection<String>) propertyValue; else { if (propertyValue != null) getLog().warn("Couldn't find what the value for '" + propertyName + "' was. It has been ignored. " + propertyName.getClass()); values = Collections.emptyList(); } //If this property was already present there (this shouldn't happen, but if it does everything must be stored if (properties.containsKey(propertyName)) { getLog().warn("Two properties had a really similar name and were merged. This shouldn't happen! " + propertyName); getLog().debug("Merged values '" + properties.get(propertyName) + "' with '" + values); values = new ArrayList<String>(values); values.addAll(properties.get(propertyName)); } properties.put(propertyName, values); } return properties; } @Override public SearchResponse search(String searchTerms, List<String> references, List<String> siteIds, int start, int end) { final Pair<SearchRequestBuilder, QueryBuilder> searchBuilders = prepareSearchRequest(searchTerms, references, siteIds, start, end); final SearchRequestBuilder searchRequestBuilder = searchBuilders.getLeft(); final QueryBuilder queryBuilder = searchBuilders.getRight(); getLog().debug("Search request from index builder [" + getName() + "]: " + searchRequestBuilder.toString()); SearchResponse response = searchRequestBuilder.execute().actionGet(); getLog().debug( "Search request from index builder [" + getName() + "] took: " + response.getTook().format()); eventTrackingService.post(eventTrackingService.newEvent(SearchService.EVENT_SEARCH, SearchService.EVENT_SEARCH_REF + queryBuilder.toString(), true, NotificationService.PREF_IMMEDIATE)); return response; } @Override public SearchResponse search(String searchTerms, List<String> references, List<String> siteIds, int start, int end, Map<String, String> additionalSearchInformation) { //The additional information will be used in specific indexes, so this method can be overrided in the index // to make use of that field. return search(searchTerms, references, siteIds, start, end); } protected Pair<SearchRequestBuilder, QueryBuilder> prepareSearchRequest(String searchTerms, List<String> references, List<String> siteIds, int start, int end) { // All this Pair<SearchRequestBuilder,QueryBuilder> business b/c: // a) Legacy eventing in search() needs the QueryBuilder, not just the SearchRequestBuilder, and // b) SiteId handling entails manipulation of both objects, so presumably completeSearchRequestBuilders() // would as well // c) There is no getQuery() on SearchRequestBuilder Pair<SearchRequestBuilder, QueryBuilder> builders = newSearchRequestAndQueryBuilders(searchTerms, references, siteIds); builders = addSearchCoreParams(builders, searchTerms, references, siteIds); builders = addSearchQuery(builders, searchTerms, references, siteIds); builders = pairOf(addSearchResultFields(builders.getLeft()), builders.getRight()); builders = pairOf(addSearchPagination(builders.getLeft(), start, end), builders.getRight()); builders = pairOf(addSearchFacetting(builders.getLeft()), builders.getRight()); return completeSearchRequestBuilders(builders, searchTerms, references, siteIds); } protected Pair<SearchRequestBuilder, QueryBuilder> newSearchRequestAndQueryBuilders(String searchTerms, List<String> references, List<String> siteIds) { return pairOf(client.prepareSearch(indexName), boolQuery()); } protected Pair<SearchRequestBuilder, QueryBuilder> addSearchCoreParams( Pair<SearchRequestBuilder, QueryBuilder> builders, String searchTerms, List<String> references, List<String> siteIds) { final SearchRequestBuilder searchRequestBuilder = builders.getLeft(); return pairOf(searchRequestBuilder.setSearchType(SearchType.QUERY_THEN_FETCH).setTypes(indexedDocumentType), builders.getRight()); } protected Pair<SearchRequestBuilder, QueryBuilder> addSearchQuery( Pair<SearchRequestBuilder, QueryBuilder> builders, String searchTerms, List<String> references, List<String> siteIds) { builders = addSearchTerms(builders, searchTerms); builders = addSearchReferences(builders, references); builders = addSearchSiteIds(builders, siteIds); return pairOf(builders.getLeft().setQuery(builders.getRight()), builders.getRight()); } protected Pair<SearchRequestBuilder, QueryBuilder> addSearchTerms( Pair<SearchRequestBuilder, QueryBuilder> builders, String searchTerms) { BoolQueryBuilder query = (BoolQueryBuilder) builders.getRight(); if (searchTerms.contains(":")) { String[] termWithType = searchTerms.split(":"); String termType = termWithType[0]; String termValue = termWithType[1]; // little fragile but seems like most providers follow this convention, there isn't a nice way to get the type // without a handle to a reference. query = query.must(termQuery(SearchService.FIELD_TYPE, "sakai:" + termType)); query = query.must(matchQuery(SearchService.FIELD_CONTENTS, termValue)); } else { query = query.must(matchQuery(SearchService.FIELD_CONTENTS, searchTerms)); } return pairOf(builders.getLeft(), query); } protected Pair<SearchRequestBuilder, QueryBuilder> addSearchReferences( Pair<SearchRequestBuilder, QueryBuilder> builders, List<String> references) { BoolQueryBuilder query = (BoolQueryBuilder) builders.getRight(); if (references.size() > 0) { query = query.must( termsQuery(SearchService.FIELD_REFERENCE, references.toArray(new String[references.size()]))); } return pairOf(builders.getLeft(), query); } protected abstract Pair<SearchRequestBuilder, QueryBuilder> addSearchSiteIds( Pair<SearchRequestBuilder, QueryBuilder> builders, List<String> siteIds); protected SearchRequestBuilder addSearchResultFields(SearchRequestBuilder searchRequestBuilder) { if (ArrayUtils.isEmpty(searchResultFieldNames)) { return searchRequestBuilder; } return searchRequestBuilder.addFields(searchResultFieldNames); } protected SearchRequestBuilder addSearchPagination(SearchRequestBuilder searchRequestBuilder, int start, int end) { return searchRequestBuilder.setFrom(start).setSize(end - start); } protected SearchRequestBuilder addSearchFacetting(SearchRequestBuilder searchRequestBuilder) { if (useFacetting) { return searchRequestBuilder .addFacet(termsFacet(facetName).field("contents.lowercase").size(facetTermSize)); } return searchRequestBuilder; } protected abstract Pair<SearchRequestBuilder, QueryBuilder> completeSearchRequestBuilders( Pair<SearchRequestBuilder, QueryBuilder> builders, String searchTerms, List<String> references, List<String> siteIds); @Override public String[] searchSuggestions(String searchString, String currentSite, boolean allMySites) { if (!useSuggestions) { return new String[0]; } final Pair<SearchRequestBuilder, QueryBuilder> builders = prepareSearchSuggestionsRequest(searchString, currentSite, allMySites); final SearchRequestBuilder searchRequestBuilder = builders.getLeft(); getLog().debug("Search request from index builder [" + getName() + "]: " + searchRequestBuilder); SearchResponse response = searchRequestBuilder.execute().actionGet(); getLog().debug( "Search request from index builder [" + getName() + "] took: " + response.getTook().format()); List<String> suggestions = Lists.newArrayList(); for (SearchHit hit : response.getHits()) { suggestions.add(getFieldFromSearchHit(suggestionMatchingFieldName, hit)); } return suggestions.toArray(new String[suggestions.size()]); } protected Pair<SearchRequestBuilder, QueryBuilder> prepareSearchSuggestionsRequest(String searchString, String currentSite, boolean allMySites) { Pair<SearchRequestBuilder, QueryBuilder> builders = newSearchSuggestionsRequestAndQueryBuilders( searchString, currentSite, allMySites); builders = addSearchSuggestionsCoreParams(builders, searchString, currentSite, allMySites); builders = addSearchSuggestionsQuery(builders, searchString, currentSite, allMySites); builders = pairOf(addSearchSuggestionResultFields(builders.getLeft()), builders.getRight()); builders = pairOf(addSearchSuggestionsPagination(builders.getLeft()), builders.getRight()); return completeSearchSuggestionsRequestBuilders(builders, searchString, currentSite, allMySites); } protected abstract Pair<SearchRequestBuilder, QueryBuilder> completeSearchSuggestionsRequestBuilders( Pair<SearchRequestBuilder, QueryBuilder> builders, String searchString, String currentSite, boolean allMySites); protected Pair<SearchRequestBuilder, QueryBuilder> newSearchSuggestionsRequestAndQueryBuilders( String searchString, String currentSite, boolean allMySites) { return pairOf(client.prepareSearch(indexName), termQuery(suggestionMatchingFieldName, searchString)); } protected Pair<SearchRequestBuilder, QueryBuilder> addSearchSuggestionsCoreParams( Pair<SearchRequestBuilder, QueryBuilder> builders, String searchString, String currentSite, boolean allMySites) { final SearchRequestBuilder searchRequestBuilder = builders.getLeft(); return pairOf(searchRequestBuilder.setSearchType(SearchType.QUERY_THEN_FETCH).setTypes(indexedDocumentType), builders.getRight()); } protected Pair<SearchRequestBuilder, QueryBuilder> addSearchSuggestionsQuery( Pair<SearchRequestBuilder, QueryBuilder> builders, String searchString, String currentSite, boolean allMySites) { builders = addSearchSuggestionsTerms(builders, searchString); builders = addSearchSuggestionsSites(builders, currentSite, allMySites); return pairOf(builders.getLeft().setQuery(builders.getRight()), builders.getRight()); } protected abstract Pair<SearchRequestBuilder, QueryBuilder> addSearchSuggestionsTerms( Pair<SearchRequestBuilder, QueryBuilder> builders, String searchString); protected abstract Pair<SearchRequestBuilder, QueryBuilder> addSearchSuggestionsSites( Pair<SearchRequestBuilder, QueryBuilder> builders, String currentSite, boolean allMySites); protected SearchRequestBuilder addSearchSuggestionResultFields(SearchRequestBuilder searchRequestBuilder) { if (ArrayUtils.isEmpty(suggestionResultFieldNames)) { return searchRequestBuilder; } return searchRequestBuilder.addFields(suggestionResultFieldNames); } protected SearchRequestBuilder addSearchSuggestionsPagination(SearchRequestBuilder searchRequestBuilder) { return searchRequestBuilder.setSize(maxNumberOfSuggestions); } public StringBuilder getStatus(StringBuilder into) { assureIndex(); IndicesStatusResponse response = client.admin().indices().status(new IndicesStatusRequest(indexName)) .actionGet(); IndexStatus status = response.getIndices().get(indexName); long pendingDocs = getPendingDocuments(); into.append("Index builder: ").append(getName()); if (pendingDocs != 0) { into.append(" active. " + pendingDocs + " pending items in queue. "); } else { into.append(" idle. "); } into.append("Index Size: " + roundTwoDecimals(status.getStoreSize().getGbFrac()) + " GB" + " Refresh Time: " + status.getRefreshStats().getTotalTimeInMillis() + "ms" + " Flush Time: " + status.getFlushStats().getTotalTimeInMillis() + "ms" + " Merge Time: " + status.getMergeStats().getTotalTimeInMillis() + "ms"); return into; } protected double roundTwoDecimals(double d) { DecimalFormat twoDForm = new DecimalFormat("#.##"); return Double.valueOf(twoDForm.format(d)); } @Override public int getNDocs() { assureIndex(); CountResponse response = client.prepareCount(indexName) .setQuery(filteredQuery(matchAllQuery(), termFilter(SearchService.FIELD_INDEXED, true))).execute() .actionGet(); return (int) response.getCount(); } @Override public SearchStatus getSearchStatus() { final String lastLoadStr = new Date(lastLoad).toString(); final String loadTimeStr = String.valueOf((double) (0.001 * lastLoad)); final String pdocs = String.valueOf(getPendingDocuments()); final String ndocs = String.valueOf(getNDocs()); return new SearchStatus() { public String getLastLoad() { return lastLoadStr; } public String getLoadTime() { return loadTimeStr; } public String getCurrentWorker() { return null; } public String getCurrentWorkerETC() { return null; } public List getWorkerNodes() { return Collections.EMPTY_LIST; } public String getNDocuments() { return ndocs; } public String getPDocuments() { return pdocs; } }; } /** * Find a {@link EntityContentProducer} capable of handling the given entity reference, or null if no * such producer has been registered. * * @param ref the entity reference */ @Override public EntityContentProducer newEntityContentProducer(String ref) { final Optional<EntityContentProducer> producer = matchEntityContentProducer(p -> p.matches(ref)); if (producer.isPresent()) { getLog().debug("Matched content producer " + producer.get() + " for reference " + ref + " in index builder " + getName()); return producer.get(); } getLog().debug( "Failed to match any content producer for reference " + ref + " in index builder " + getName()); return null; } /** * Find a {@link EntityContentProducer} capable of handling the given {@code Event}, or null if no * such producer has been registered. * * @param event * @return */ @Override public EntityContentProducer newEntityContentProducer(Event event) { final Optional<EntityContentProducer> producer = matchEntityContentProducer(p -> p.matches(event)); if (producer.isPresent()) { getLog().debug("Matched content producer " + producer.get() + " for event " + event + " in index builder " + getName()); return producer.get(); } getLog().debug( "Failed to match any content producer for event " + event + " in index builder " + getName()); return null; } protected Optional<EntityContentProducer> matchEntityContentProducer(Predicate<EntityContentProducer> matcher) { return producers.stream().filter(matcher).findFirst(); } /** * get all the producers registered, as a clone to avoid concurrent * modification exceptions * * @return */ @Override public List<EntityContentProducer> getContentProducers() { return Lists.newArrayList(producers); } /** * register an entity content producer to provide content to the search * engine {@inheritDoc} */ @Override public void registerEntityContentProducer(EntityContentProducer ecp) { getLog().debug("register " + ecp); // no synchronization here b/c: // a) producers collection internally threadsafe, and // b) event registrations are append-only and order isn't important, so interleaved updateEventsFor() calls // will result in the same functional end state // c) we know the updateEventsFor() impl will serialize invocations anyway producers.add(ecp); if (eventRegistrar != null) { eventRegistrar.updateEventsFor(this); } } @Override public Set<String> getContentFunctions() { return producers.stream().filter(ecp -> ecp instanceof EntityContentProducerEvents) .flatMap(ecp -> ((EntityContentProducerEvents) ecp).getTriggerFunctions().stream()) .collect(Collectors.toSet()); } /** * Establish a security advisor to allow the "embedded" work to occur with no need for additional security permissions. */ protected void enableAzgSecurityAdvisor() { // put in a security advisor so we can do our work without need of further permissions securityService.pushAdvisor(allowAllAdvisor); } /** * Disable the security advisor. */ protected void disableAzgSecurityAdvisor() { SecurityAdvisor popped = securityService.popAdvisor(allowAllAdvisor); if (!allowAllAdvisor.equals(popped)) { if (popped == null) { getLog().debug("Someone has removed our advisor."); } else { getLog().debug("Removed someone elses advisor, adding it back."); securityService.pushAdvisor(popped); } } } /** * loads the field from the SearchHit. Loads from field not from source since * we aren't storing the source. * @param field * @param hit * @return */ @Override public String getFieldFromSearchHit(String field, SearchHit hit) { if (hit != null && hit.getFields() != null && hit.getFields().get(field) != null) { return hit.getFields().get(field).value(); } return null; } protected <L, R> Pair<L, R> pairOf(L left, R right) { return new ImmutablePair<>(left, right); } @Override public List<SearchBuilderItem> getAllSearchItems() { return null; } public void setIndexedDocumentType(String indexedDocumentType) { this.indexedDocumentType = indexedDocumentType; } public void setTestMode(boolean testMode) { this.testMode = testMode; } public void setRebuildIndexOnStartup(boolean rebuildIndexOnStartup) { this.rebuildIndexOnStartup = rebuildIndexOnStartup; } @Override public boolean getUseFacetting() { return this.useFacetting; } public void setUseFacetting(boolean useFacetting) { this.useFacetting = useFacetting; } public void setFacetName(String facetName) { this.facetName = facetName; } public void setMaxNumberOfSuggestions(int maxNumberOfSuggestions) { this.maxNumberOfSuggestions = maxNumberOfSuggestions; } public void setUseSuggestions(boolean useSuggestions) { this.useSuggestions = useSuggestions; } public void setSuggestionResultFieldNames(String[] suggestionResultFieldNames) { this.suggestionResultFieldNames = suggestionResultFieldNames; } public void setSuggestionMatchingFieldName(String suggestionMatchingFieldName) { this.suggestionMatchingFieldName = suggestionMatchingFieldName; } public void setSearchResultFieldNames(String[] searchResultFieldNames) { this.searchResultFieldNames = searchResultFieldNames; } public void setFilter(SearchItemFilter filter) { this.filter = filter; } @Override public SearchItemFilter getFilter() { return filter; } @Override public String getFacetName() { return facetName; } public void setFacetTermSize(int facetTermSize) { this.facetTermSize = facetTermSize; } @Override public String getName() { return this.name; } public void setName(String name) { this.name = name; } public void setIndexName(String indexName) { //elasticsearch wants lowers case index names this.indexName = indexName.toLowerCase(); } public void setDefaultIndexSettingsResource(String defaultIndexSettingsResource) { this.defaultIndexSettingsResource = defaultIndexSettingsResource; } public void setMapping(String mapping) { this.mapping = mapping; } public void setIndexSettings(String indexSettings) { this.indexSettings = indexSettings; } public void setDefaultMappingResource(String defaultMappingResource) { this.defaultMappingResource = defaultMappingResource; } public void setDelay(int delay) { this.delay = delay; } public void setPeriod(int period) { this.period = period; } public void setContentIndexBatchSize(int contentIndexBatchSize) { this.contentIndexBatchSize = contentIndexBatchSize; } public void setBulkRequestSize(int bulkRequestSize) { this.bulkRequestSize = bulkRequestSize; } public void setSecurityService(SecurityService securityService) { this.securityService = securityService; } public void setServerConfigurationService(ServerConfigurationService serverConfigurationService) { this.serverConfigurationService = serverConfigurationService; } public void setEventTrackingService(EventTrackingService eventTrackingService) { this.eventTrackingService = eventTrackingService; } public void setTriggerFunctions(Collection<String> triggerFunctions) { this.triggerFunctions = (triggerFunctions instanceof Set) ? (Set) triggerFunctions : Sets.newHashSet(triggerFunctions); } @Override public Set<String> getTriggerFunctions() { return triggerFunctions; } @Override public List<SearchBuilderItem> getGlobalMasterSearchItems() { return Collections.emptyList(); } protected abstract Logger getLog(); public enum IndexAction { /** * Action Unknown, usually because the record has just been created */ UNKNOWN(SearchBuilderItem.ACTION_UNKNOWN), /** * Action ADD the record to the search engine, if the doc ID is set, then * remove first, if not set, check its not there. */ ADD(SearchBuilderItem.ACTION_ADD), /** * Action DELETE the record from the search engine, once complete delete the * record */ DELETE(SearchBuilderItem.ACTION_DELETE), /** * The action REBUILD causes the indexer thread to rebuild the index from * scratch, re-fetching all entities This should only ever appear on the * master record */ REBUILD(SearchBuilderItem.ACTION_REBUILD), /** * The action REFRESH causes the indexer thread to refresh the search index * from the current set of entities. If a Rebuild is in progress, the * refresh will not override the rebuild */ REFRESH(SearchBuilderItem.ACTION_REFRESH); private final int itemAction; private IndexAction(int itemAction) { this.itemAction = itemAction; } /** * Generate an IndexAction based on an action ID provided by the Search API * * @param itemActionId action ID used by the Search API * @return IndexAction matching the given ID, null if nothing has been found */ public static IndexAction getAction(int itemActionId) { for (IndexAction indexAction : values()) { if (indexAction.getItemAction() == itemActionId) return indexAction; } return null; } public int getItemAction() { return itemAction; } } }