org.alfresco.solr.SolrInformationServer.java Source code

Java tutorial

Introduction

Here is the source code for org.alfresco.solr.SolrInformationServer.java

Source

/*
 * #%L
 * Alfresco Solr 4
 * %%
 * Copyright (C) 2005 - 2016 Alfresco Software Limited
 * %%
 * This file is part of the Alfresco software. 
 * If the software was purchased under a paid Alfresco license, the terms of 
 * the paid license agreement will prevail.  Otherwise, the software is 
 * provided under the following open source license terms:
 * 
 * Alfresco is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 * 
 * Alfresco is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 * 
 * You should have received a copy of the GNU Lesser General Public License
 * along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
 * #L%
 */
package org.alfresco.solr;

import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_ACLID;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_ACLTXCOMMITTIME;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_ACLTXID;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_ANCESTOR;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_ASPECT;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_ASSOCTYPEQNAME;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_DBID;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_DENIED;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_DOC_TYPE;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_EXCEPTION_MESSAGE;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_EXCEPTION_STACK;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_FIELDS;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_FTSSTATUS;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_GEO;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_INACLTXID;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_INTXID;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_ISNODE;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_LID;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_NPATH;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_NULLPROPERTIES;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_OWNER;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_PARENT;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_PARENT_ASSOC_CRC;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_PATH;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_PNAME;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_PRIMARYASSOCQNAME;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_PRIMARYASSOCTYPEQNAME;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_PRIMARYPARENT;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_PROPERTIES;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_QNAME;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_READER;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_SITE;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_SOLR4_ID;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_TAG_SUGGEST;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_S_ACLTXCOMMITTIME;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_S_ACLTXID;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_S_INACLTXID;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_S_INTXID;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_S_TXCOMMITTIME;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_S_TXID;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_TAG;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_TENANT;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_TXCOMMITTIME;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_TXID;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_TYPE;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_VERSION;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_CASCADETX;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.io.UnsupportedEncodingException;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.EnumMap;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;

import javax.management.*;

import org.alfresco.httpclient.AuthenticationException;
import org.alfresco.model.ContentModel;
import org.alfresco.opencmis.dictionary.CMISStrictDictionaryService;
import org.alfresco.repo.content.ContentContext;
import org.alfresco.repo.dictionary.DictionaryComponent;
import org.alfresco.repo.dictionary.M2Model;
import org.alfresco.repo.dictionary.NamespaceDAO;
import org.alfresco.repo.search.adaptor.lucene.QueryConstants;
import org.alfresco.service.cmr.dictionary.AspectDefinition;
import org.alfresco.service.cmr.dictionary.TypeDefinition;
import org.alfresco.service.cmr.repository.ChildAssociationRef;
import org.alfresco.service.cmr.repository.ContentReader;
import org.alfresco.service.cmr.repository.ContentWriter;
import org.alfresco.service.cmr.repository.NodeRef;
import org.alfresco.service.cmr.repository.datatype.DefaultTypeConverter;
import org.alfresco.service.cmr.security.AuthorityType;
import org.alfresco.service.namespace.QName;
import org.alfresco.solr.AlfrescoSolrDataModel.FieldInstance;
import org.alfresco.solr.AlfrescoSolrDataModel.IndexedField;
import org.alfresco.solr.AlfrescoSolrDataModel.TenantAclIdDbId;
import org.alfresco.solr.adapters.IOpenBitSet;
import org.alfresco.solr.adapters.ISimpleOrderedMap;
import org.alfresco.solr.adapters.SolrOpenBitSetAdapter;
import org.alfresco.solr.adapters.SolrSimpleOrderedMap;
import org.alfresco.solr.client.AclChangeSet;
import org.alfresco.solr.client.AclReaders;
import org.alfresco.solr.client.AlfrescoModel;
import org.alfresco.solr.client.ContentPropertyValue;
import org.alfresco.solr.client.MLTextPropertyValue;
import org.alfresco.solr.client.MultiPropertyValue;
import org.alfresco.solr.client.Node;
import org.alfresco.solr.client.Node.SolrApiNodeStatus;
import org.alfresco.solr.client.NodeMetaData;
import org.alfresco.solr.client.NodeMetaDataParameters;
import org.alfresco.solr.client.PropertyValue;
import org.alfresco.solr.client.SOLRAPIClient;
import org.alfresco.solr.client.SOLRAPIClient.GetTextContentResponse;
import org.alfresco.solr.client.StringPropertyValue;
import org.alfresco.solr.client.Transaction;
import org.alfresco.solr.content.SolrContentStore;
import org.alfresco.solr.content.SolrContentUrlBuilder;
import org.alfresco.solr.tracker.IndexHealthReport;
import org.alfresco.solr.tracker.TrackerStats;
import org.alfresco.util.ISO9075;
import org.alfresco.util.Pair;
import org.apache.commons.io.input.BoundedInputStream;
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.JavaBinCodec;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.core.IndexDeletionPolicyWrapper;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrInfoMBean;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.response.ResultContext;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.search.DocIterator;
import org.apache.solr.search.DocList;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.update.AddUpdateCommand;
import org.apache.solr.update.CommitUpdateCommand;
import org.apache.solr.update.DeleteUpdateCommand;
import org.apache.solr.update.RollbackUpdateCommand;
import org.apache.solr.update.processor.UpdateRequestProcessor;
import org.apache.solr.util.ConcurrentLRUCache;
import org.apache.solr.util.RefCounted;
import org.json.JSONException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.extensions.surf.util.I18NUtil;
import org.springframework.util.FileCopyUtils;

/**
 * This is the Solr4 implementation of the information server (index).
 * @author Ahmed Owian
 * @since 5.0
 */
public class SolrInformationServer implements InformationServer {
    private static final String NO_SITE = "_REPOSITORY_";
    private static final String SHARED_FILES = "_SHARED_FILES_";
    public static final String AND = " AND ";
    public static final String OR = " OR ";
    public static final String REQUEST_HANDLER_ALFRESCO_FULL_TEXT_SEARCH = "/afts";
    public static final String REQUEST_HANDLER_NATIVE = "/native";
    public static final String REQUEST_HANDLER_ALFRESCO = "/alfresco";
    public static final String REQUEST_HANDLER_SELECT = "/select";
    public static final String REQUEST_HANDLER_GET = "/get";
    public static final String RESPONSE_DEFAULT_IDS = "response";
    public static final String RESPONSE_DEFAULT_ID = "doc";

    public static final String PREFIX_ERROR = "ERROR-";

    public static final String DOC_TYPE_NODE = "Node";
    public static final String DOC_TYPE_UNINDEXED_NODE = "UnindexedNode";
    public static final String DOC_TYPE_ERROR_NODE = "ErrorNode";
    public static final String DOC_TYPE_ACL = "Acl";
    public static final String DOC_TYPE_TX = "Tx";
    public static final String DOC_TYPE_ACL_TX = "AclTx";
    public static final String DOC_TYPE_STATE = "State";

    private static final Pattern CAPTURE_SITE = Pattern.compile(
            "^/\\{http\\://www\\.alfresco\\.org/model/application/1\\.0\\}company\\_home/\\{http\\://www\\.alfresco\\.org/model/site/1\\.0\\}sites/\\{http\\://www\\.alfresco\\.org/model/content/1\\.0}([^/]*)/.*");
    private static final Pattern CAPTURE_TAG = Pattern.compile(
            "^/\\{http\\://www\\.alfresco\\.org/model/content/1\\.0\\}taggable/\\{http\\://www\\.alfresco\\.org/model/content/1\\.0\\}([^/]*)/\\{\\}member");
    private static final Pattern CAPTURE_SHARED_FILES = Pattern.compile(
            "^/\\{http\\://www\\.alfresco\\.org/model/application/1\\.0\\}company\\_home/\\{http\\://www\\.alfresco\\.org/model/application/1\\.0\\}shared/.*");

    /* 4096 is 2 to the power of (6*2), and we do this because the precision step for the long is 6, 
     * and the transactions are long
     */
    public static final int BATCH_FACET_TXS = 4096;

    private AlfrescoCoreAdminHandler adminHandler;
    private SolrCore core;
    private SolrRequestHandler nativeRequestHandler;
    private Cloud cloud;
    private TrackerStats trackerStats = new TrackerStats(this);
    private AlfrescoSolrDataModel dataModel;
    private SolrContentStore solrContentStore;
    private String alfrescoVersion;
    private boolean transformContent = true;
    private boolean recordUnindexedNodes = true;
    private long lag;
    private long holeRetention;
    private int contentStreamLimit;

    // Metadata pulling control
    private boolean skipDescendantDocsForSpecificTypes;
    private boolean skipDescendantDocsForSpecificAspects;
    private Set<QName> typesForSkippingDescendantDocs = new HashSet<QName>();
    private Set<QName> aspectsForSkippingDescendantDocs = new HashSet<QName>();
    private String skippingDocsQueryString;
    private SOLRAPIClient repositoryClient;
    private boolean isSkippingDocsInitialized = false;

    protected final static Logger log = LoggerFactory.getLogger(SolrInformationServer.class);

    protected enum FTSStatus {
        New, Dirty, Clean
    };

    private ConcurrentLRUCache<String, Boolean> isIdIndexCache = new ConcurrentLRUCache<String, Boolean>(
            60 * 60 * 100, 60 * 60 * 50);

    private ReentrantReadWriteLock activeTrackerThreadsLock = new ReentrantReadWriteLock();

    private HashSet<Long> activeTrackerThreads = new HashSet<Long>();

    private ReentrantReadWriteLock commitAndRollbackLock = new ReentrantReadWriteLock();

    private int port;

    private String hostName;

    private String baseUrl;

    // write a BytesRef as a byte array
    JavaBinCodec.ObjectResolver resolver = new JavaBinCodec.ObjectResolver() {
        @Override
        public Object resolve(Object o, JavaBinCodec codec) throws IOException {
            if (o instanceof BytesRef) {
                BytesRef br = (BytesRef) o;
                codec.writeByteArray(br.bytes, br.offset, br.length);
                return null;
            }
            return o;
        }
    };

    @Override
    public AlfrescoCoreAdminHandler getAdminHandler() {
        return this.adminHandler;
    }

    public SolrInformationServer(AlfrescoCoreAdminHandler adminHandler, SolrCore core,
            SOLRAPIClient repositoryClient, SolrContentStore solrContentStore) {
        this.adminHandler = adminHandler;
        this.core = core;
        this.nativeRequestHandler = core.getRequestHandler(REQUEST_HANDLER_NATIVE);
        this.cloud = new Cloud();
        this.repositoryClient = repositoryClient;
        this.solrContentStore = solrContentStore;

        Properties p = core.getResourceLoader().getCoreProperties();
        alfrescoVersion = p.getProperty("alfresco.version", "5.0.0");
        transformContent = Boolean.parseBoolean(p.getProperty("alfresco.index.transformContent", "true"));
        recordUnindexedNodes = Boolean.parseBoolean(p.getProperty("alfresco.recordUnindexedNodes", "true"));
        lag = Integer.parseInt(p.getProperty("alfresco.lag", "1000"));
        holeRetention = Integer.parseInt(p.getProperty("alfresco.hole.retention", "3600000"));

        dataModel = AlfrescoSolrDataModel.getInstance();

        contentStreamLimit = Integer.parseInt(p.getProperty("alfresco.contentStreamLimit", "10000000"));

        // build base URL - host and port have to come from configuration.

        Properties props = AlfrescoSolrDataModel.getCommonConfig();

        port = Integer.parseInt(props.getProperty("solr.port", getHttpPort("8080")));
        String defaultHost;
        try {
            defaultHost = InetAddress.getLocalHost().getHostName();
        } catch (UnknownHostException e) {
            defaultHost = "localhost";
        }
        hostName = props.getProperty("solr.host", defaultHost);
        baseUrl = props.getProperty("solr.baseUrl", "/solr4");
        baseUrl = (baseUrl.startsWith("/") ? "" : "/") + baseUrl + "/" + core.getName() + "/";
    }

    synchronized public void initSkippingDescendantDocs() {
        if (isSkippingDocsInitialized) {
            return;
        }
        Properties p = core.getResourceLoader().getCoreProperties();
        skipDescendantDocsForSpecificTypes = Boolean
                .parseBoolean(p.getProperty("alfresco.metadata.skipDescendantDocsForSpecificTypes", "false"));
        if (skipDescendantDocsForSpecificTypes) {
            initSkippingDescendantDocs(p, typesForSkippingDescendantDocs, PROP_PREFIX_PARENT_TYPE, FIELD_TYPE,
                    new DefinitionExistChecker() {
                        @Override
                        public boolean isDefinitionExists(QName qName) {
                            return (null != dataModel.getDictionaryService(CMISStrictDictionaryService.DEFAULT)
                                    .getType(qName));
                        }
                    });
        }
        skipDescendantDocsForSpecificAspects = Boolean
                .parseBoolean(p.getProperty("alfresco.metadata.skipDescendantDocsForSpecificAspects", "false"));
        if (skipDescendantDocsForSpecificAspects) {
            initSkippingDescendantDocs(p, aspectsForSkippingDescendantDocs, PROP_PREFIX_PARENT_ASPECT, FIELD_ASPECT,
                    new DefinitionExistChecker() {
                        @Override
                        public boolean isDefinitionExists(QName qName) {
                            return (null != dataModel.getDictionaryService(CMISStrictDictionaryService.DEFAULT)
                                    .getAspect(qName));
                        }
                    });
        }
        isSkippingDocsInitialized = true;
    }

    private interface DefinitionExistChecker {
        boolean isDefinitionExists(QName qName);
    }

    private void initSkippingDescendantDocs(Properties p, Set<QName> dataForSkippingDescendantDocs,
            String propPrefixParent, String skipByField, DefinitionExistChecker dec) {
        int i = 0;
        for (String key = new StringBuilder(propPrefixParent).append(i).toString(); p
                .containsKey(key); key = new StringBuilder(propPrefixParent).append(++i).toString()) {
            String qNameInString = p.getProperty(key);
            if ((null != qNameInString) && !qNameInString.isEmpty()) {
                QName qName = QName.resolveToQName(dataModel.getNamespaceDAO(), qNameInString);
                if (qName == null && log.isWarnEnabled()) {
                    log.warn("QName was not found for " + qNameInString);
                }
                if (dec.isDefinitionExists(qName)) {
                    dataForSkippingDescendantDocs.add(qName);
                }
            }
        }

        if (null == skippingDocsQueryString) {
            skippingDocsQueryString = this.cloud.getQuery(skipByField, OR, dataForSkippingDescendantDocs);
        } else {
            skippingDocsQueryString += OR + this.cloud.getQuery(skipByField, OR, dataForSkippingDescendantDocs);
        }
    }

    @SuppressWarnings({ "rawtypes", "unchecked" })
    public void addFTSStatusCounts(NamedList<Object> report) {
        SolrQueryRequest request = null;
        try {
            request = getLocalSolrQueryRequest();
            ModifiableSolrParams params = new ModifiableSolrParams(request.getParams());
            params.set("q", "*:*").set("rows", 0).set("facet", true).set("facet.field", FIELD_FTSSTATUS);
            SolrQueryResponse response = cloud.getResponse(nativeRequestHandler, request, params);
            NamedList facetCounts = (NamedList) response.getValues().get("facet_counts");
            NamedList facetFields = (NamedList) facetCounts.get("facet_fields");
            NamedList<Integer> ftsStatusCounts = (NamedList) facetFields.get(FIELD_FTSSTATUS);
            long cleanCount = this.getSafeCount(ftsStatusCounts, FTSStatus.Clean.toString());
            report.add("Node count with FTSStatus Clean", cleanCount);

            long dirtyCount = this.getSafeCount(ftsStatusCounts, FTSStatus.Dirty.toString());
            report.add("Node count with FTSStatus Dirty", dirtyCount);

            long newCount = this.getSafeCount(ftsStatusCounts, FTSStatus.New.toString());
            report.add("Node count with FTSStatus New", newCount);
        } finally {
            if (request != null) {
                request.close();
            }
        }
    }

    public String getAlfrescoVersion() {
        return this.alfrescoVersion;
    }

    @Override
    public void afterInitModels() {
        this.dataModel.afterInitModels();
    }

    @Override
    public AclReport checkAclInIndex(Long aclid, AclReport aclReport) {
        String query = FIELD_ACLID + ":" + aclid + AND + FIELD_DOC_TYPE + ":" + DOC_TYPE_ACL;
        int count = this.getDocListSize(query);
        aclReport.setIndexedAclDocCount(new Long(count));

        // TODO Could add INACLTXID later, but would need acl change set id.
        return aclReport;
    }

    @Override
    public IndexHealthReport reportIndexTransactions(Long minTxId, IOpenBitSet txIdsInDb, long maxTxId)
            throws IOException {
        SolrQueryRequest request = null;
        try {
            request = getLocalSolrQueryRequest();
            NamedList<Integer> docTypeCounts = this.getFacets(request, "*:*", FIELD_DOC_TYPE, 0);

            // TX
            IndexHealthReport report = new IndexHealthReport(this);
            TransactionInfoReporter txReporter = new TransactionInfoReporter(report) {
                @Override
                void reportIdInIndexButNotInDb(long txid) {
                    report.setTxInIndexButNotInDb(txid);
                }

                @Override
                void reportIdInDbButNotInIndex(long id) {
                    report.setMissingTxFromIndex(id);
                }

                @Override
                void reportDuplicatedIdInIndex(long id) {
                    report.setDuplicatedTxInIndex(id);
                }

                @Override
                void reportUniqueIdsInIndex(long count) {
                    report.setUniqueTransactionDocsInIndex(count);
                }
            };
            reportTransactionInfo(txReporter, minTxId, maxTxId, txIdsInDb, request, FIELD_TXID);
            long transactionDocsInIndex = getSafeCount(docTypeCounts, DOC_TYPE_TX);
            report.setTransactionDocsInIndex(transactionDocsInIndex);
            report.setDbTransactionCount(txIdsInDb.cardinality());

            // NODE
            setDuplicates(report, request, DOC_TYPE_NODE, new SetDuplicatesCommand() {
                public void execute(IndexHealthReport indexHealthReport, long id) {
                    indexHealthReport.setDuplicatedLeafInIndex(id);
                }
            });
            long leafDocCountInIndex = getSafeCount(docTypeCounts, DOC_TYPE_NODE);
            report.setLeafDocCountInIndex(leafDocCountInIndex);

            // ERROR
            setDuplicates(report, request, DOC_TYPE_ERROR_NODE, new SetDuplicatesCommand() {
                public void execute(IndexHealthReport indexHealthReport, long id) {
                    indexHealthReport.setDuplicatedErrorInIndex(id);
                }
            });
            long errorCount = getSafeCount(docTypeCounts, DOC_TYPE_ERROR_NODE);
            report.setErrorDocCountInIndex(errorCount);

            // UNINDEXED
            setDuplicates(report, request, DOC_TYPE_UNINDEXED_NODE, new SetDuplicatesCommand() {
                public void execute(IndexHealthReport indexHealthReport, long id) {
                    indexHealthReport.setDuplicatedUnindexedInIndex(id);
                }
            });
            long unindexedDocCountInIndex = getSafeCount(docTypeCounts, DOC_TYPE_UNINDEXED_NODE);
            report.setUnindexedDocCountInIndex(unindexedDocCountInIndex);
            return report;
        } finally {
            if (request != null) {
                request.close();
            }
        }
    }

    @Override
    public IndexHealthReport reportAclTransactionsInIndex(Long minAclTxId, IOpenBitSet aclTxIdsInDb,
            long maxAclTxId) {
        SolrQueryRequest request = null;
        try {
            request = getLocalSolrQueryRequest();
            NamedList<Integer> docTypeCounts = this.getFacets(request, "*:*", FIELD_DOC_TYPE, 0);
            IndexHealthReport report = new IndexHealthReport(this);
            TransactionInfoReporter aclTxReporter = new TransactionInfoReporter(report) {
                @Override
                void reportIdInIndexButNotInDb(long txid) {
                    report.setAclTxInIndexButNotInDb(txid);
                }

                @Override
                void reportIdInDbButNotInIndex(long id) {
                    report.setMissingAclTxFromIndex(id);
                }

                @Override
                void reportDuplicatedIdInIndex(long id) {
                    report.setDuplicatedAclTxInIndex(id);
                }

                @Override
                void reportUniqueIdsInIndex(long count) {
                    report.setUniqueAclTransactionDocsInIndex(count);
                }
            };
            reportTransactionInfo(aclTxReporter, minAclTxId, maxAclTxId, aclTxIdsInDb, request, FIELD_ACLTXID);
            long aclTransactionDocsInIndex = getSafeCount(docTypeCounts, DOC_TYPE_ACL_TX);
            report.setAclTransactionDocsInIndex(aclTransactionDocsInIndex);
            report.setDbAclTransactionCount(aclTxIdsInDb.cardinality());
            return report;
        } finally {
            if (request != null) {
                request.close();
            }
        }
    }

    private void reportTransactionInfo(TransactionInfoReporter reporter, Long minId, long maxId,
            IOpenBitSet idsInDb, SolrQueryRequest request, String field) {
        if (minId != null) {
            IOpenBitSet idsInIndex = this.getOpenBitSetInstance();
            long batchStartId = minId;
            long batchEndId = Math.min(batchStartId + BATCH_FACET_TXS, maxId);

            // Continues as long as the batch does not pass the maximum
            while (batchStartId <= maxId) {
                long iterationStart = batchStartId;
                NamedList<Integer> idCounts = this.getFacets(request,
                        field + ":[" + batchStartId + " TO " + batchEndId + "]", field, 1); // Min count of 1 ensures that the id returned is in the index
                for (Map.Entry<String, Integer> idCount : idCounts) {
                    long idInIndex = Long.valueOf(idCount.getKey());

                    // Only looks at facet values that fit the query
                    if (batchStartId <= idInIndex && idInIndex <= batchEndId) {
                        idsInIndex.set(idInIndex);

                        // The sequence of ids in the index could look like this: 1, 2, 5, 7... 
                        for (long id = iterationStart; id <= idInIndex; id++) {
                            if (id == idInIndex) {
                                iterationStart = idInIndex + 1;

                                if (!idsInDb.get(id)) {
                                    reporter.reportIdInIndexButNotInDb(id);
                                }
                            } else if (idsInDb.get(id)) {
                                reporter.reportIdInDbButNotInIndex(id);
                            }
                        }

                        if (idCount.getValue().intValue() > 1) {
                            reporter.reportDuplicatedIdInIndex(idInIndex);
                        }
                    } else {
                        break;
                    }
                }

                batchStartId = batchEndId + 1;
                batchEndId = Math.min(batchStartId + BATCH_FACET_TXS, maxId);
            }

            reporter.reportUniqueIdsInIndex(idsInIndex.cardinality());
        }
    }

    abstract class TransactionInfoReporter {
        protected IndexHealthReport report;

        TransactionInfoReporter(IndexHealthReport report) {
            this.report = report;
        }

        abstract void reportIdInIndexButNotInDb(long id);

        abstract void reportIdInDbButNotInIndex(long id);

        abstract void reportDuplicatedIdInIndex(long id);

        abstract void reportUniqueIdsInIndex(long count);
    }

    private void setDuplicates(IndexHealthReport report, SolrQueryRequest request, String docType,
            SetDuplicatesCommand cmd) {
        // A mincount of 2 checks for duplicates in solr
        NamedList<Integer> dbIdCounts = getFacets(request, FIELD_DOC_TYPE + ":" + docType, FIELD_DBID, 2);
        for (Map.Entry<String, Integer> dbId : dbIdCounts) {
            Long duplicatedDbId = Long.parseLong(dbId.getKey());
            cmd.execute(report, duplicatedDbId);
        }
    }

    @SuppressWarnings({ "unchecked", "rawtypes" })
    private NamedList<Integer> getFacets(SolrQueryRequest request, String query, String field, int minCount) {
        ModifiableSolrParams params = new ModifiableSolrParams(request.getParams());
        params.set("q", query).set("rows", 0).set("facet", true).set("facet.field", field).set("facet.mincount",
                minCount);
        SolrQueryResponse response = cloud.getResponse(nativeRequestHandler, request, params);
        NamedList facetCounts = (NamedList) response.getValues().get("facet_counts");
        NamedList facetFields = (NamedList) facetCounts.get("facet_fields");
        NamedList<Integer> counts = (NamedList) facetFields.get(field);
        return counts;
    }

    interface SetDuplicatesCommand {
        void execute(IndexHealthReport indexHealthReport, long id);
    }

    @Override
    public List<TenantAclIdDbId> getDocsWithUncleanContent(int start, int rows) throws IOException {
        SolrQueryRequest request = null;
        try {
            request = getLocalSolrQueryRequest();
            ModifiableSolrParams params = new ModifiableSolrParams(request.getParams());
            String query = FIELD_FTSSTATUS + ":" + FTSStatus.Dirty + " OR " + FIELD_FTSSTATUS + ":" + FTSStatus.New;
            params.set("q", query).set("fl", FIELD_SOLR4_ID).set("rows", rows).set("start", start).set("sort",
                    "_docid_ asc");
            // no scoring !!
            List<TenantAclIdDbId> docIds = new ArrayList<>();
            SolrDocumentList docList = cloud.getSolrDocumentList(nativeRequestHandler, request, params);
            if (docList != null) {
                for (SolrDocument doc : docList) {
                    String id = getFieldValueString(doc, FIELD_SOLR4_ID);
                    TenantAclIdDbId tenantAndDbId = AlfrescoSolrDataModel.decodeNodeDocumentId(id);
                    docIds.add(tenantAndDbId);
                }
            }

            return docIds;
        } finally {
            if (request != null) {
                request.close();
            }
        }
    }

    private String getFieldValueString(SolrDocument doc, String fieldName) {
        IndexableField field = (IndexableField) doc.getFieldValue(fieldName);
        String value = null;
        if (field != null) {
            value = field.stringValue();
        }
        return value;
    }

    private long getFieldValueLong(SolrDocument doc, String fieldName) {
        return Long.parseLong(getFieldValueString(doc, fieldName));
    }

    @Override
    public void addCommonNodeReportInfo(NodeReport nodeReport) {
        long dbId = nodeReport.getDbid();
        String query = FIELD_DBID + ":" + dbId + AND + FIELD_DOC_TYPE + ":" + DOC_TYPE_NODE;
        int count = this.getDocListSize(query);
        nodeReport.setIndexedNodeDocCount(new Long(count));
    }

    @Override
    public void commit() throws IOException {
        // avoid multiple commits and warming searchers
        commitAndRollbackLock.writeLock().lock();
        try {
            canUpdate();
            SolrQueryRequest request = null;
            UpdateRequestProcessor processor = null;
            try {
                request = getLocalSolrQueryRequest();
                processor = this.core.getUpdateProcessingChain(null).createProcessor(request,
                        new SolrQueryResponse());
                processor.processCommit(new CommitUpdateCommand(request, false));
            } finally {
                if (processor != null) {
                    processor.finish();
                }
                if (request != null) {
                    request.close();
                }
            }
        } finally {
            commitAndRollbackLock.writeLock().unlock();
        }
    }

    private void deleteById(String field, Long id) throws IOException {
        String query = field + ":" + id;
        deleteByQuery(query);
    }

    @Override
    public void deleteByAclChangeSetId(Long aclChangeSetId) throws IOException {
        deleteById(FIELD_INACLTXID, aclChangeSetId);
    }

    @Override
    public void deleteByAclId(Long aclId) throws IOException {
        isIdIndexCache.clear();
        deleteById(FIELD_ACLID, aclId);
    }

    @Override
    public void deleteByNodeId(Long nodeId) throws IOException {
        deleteById(FIELD_DBID, nodeId);
    }

    @Override
    public void deleteByTransactionId(Long transactionId) throws IOException {
        isIdIndexCache.clear();
        deleteById(FIELD_INTXID, transactionId);
    }

    private void deleteByQuery(String query) throws IOException {
        SolrQueryRequest request = null;
        UpdateRequestProcessor processor = null;
        try {
            request = getLocalSolrQueryRequest();
            processor = this.core.getUpdateProcessingChain(null).createProcessor(request, new SolrQueryResponse());
            DeleteUpdateCommand delDocCmd = new DeleteUpdateCommand(request);
            delDocCmd.setQuery(query);
            processor.processDelete(delDocCmd);
        } finally {
            if (processor != null) {
                processor.finish();
            }
            if (request != null) {
                request.close();
            }
        }
    }

    @Override
    public List<AlfrescoModel> getAlfrescoModels() {
        return this.dataModel.getAlfrescoModels();
    }

    private long getSafeCount(NamedList<Integer> counts, String countType) {
        Integer count = counts.get(countType);
        return (count == null ? 0 : count.longValue());
    }

    @SuppressWarnings({ "unchecked", "rawtypes" })
    @Override
    public Iterable<Entry<String, Object>> getCoreStats() throws IOException {
        // This is still local, not totally cloud-friendly
        // TODO Make this cloud-friendly by aggregating the stats across the cloud

        SolrQueryRequest request = null;
        NamedList<Object> coreSummary = new SimpleOrderedMap<Object>();
        RefCounted<SolrIndexSearcher> refCounted = null;
        try {
            request = getLocalSolrQueryRequest();
            NamedList docTypeCounts = this.getFacets(request, "*:*", FIELD_DOC_TYPE, 0);
            long aclCount = getSafeCount(docTypeCounts, DOC_TYPE_ACL);
            coreSummary.add("Alfresco Acls in Index", aclCount);
            long nodeCount = getSafeCount(docTypeCounts, DOC_TYPE_NODE);
            coreSummary.add("Alfresco Nodes in Index", nodeCount);
            long txCount = getSafeCount(docTypeCounts, DOC_TYPE_TX);
            coreSummary.add("Alfresco Transactions in Index", txCount);
            long aclTxCount = getSafeCount(docTypeCounts, DOC_TYPE_ACL_TX);
            coreSummary.add("Alfresco Acl Transactions in Index", aclTxCount);
            long stateCount = getSafeCount(docTypeCounts, DOC_TYPE_STATE);
            coreSummary.add("Alfresco States in Index", stateCount);
            long unindexedNodeCount = getSafeCount(docTypeCounts, DOC_TYPE_UNINDEXED_NODE);
            coreSummary.add("Alfresco Unindexed Nodes", unindexedNodeCount);
            long errorNodeCount = getSafeCount(docTypeCounts, DOC_TYPE_ERROR_NODE);
            coreSummary.add("Alfresco Error Nodes in Index", errorNodeCount);

            refCounted = core.getSearcher(false, true, null);
            SolrIndexSearcher solrIndexSearcher = refCounted.get();
            coreSummary.add("Searcher", solrIndexSearcher.getStatistics());
            Map<String, SolrInfoMBean> infoRegistry = core.getInfoRegistry();
            for (String key : infoRegistry.keySet()) {
                SolrInfoMBean infoMBean = infoRegistry.get(key);
                if (key.equals("/alfresco")) {
                    // TODO Do we really need to fixStats in solr4?
                    coreSummary.add("/alfresco", fixStats(infoMBean.getStatistics()));
                }
                if (key.equals("/afts")) {
                    coreSummary.add("/afts", fixStats(infoMBean.getStatistics()));
                }
                if (key.equals("/cmis")) {
                    coreSummary.add("/cmis", fixStats(infoMBean.getStatistics()));
                }
                if (key.equals("filterCache")) {
                    coreSummary.add("/filterCache", infoMBean.getStatistics());
                }
                if (key.equals("queryResultCache")) {
                    coreSummary.add("/queryResultCache", infoMBean.getStatistics());
                }
                if (key.equals("alfrescoAuthorityCache")) {
                    coreSummary.add("/alfrescoAuthorityCache", infoMBean.getStatistics());
                }
                if (key.equals("alfrescoPathCache")) {
                    coreSummary.add("/alfrescoPathCache", infoMBean.getStatistics());
                }
            }

            // Adds detailed stats for each registered searcher
            int searcherIndex = 0;
            List<SolrIndexSearcher> searchers = getRegisteredSearchers();
            for (SolrIndexSearcher searcher : searchers) {
                NamedList<Object> details = new SimpleOrderedMap<Object>();
                details.add("Searcher", searcher.getStatistics());
                coreSummary.add("Searcher-" + searcherIndex, details);
                searcherIndex++;
            }

            coreSummary.add("Number of Searchers", searchers.size());
            // This is zero for Solr4, whereas we had some local caches before
            coreSummary.add("Total Searcher Cache (GB)", 0);

            IndexDeletionPolicyWrapper delPolicy = core.getDeletionPolicy();
            IndexCommit indexCommit = delPolicy.getLatestCommit();
            // race?
            if (indexCommit == null) {
                indexCommit = solrIndexSearcher.getIndexReader().getIndexCommit();
            }
            if (indexCommit != null) {
                // Tells Solr to stop deleting things for 20 seconds so we can get a snapshot of all the files on the index
                delPolicy.setReserveDuration(solrIndexSearcher.getIndexReader().getVersion(), 20000);
                Long fileSize = 0L;

                File dir = new File(solrIndexSearcher.getPath());
                for (String name : indexCommit.getFileNames()) {
                    File file = new File(dir, name);
                    if (file.exists()) {
                        fileSize += file.length();
                    }
                }

                DecimalFormat df = new DecimalFormat("###,###.######");
                coreSummary.add("On disk (GB)", df.format(fileSize / 1024.0f / 1024.0f / 1024.0f));
                coreSummary.add("Per node B", nodeCount > 0 ? fileSize / nodeCount : 0);
            }
        } finally {
            if (request != null) {
                request.close();
            }

            if (refCounted != null) {
                refCounted.decref();
            }
        }

        return coreSummary;
    }

    private NamedList<Object> fixStats(NamedList<Object> namedList) {
        int sz = namedList.size();

        for (int i = 0; i < sz; i++) {
            Object value = namedList.getVal(i);
            if (value instanceof Number) {
                Number number = (Number) value;
                if (Float.isInfinite(number.floatValue()) || Float.isNaN(number.floatValue())
                        || Double.isInfinite(number.doubleValue()) || Double.isNaN(number.doubleValue())) {
                    namedList.setVal(i, null);
                }
            }
        }
        return namedList;
    }

    private List<SolrIndexSearcher> getRegisteredSearchers() {
        List<SolrIndexSearcher> searchers = new ArrayList<SolrIndexSearcher>();
        for (Entry<String, SolrInfoMBean> entry : core.getInfoRegistry().entrySet()) {
            if (entry.getValue() != null) {
                if (entry.getValue().getName().equals(SolrIndexSearcher.class.getName())) {
                    if (!entry.getKey().equals("searcher")) {
                        searchers.add((SolrIndexSearcher) entry.getValue());
                    }
                }
            }
        }
        return searchers;
    }

    @Override
    public DictionaryComponent getDictionaryService(String alternativeDictionary) {
        return this.dataModel.getDictionaryService(alternativeDictionary);
    }

    @Override
    public int getTxDocsSize(String targetTxId, String targetTxCommitTime) throws IOException {
        return getDocListSize(FIELD_TXID + ":" + targetTxId + AND + FIELD_TXCOMMITTIME + ":" + targetTxCommitTime);
    }

    @Override
    public int getAclTxDocsSize(String aclTxId, String aclTxCommitTime) throws IOException {
        return getDocListSize(FIELD_ACLTXID + ":" + aclTxId + AND + FIELD_ACLTXCOMMITTIME + ":" + aclTxCommitTime);
    }

    private int getDocListSize(String query) {
        SolrQueryRequest request = null;
        try {
            request = this.getLocalSolrQueryRequest();
            ModifiableSolrParams params = new ModifiableSolrParams(request.getParams());
            params.set("q", query);
            // Sets the rows to zero, because we actually just want the count
            params.set("rows", 0);
            ResultContext resultContext = cloud.getResultContext(nativeRequestHandler, request, params);
            int matches = resultContext.docs.matches();
            return matches;
        } finally {
            if (request != null) {
                request.close();
            }
        }
    }

    @Override
    public Set<Long> getErrorDocIds() throws IOException {
        HashSet<Long> errorDocIds = new HashSet<Long>();

        SolrQueryRequest request = null;
        try {
            request = this.getLocalSolrQueryRequest();
            String query = FIELD_DOC_TYPE + ":" + DOC_TYPE_ERROR_NODE;
            ModifiableSolrParams params = new ModifiableSolrParams(request.getParams());
            // Sets MAX_VALUE to get all the rows
            params.set("q", query).set("fl", FIELD_SOLR4_ID).set("rows", Integer.MAX_VALUE);
            SolrDocumentList docs = cloud.getSolrDocumentList(nativeRequestHandler, request, params);
            for (SolrDocument doc : docs) {
                String id = getFieldValueString(doc, FIELD_SOLR4_ID);
                if (id.startsWith(PREFIX_ERROR)) {
                    String nodeId = id.substring(PREFIX_ERROR.length());
                    errorDocIds.add(Long.valueOf(nodeId));
                }
            }
        } finally {
            if (request != null) {
                request.close();
            }
        }
        return errorDocIds;
    }

    @Override
    public long getHoleRetention() {
        return this.holeRetention;
    }

    @Override
    public M2Model getM2Model(QName modelQName) {
        return this.dataModel.getM2Model(modelQName);
    }

    @Override
    public Map<String, Set<String>> getModelErrors() {
        return AlfrescoSolrDataModel.getInstance().getModelErrors();
    }

    @Override
    public NamespaceDAO getNamespaceDAO() {
        return this.dataModel.getNamespaceDAO();
    }

    @Override
    public IOpenBitSet getOpenBitSetInstance() {
        return new SolrOpenBitSetAdapter();
    }

    @Override
    public int getRegisteredSearcherCount() {
        int count = getRegisteredSearchers().size();
        log.info(".... registered Searchers for " + core.getName() + " = " + count);
        return count;
    }

    @Override
    public <T> ISimpleOrderedMap<T> getSimpleOrderedMapInstance() {
        return new SolrSimpleOrderedMap<T>();
    }

    @Override
    public TrackerStats getTrackerStats() {
        return this.trackerStats;
    }

    @Override
    public TrackerState getTrackerInitialState() {
        SolrQueryRequest request = null;
        try {
            request = getLocalSolrQueryRequest();

            TrackerState state = new TrackerState();
            SolrRequestHandler handler = core.getRequestHandler(REQUEST_HANDLER_GET);
            SolrQueryResponse rsp = new SolrQueryResponse();

            ModifiableSolrParams newParams = new ModifiableSolrParams(request.getParams());
            newParams.set("ids", "TRACKER!STATE!ACLTX,TRACKER!STATE!TX");
            request.setParams(newParams);

            handler.handleRequest(request, rsp);

            @SuppressWarnings("rawtypes")
            NamedList values = rsp.getValues();
            SolrDocumentList response = (SolrDocumentList) values.get(RESPONSE_DEFAULT_IDS);

            // We can find either or both docs here.
            for (int i = 0; i < response.getNumFound(); i++) {
                SolrDocument current = response.get(i);
                // ACLTX
                if (current.getFieldValue(FIELD_S_ACLTXCOMMITTIME) != null) {
                    if (state.getLastIndexedChangeSetCommitTime() == 0) {
                        state.setLastIndexedChangeSetCommitTime(
                                getFieldValueLong(current, FIELD_S_ACLTXCOMMITTIME));
                    }

                    if (state.getLastIndexedChangeSetId() == 0) {
                        state.setLastIndexedChangeSetId(getFieldValueLong(current, FIELD_S_ACLTXID));
                    }
                }

                // TX
                if (current.getFieldValue(FIELD_S_TXCOMMITTIME) != null) {
                    if (state.getLastIndexedTxCommitTime() == 0) {
                        state.setLastIndexedTxCommitTime(getFieldValueLong(current, FIELD_S_TXCOMMITTIME));
                    }
                    if (state.getLastIndexedTxId() == 0) {
                        state.setLastIndexedTxId(getFieldValueLong(current, FIELD_S_TXID));
                    }
                }

            }

            long startTime = System.currentTimeMillis();
            state.setTimeToStopIndexing(startTime - lag);
            state.setTimeBeforeWhichThereCanBeNoHoles(startTime - holeRetention);

            long timeBeforeWhichThereCanBeNoTxHolesInIndex = state.getLastIndexedTxCommitTime() - holeRetention;
            state.setLastGoodTxCommitTimeInIndex(
                    timeBeforeWhichThereCanBeNoTxHolesInIndex > 0 ? timeBeforeWhichThereCanBeNoTxHolesInIndex : 0);

            long timeBeforeWhichThereCanBeNoChangeSetHolesInIndex = state.getLastIndexedChangeSetCommitTime()
                    - holeRetention;
            state.setLastGoodChangeSetCommitTimeInIndex(timeBeforeWhichThereCanBeNoChangeSetHolesInIndex > 0
                    ? timeBeforeWhichThereCanBeNoChangeSetHolesInIndex
                    : 0);

            return state;

        } finally {
            if (request != null) {
                request.close();
            }
        }
    }

    @Override
    public long indexAcl(List<AclReaders> aclReaderList, boolean overwrite) throws IOException {
        long start = System.nanoTime();

        SolrQueryRequest request = null;
        UpdateRequestProcessor processor = null;
        try {
            request = getLocalSolrQueryRequest();
            processor = this.core.getUpdateProcessingChain(null).createProcessor(request, new SolrQueryResponse());

            for (AclReaders aclReaders : aclReaderList) {
                AddUpdateCommand cmd = new AddUpdateCommand(request);
                cmd.overwrite = overwrite;
                SolrInputDocument input = new SolrInputDocument();
                String id = AlfrescoSolrDataModel.getAclDocumentId(aclReaders.getTenantDomain(),
                        aclReaders.getId());
                input.addField(FIELD_SOLR4_ID, id);
                input.addField(FIELD_VERSION, "0");
                input.addField(FIELD_ACLID, aclReaders.getId());
                input.addField(FIELD_INACLTXID, aclReaders.getAclChangeSetId());
                String tenant = aclReaders.getTenantDomain();
                for (String reader : aclReaders.getReaders()) {
                    reader = addTenantToAuthority(reader, tenant);
                    input.addField(FIELD_READER, reader);
                }
                for (String denied : aclReaders.getDenied()) {
                    denied = addTenantToAuthority(denied, tenant);
                    input.addField(FIELD_DENIED, denied);
                }
                input.addField(FIELD_DOC_TYPE, DOC_TYPE_ACL);
                cmd.solrDoc = input;
                processor.processAdd(cmd);
            }
        } finally {
            if (processor != null) {
                processor.finish();
            }
            if (request != null) {
                request.close();
            }
        }

        long end = System.nanoTime();
        return (end - start);
    }

    /**
     * Adds tenant information to an authority, <strong>if required</strong>, such that jbloggs for tenant example.com
     * would become jbloggs@example.com
     * 
     * @param authority   The authority to mutate, if it matches certain types.
     * @param tenant      The tenant that will be added to the authority.
     * @return The new authority information
     */
    private String addTenantToAuthority(String authority, String tenant) {
        switch (AuthorityType.getAuthorityType(authority)) {
        case GROUP:
        case EVERYONE:
        case GUEST:
            if (tenant.length() == 0) {
                // Default tenant matches 4.0
            } else {
                authority = authority + "@" + tenant;
            }
            break;
        default:
            break;
        }
        return authority;
    }

    private LocalSolrQueryRequest getLocalSolrQueryRequest() {
        LocalSolrQueryRequest req = new LocalSolrQueryRequest(core, new NamedList<>());
        return req;
    }

    @Override
    public void indexAclTransaction(AclChangeSet changeSet, boolean overwrite) throws IOException {
        canUpdate();
        SolrQueryRequest request = null;
        UpdateRequestProcessor processor = null;
        try {
            request = getLocalSolrQueryRequest();
            processor = this.core.getUpdateProcessingChain(null).createProcessor(request, new SolrQueryResponse());
            AddUpdateCommand cmd = new AddUpdateCommand(request);
            cmd.overwrite = overwrite;
            SolrInputDocument input = new SolrInputDocument();
            input.addField(FIELD_SOLR4_ID, AlfrescoSolrDataModel.getAclChangeSetDocumentId(changeSet.getId()));
            input.addField(FIELD_VERSION, "0");
            input.addField(FIELD_ACLTXID, changeSet.getId());
            input.addField(FIELD_INACLTXID, changeSet.getId());
            input.addField(FIELD_ACLTXCOMMITTIME, changeSet.getCommitTimeMs());
            input.addField(FIELD_DOC_TYPE, DOC_TYPE_ACL_TX);
            cmd.solrDoc = input;
            processor.processAdd(cmd);
            putAclTransactionState(processor, request, changeSet);
        } finally {
            if (processor != null) {
                processor.finish();
            }
            if (request != null) {
                request.close();
            }
        }
    }

    @Override
    public AclChangeSet getMaxAclChangeSetIdAndCommitTimeInIndex() {
        SolrQueryRequest request = null;
        try {
            request = getLocalSolrQueryRequest();
            SolrDocument aclState = getState(request, "TRACKER!STATE!ACLTX");
            AclChangeSet maxAclChangeSet = null;
            if (aclState != null) {
                long id = this.getFieldValueLong(aclState, FIELD_S_ACLTXID);
                long commitTime = this.getFieldValueLong(aclState, FIELD_S_ACLTXCOMMITTIME);
                int aclCount = -1; // Irrelevant for this method
                maxAclChangeSet = new AclChangeSet(id, commitTime, aclCount);
            } else {
                maxAclChangeSet = new AclChangeSet(0, 0, -1);
            }
            return maxAclChangeSet;
        } finally {
            if (request != null) {
                request.close();
            }
        }
    }

    private SolrDocument getState(SolrQueryRequest request, String id) {
        ModifiableSolrParams newParams = new ModifiableSolrParams(request.getParams());
        newParams.set("id", id);
        request.setParams(newParams);
        SolrRequestHandler handler = core.getRequestHandler(REQUEST_HANDLER_GET);
        SolrQueryResponse rsp = new SolrQueryResponse();
        handler.handleRequest(request, rsp);

        @SuppressWarnings("rawtypes")
        NamedList values = rsp.getValues();
        SolrDocument state = (SolrDocument) values.get(RESPONSE_DEFAULT_ID);
        return state;
    }

    public void putAclTransactionState(UpdateRequestProcessor processor, SolrQueryRequest request,
            AclChangeSet changeSet) throws IOException {
        String version;
        SolrDocument aclState = getState(request, "TRACKER!STATE!ACLTX");
        if (aclState != null) {
            long aclTxCommitTime = this.getFieldValueLong(aclState, FIELD_S_ACLTXCOMMITTIME);
            long aclTxId = this.getFieldValueLong(aclState, FIELD_S_ACLTXID);
            // Acl change sets are ordered by commit time and tie-broken by id
            if (changeSet.getCommitTimeMs() > aclTxCommitTime
                    || changeSet.getCommitTimeMs() == aclTxCommitTime && changeSet.getId() > aclTxId) {
                // Uses optimistic concurrency 
                version = this.getFieldValueString(aclState, FIELD_VERSION);
            } else {
                version = null; // Should not update in this case
            }
        } else {
            version = "0";
        }

        if (version != null) {
            AddUpdateCommand cmd = new AddUpdateCommand(request);
            cmd.overwrite = true;
            SolrInputDocument input = new SolrInputDocument();
            input.addField(FIELD_SOLR4_ID, "TRACKER!STATE!ACLTX");
            input.addField(FIELD_VERSION, version);
            input.addField(FIELD_S_ACLTXID, changeSet.getId());
            input.addField(FIELD_S_INACLTXID, changeSet.getId());
            input.addField(FIELD_S_ACLTXCOMMITTIME, changeSet.getCommitTimeMs());
            input.addField(FIELD_DOC_TYPE, DOC_TYPE_STATE);
            cmd.solrDoc = input;
            processor.processAdd(cmd);
        }
    }

    @Override
    public void indexNode(Node node, boolean overwrite) throws IOException, AuthenticationException, JSONException {
        SolrQueryRequest request = null;
        UpdateRequestProcessor processor = null;
        try {
            request = getLocalSolrQueryRequest();
            processor = this.core.getUpdateProcessingChain(null).createProcessor(request, new SolrQueryResponse());
            long start = System.nanoTime();

            if ((node.getStatus() == SolrApiNodeStatus.DELETED)
                    || (node.getStatus() == SolrApiNodeStatus.NON_SHARD_DELETED)
                    || (node.getStatus() == SolrApiNodeStatus.NON_SHARD_UPDATED)
                    || (node.getStatus() == SolrApiNodeStatus.UNKNOWN)) {
                // fix up any secondary paths
                NodeMetaDataParameters nmdp = new NodeMetaDataParameters();
                nmdp.setFromNodeId(node.getId());
                nmdp.setToNodeId(node.getId());
                List<NodeMetaData> nodeMetaDatas;
                if ((node.getStatus() == SolrApiNodeStatus.DELETED)
                        || (node.getStatus() == SolrApiNodeStatus.NON_SHARD_DELETED)
                        || (node.getStatus() == SolrApiNodeStatus.NON_SHARD_UPDATED)) {
                    // Fake the empty node metadata for this parent deleted node
                    NodeMetaData nodeMetaData = createDeletedNodeMetaData(node);
                    nodeMetaDatas = Collections.singletonList(nodeMetaData);
                } else {
                    nodeMetaDatas = repositoryClient.getNodesMetaData(nmdp, Integer.MAX_VALUE);
                }

                NodeMetaData nodeMetaData = null;
                if (!nodeMetaDatas.isEmpty()) {
                    nodeMetaData = nodeMetaDatas.get(0);
                    if (!(nodeMetaData.getTxnId() > node.getTxnId())) {
                        if (node.getStatus() == SolrApiNodeStatus.DELETED) {
                            this.removeDocFromContentStore(nodeMetaData);
                        }
                    }
                    // else, the node has moved on to a later transaction, and it will be indexed later
                }

                if (log.isDebugEnabled()) {
                    log.debug(".. deleting");
                }
                deleteNode(processor, request, node);
            }

            if ((node.getStatus() == SolrApiNodeStatus.UPDATED) || (node.getStatus() == SolrApiNodeStatus.UNKNOWN)
                    || (node.getStatus() == SolrApiNodeStatus.NON_SHARD_UPDATED)) {
                log.info(".. updating");
                NodeMetaDataParameters nmdp = new NodeMetaDataParameters();
                nmdp.setFromNodeId(node.getId());
                nmdp.setToNodeId(node.getId());

                List<NodeMetaData> nodeMetaDatas = repositoryClient.getNodesMetaData(nmdp, Integer.MAX_VALUE);

                AddUpdateCommand addDocCmd = new AddUpdateCommand(request);
                addDocCmd.overwrite = overwrite;

                if (!nodeMetaDatas.isEmpty()) {
                    NodeMetaData nodeMetaData = nodeMetaDatas.get(0);
                    if (!(nodeMetaData.getTxnId() > node.getTxnId())) {
                        if (mayHaveChildren(nodeMetaData)) {
                            cascadeUpdate(nodeMetaData, overwrite, request, processor);
                        }
                    }
                    // else, the node has moved on to a later transaction, and it will be indexed later

                    if ((node.getStatus() == SolrApiNodeStatus.UPDATED)
                            || (node.getStatus() == SolrApiNodeStatus.UNKNOWN)) {
                        // check index control
                        Map<QName, PropertyValue> properties = nodeMetaData.getProperties();
                        StringPropertyValue pValue = (StringPropertyValue) properties
                                .get(ContentModel.PROP_IS_INDEXED);
                        if (pValue != null) {
                            Boolean isIndexed = Boolean.valueOf(pValue.getValue());
                            if (!isIndexed.booleanValue()) {
                                if (log.isDebugEnabled()) {
                                    log.debug(".. clearing unindexed");
                                }
                                deleteNode(processor, request, node);

                                SolrInputDocument doc = createNewDoc(nodeMetaData, DOC_TYPE_UNINDEXED_NODE);
                                storeDocOnSolrContentStore(nodeMetaData, doc);
                                addDocCmd.solrDoc = doc;
                                if (recordUnindexedNodes) {
                                    processor.processAdd(addDocCmd);
                                }

                                long end = System.nanoTime();
                                this.trackerStats.addNodeTime(end - start);
                                return;
                            }
                        }
                        // Make sure any unindexed or error doc is removed.
                        if (log.isDebugEnabled()) {
                            log.debug(".. deleting node " + node.getId());
                        }
                        deleteNode(processor, request, node);

                        SolrInputDocument doc = createNewDoc(nodeMetaData, DOC_TYPE_NODE);
                        addToNewDocAndCache(nodeMetaData, doc);
                        addDocCmd.solrDoc = doc;
                        processor.processAdd(addDocCmd);
                    }
                } // Ends checking for a nodeMetaData
            } // Ends checking for updated or unknown node status
            long end = System.nanoTime();
            this.trackerStats.addNodeTime(end - start);
        } catch (Exception e) {
            log.warn("Node index failed and skipped for " + node.getId() + " in Tx " + node.getTxnId(), e);

            if (processor == null) {
                if (request == null) {
                    request = getLocalSolrQueryRequest();
                }
                processor = this.core.getUpdateProcessingChain(null).createProcessor(request,
                        new SolrQueryResponse());
            }

            // TODO: retry failed

            if (log.isDebugEnabled()) {
                log.debug(".. deleting on exception");
            }
            deleteNode(processor, request, node);

            AddUpdateCommand addDocCmd = new AddUpdateCommand(request);
            addDocCmd.overwrite = overwrite;

            SolrInputDocument doc = new SolrInputDocument();
            doc.addField(FIELD_SOLR4_ID, PREFIX_ERROR + node.getId());
            doc.addField(FIELD_VERSION, "0");
            doc.addField(FIELD_DBID, node.getId());
            doc.addField(FIELD_INTXID, node.getTxnId());
            doc.addField(FIELD_EXCEPTION_MESSAGE, e.getMessage());
            doc.addField(FIELD_DOC_TYPE, DOC_TYPE_ERROR_NODE);

            StringWriter stringWriter = new StringWriter(4096);
            PrintWriter printWriter = new PrintWriter(stringWriter, true);
            try {
                e.printStackTrace(printWriter);
                String stack = stringWriter.toString();
                doc.addField(FIELD_EXCEPTION_STACK, stack.length() < 32766 ? stack : stack.substring(0, 32765));
            } finally {
                printWriter.close();
            }

            addDocCmd.solrDoc = doc;
            processor.processAdd(addDocCmd);
        } finally {
            if (processor != null) {
                processor.finish();
            }
            if (request != null) {
                request.close();
            }
        }
    }

    private void updateDescendantDocs(NodeMetaData parentNodeMetaData, boolean overwrite, SolrQueryRequest request,
            UpdateRequestProcessor processor, LinkedHashSet<Long> stack)
            throws AuthenticationException, IOException, JSONException {
        if (stack.contains(parentNodeMetaData.getId())) {
            log.warn("Found descendant data loop for node id " + parentNodeMetaData.getId());
            log.warn("... stack to node =" + stack);
            return;
        } else {
            try {
                stack.add(parentNodeMetaData.getId());
                doUpdateDescendantDocs(parentNodeMetaData, overwrite, request, processor, stack);
            } finally {
                stack.remove(parentNodeMetaData.getId());
            }
        }
    }

    private boolean shouldBeIgnoredByAnyAspect(Set<QName> aspects) {
        if (null == aspects) {
            return false;
        }
        for (QName aspectForSkipping : aspectsForSkippingDescendantDocs) {
            if (aspects.contains(aspectForSkipping)) {
                return true;
            }
        }
        return false;
    }

    private void doUpdateDescendantDocs(NodeMetaData parentNodeMetaData, boolean overwrite,
            SolrQueryRequest request, UpdateRequestProcessor processor, LinkedHashSet<Long> stack)
            throws AuthenticationException, IOException, JSONException {
        if ((skipDescendantDocsForSpecificTypes
                && typesForSkippingDescendantDocs.contains(parentNodeMetaData.getType()))
                || (skipDescendantDocsForSpecificAspects
                        && shouldBeIgnoredByAnyAspect(parentNodeMetaData.getAspects()))) {
            return;
        }

        HashSet<Long> childIds = new HashSet<Long>();

        if (parentNodeMetaData.getChildIds() != null) {
            childIds.addAll(parentNodeMetaData.getChildIds());
        }

        String query = FIELD_PARENT + ":\"" + parentNodeMetaData.getNodeRef() + "\"";
        ModifiableSolrParams params = new ModifiableSolrParams(request.getParams());
        params.set("q", query).set("fl", FIELD_SOLR4_ID);
        if (skippingDocsQueryString != null && !skippingDocsQueryString.isEmpty()) {
            params.set("fq", "NOT ( " + skippingDocsQueryString + " )");
        }
        SolrDocumentList docs = cloud.getSolrDocumentList(nativeRequestHandler, request, params);
        for (SolrDocument doc : docs) {
            String id = getFieldValueString(doc, FIELD_SOLR4_ID);
            TenantAclIdDbId ids = AlfrescoSolrDataModel.decodeNodeDocumentId(id);
            childIds.add(ids.dbId);
        }

        for (Long childId : childIds) {
            NodeMetaDataParameters nmdp = new NodeMetaDataParameters();
            nmdp.setFromNodeId(childId);
            nmdp.setToNodeId(childId);
            nmdp.setIncludeAclId(false);
            nmdp.setIncludeAspects(false);
            nmdp.setIncludeChildAssociations(false);
            nmdp.setIncludeChildIds(true);
            nmdp.setIncludeNodeRef(false);
            nmdp.setIncludeOwner(false);
            nmdp.setIncludeParentAssociations(false);
            // We only care about the path and ancestors (which is included) for this case
            nmdp.setIncludePaths(true);
            nmdp.setIncludeProperties(false);
            nmdp.setIncludeType(false);
            nmdp.setIncludeTxnId(false);
            // Gets only one 
            List<NodeMetaData> nodeMetaDatas = repositoryClient.getNodesMetaData(nmdp, 1);

            if (!nodeMetaDatas.isEmpty()) {
                NodeMetaData nodeMetaData = nodeMetaDatas.get(0);
                if (mayHaveChildren(nodeMetaData)) {
                    updateDescendantDocs(nodeMetaData, overwrite, request, processor, stack);
                }

                if (log.isDebugEnabled()) {
                    log.debug("... cascade update child doc " + childId);
                }
                // Gets the document that we have from the content store and updates it 
                String fixedTenantDomain = AlfrescoSolrDataModel.getTenantId(nodeMetaData.getTenantDomain());
                SolrInputDocument cachedDoc = retrieveDocFromSolrContentStore(fixedTenantDomain,
                        nodeMetaData.getId());

                if (cachedDoc != null) {
                    updatePathRelatedFields(nodeMetaData, cachedDoc);
                    updateNamePathRelatedFields(nodeMetaData, cachedDoc);
                    updateAncestorRelatedFields(nodeMetaData, cachedDoc);

                    AddUpdateCommand addDocCmd = new AddUpdateCommand(request);
                    addDocCmd.overwrite = overwrite;
                    addDocCmd.solrDoc = cachedDoc;

                    processor.processAdd(addDocCmd);
                    storeDocOnSolrContentStore(fixedTenantDomain, nodeMetaData.getId(), cachedDoc);
                } else {
                    if (log.isDebugEnabled()) {
                        log.debug("... no child doc found to update " + childId);
                    }
                }
            }
        }
    }

    private void cascadeUpdate(NodeMetaData nodeMetaData, boolean overwrite, SolrQueryRequest request,
            UpdateRequestProcessor processor) throws AuthenticationException, IOException, JSONException {
        if (request.getSchema().getFieldOrNull(FIELD_CASCADETX) == null) {
            cascadeUpdateV1(nodeMetaData, overwrite, request, processor);
        } else {
            cascadeUpdateV2(nodeMetaData, overwrite, request, processor);
        }
    }

    private void cascadeUpdateV2(NodeMetaData parentNodeMetaData, boolean overwrite, SolrQueryRequest request,
            UpdateRequestProcessor processor) throws AuthenticationException, IOException, JSONException {
        StringPropertyValue cascadeTx = (StringPropertyValue) parentNodeMetaData.getProperties()
                .get(ContentModel.PROP_CASCADE_TX);
        if (cascadeTx != null) {
            StringBuilder builder = new StringBuilder();
            builder.append(FIELD_ANCESTOR).append(":\"").append(parentNodeMetaData.getNodeRef()).append("\"");
            builder.append(" AND -").append(FIELD_CASCADETX).append(":[").append(cascadeTx.getValue())
                    .append(" TO MAX]");
            builder.append(" AND ").append(FIELD_TENANT).append(":\"")
                    .append(AlfrescoSolrDataModel.getTenantId(parentNodeMetaData.getTenantDomain())).append("\"");

            ModifiableSolrParams params = new ModifiableSolrParams(request.getParams());
            params.set("q", builder.toString()).set("fl", FIELD_SOLR4_ID).set("rows", Integer.MAX_VALUE);
            if (skippingDocsQueryString != null && !skippingDocsQueryString.isEmpty()) {
                params.set("fq", "NOT ( " + skippingDocsQueryString + " )");
            }

            SolrDocumentList docs = cloud.getSolrDocumentList(core.getRequestHandler(REQUEST_HANDLER_SELECT),
                    request, params);

            HashSet<Long> childIds = new HashSet<Long>();
            for (SolrDocument doc : docs) {
                String id = getFieldValueString(doc, FIELD_SOLR4_ID);
                TenantAclIdDbId ids = AlfrescoSolrDataModel.decodeNodeDocumentId(id);
                childIds.add(ids.dbId);
            }

            for (Long childId : childIds) {
                NodeMetaDataParameters nmdp = new NodeMetaDataParameters();
                nmdp.setFromNodeId(childId);
                nmdp.setToNodeId(childId);
                nmdp.setIncludeAclId(false);
                nmdp.setIncludeAspects(false);
                nmdp.setIncludeChildAssociations(false);
                nmdp.setIncludeChildIds(true);
                nmdp.setIncludeNodeRef(false);
                nmdp.setIncludeOwner(false);
                nmdp.setIncludeParentAssociations(false);
                // We only care about the path and ancestors (which is included) for this case
                nmdp.setIncludePaths(true);
                nmdp.setIncludeProperties(false);
                nmdp.setIncludeType(false);
                nmdp.setIncludeTxnId(true);
                // Gets only one 
                List<NodeMetaData> nodeMetaDatas = repositoryClient.getNodesMetaData(nmdp, 1);

                if (!nodeMetaDatas.isEmpty()) {
                    NodeMetaData nodeMetaData = nodeMetaDatas.get(0);

                    // Only cascade update nods we know can not have changed and must be in this shard
                    // Node in the current TX will be explicitly updated in the outer loop
                    // We do not bring in changes from the future as nodes may switch shards and we do not want the logic here.
                    if (nodeMetaData.getTxnId() < parentNodeMetaData.getTxnId()) {
                        if (log.isDebugEnabled()) {
                            log.debug("... cascade update child doc " + childId);
                        }
                        // Gets the document that we have from the content store and updates it 
                        String fixedTenantDomain = AlfrescoSolrDataModel
                                .getTenantId(nodeMetaData.getTenantDomain());
                        SolrInputDocument cachedDoc = retrieveDocFromSolrContentStore(fixedTenantDomain,
                                nodeMetaData.getId());

                        if (cachedDoc == null) {
                            cachedDoc = recreateSolrDoc(nodeMetaData.getId(), fixedTenantDomain);

                            // if we did not build it again it has been deleted
                            // We do the delete here to avoid doing this again if it for some reason persists in teh index
                            // This is a work around for ACE-3228/ACE-3258 and the way stores are expunged when deleting a tenant
                            if (cachedDoc == null) {
                                deleteNode(processor, request, nodeMetaData.getId());
                            }
                        }

                        if (cachedDoc != null) {
                            updatePathRelatedFields(nodeMetaData, cachedDoc);
                            updateNamePathRelatedFields(nodeMetaData, cachedDoc);
                            updateAncestorRelatedFields(nodeMetaData, cachedDoc);

                            cachedDoc.removeField(FIELD_CASCADETX);
                            cachedDoc.addField(FIELD_CASCADETX, cascadeTx.getValue());

                            AddUpdateCommand addDocCmd = new AddUpdateCommand(request);
                            addDocCmd.overwrite = overwrite;
                            addDocCmd.solrDoc = cachedDoc;

                            processor.processAdd(addDocCmd);
                            storeDocOnSolrContentStore(fixedTenantDomain, nodeMetaData.getId(), cachedDoc);
                        } else {
                            if (log.isDebugEnabled()) {
                                log.debug("... no child doc found to update " + childId);
                            }
                        }
                    }

                }
            }

        }

    }

    /**
     * Checks if a cascade update is necessary, and then updates descendants
     */
    private void cascadeUpdateV1(NodeMetaData nodeMetaData, boolean overwrite, SolrQueryRequest request,
            UpdateRequestProcessor processor) throws AuthenticationException, IOException, JSONException {
        log.info(".. checking for path change");
        String query = FIELD_DBID + ":" + nodeMetaData.getId() + AND + FIELD_PARENT_ASSOC_CRC + ":"
                + nodeMetaData.getParentAssocsCrc();
        boolean nodeHasSamePathAsBefore = cloud.exists(nativeRequestHandler, request, query);
        if (nodeHasSamePathAsBefore) {
            if (log.isDebugEnabled()) {
                log.debug("... found match");
            }
        } else {
            query = FIELD_DBID + ":" + nodeMetaData.getId();
            boolean nodeHasBeenIndexed = cloud.exists(nativeRequestHandler, request, query);
            if (nodeHasBeenIndexed) {
                if (log.isDebugEnabled()) {
                    log.debug("... cascade updating docs");
                }
                LinkedHashSet<Long> visited = new LinkedHashSet<Long>();
                updateDescendantDocs(nodeMetaData, overwrite, request, processor, visited);
            } else {
                if (log.isDebugEnabled()) {
                    log.debug("... no doc to update");
                }
            }
        }
    }

    private NodeMetaData createDeletedNodeMetaData(Node node) {
        NodeMetaData nodeMetaData = new NodeMetaData();
        nodeMetaData.setId(node.getId());
        nodeMetaData.setType(ContentModel.TYPE_DELETED);
        nodeMetaData.setNodeRef(new NodeRef(node.getNodeRef()));
        nodeMetaData.setTxnId(node.getTxnId());
        return nodeMetaData;
    }

    private SolrInputDocument createNewDoc(NodeMetaData nodeMetaData, String docType) {
        SolrInputDocument doc = new SolrInputDocument();
        doc.addField(FIELD_SOLR4_ID, AlfrescoSolrDataModel.getNodeDocumentId(nodeMetaData.getTenantDomain(),
                nodeMetaData.getAclId(), nodeMetaData.getId()));
        doc.addField(FIELD_VERSION, 0);
        doc.addField(FIELD_DBID, nodeMetaData.getId());
        doc.addField(FIELD_LID, nodeMetaData.getNodeRef().toString());
        doc.addField(FIELD_INTXID, nodeMetaData.getTxnId());
        doc.addField(FIELD_DOC_TYPE, docType);
        doc.addField(FIELD_ACLID, nodeMetaData.getAclId());
        return doc;
    }

    @Override
    public void indexNodes(List<Node> nodes, boolean overwrite)
            throws IOException, AuthenticationException, JSONException {
        SolrQueryRequest request = null;
        UpdateRequestProcessor processor = null;
        try {
            request = getLocalSolrQueryRequest();
            processor = this.core.getUpdateProcessingChain(null).createProcessor(request, new SolrQueryResponse());

            Map<Long, Node> nodeIdsToNodes = new HashMap<>();
            EnumMap<SolrApiNodeStatus, List<Long>> nodeStatusToNodeIds = new EnumMap<SolrApiNodeStatus, List<Long>>(
                    SolrApiNodeStatus.class);
            categorizeNodes(nodes, nodeIdsToNodes, nodeStatusToNodeIds);
            List<Long> deletedNodeIds = mapNullToEmptyList(nodeStatusToNodeIds.get(SolrApiNodeStatus.DELETED));
            List<Long> shardDeletedNodeIds = mapNullToEmptyList(
                    nodeStatusToNodeIds.get(SolrApiNodeStatus.NON_SHARD_DELETED));
            List<Long> shardUpdatedNodeIds = mapNullToEmptyList(
                    nodeStatusToNodeIds.get(SolrApiNodeStatus.NON_SHARD_UPDATED));
            List<Long> unknownNodeIds = mapNullToEmptyList(nodeStatusToNodeIds.get(SolrApiNodeStatus.UNKNOWN));
            List<Long> updatedNodeIds = mapNullToEmptyList(nodeStatusToNodeIds.get(SolrApiNodeStatus.UPDATED));

            if (!deletedNodeIds.isEmpty() || !shardDeletedNodeIds.isEmpty() || !shardUpdatedNodeIds.isEmpty()
                    || !unknownNodeIds.isEmpty()) {
                // Delete any cached SOLR content documents  appropriate for this shard
                List<NodeMetaData> nodeMetaDatas = new ArrayList<>();

                // For all deleted nodes, fake the node metadata
                for (Long deletedNodeId : deletedNodeIds) {
                    Node node = nodeIdsToNodes.get(deletedNodeId);
                    NodeMetaData nodeMetaData = createDeletedNodeMetaData(node);
                    nodeMetaDatas.add(nodeMetaData);
                }

                if (!unknownNodeIds.isEmpty()) {
                    NodeMetaDataParameters nmdp = new NodeMetaDataParameters();
                    nmdp.setNodeIds(unknownNodeIds);
                    nodeMetaDatas.addAll(repositoryClient.getNodesMetaData(nmdp, Integer.MAX_VALUE));
                }

                for (NodeMetaData nodeMetaData : nodeMetaDatas) {
                    Node node = nodeIdsToNodes.get(nodeMetaData.getId());
                    if (nodeMetaData.getTxnId() > node.getTxnId()) {
                        // the node has moved on to a later transaction
                        // it will be indexed later
                        continue;
                    }
                    if (nodeMetaData != null) {
                        this.removeDocFromContentStore(nodeMetaData);
                    }
                }

                // Delete the nodes from the index
                if (log.isDebugEnabled()) {
                    log.debug(".. deleting");
                }
                DeleteUpdateCommand delDocCmd = new DeleteUpdateCommand(request);
                String query = this.cloud.getQuery(FIELD_DBID, OR, deletedNodeIds, shardDeletedNodeIds,
                        shardUpdatedNodeIds, unknownNodeIds);
                delDocCmd.setQuery(query);
                processor.processDelete(delDocCmd);
            }

            if (!updatedNodeIds.isEmpty() || !unknownNodeIds.isEmpty() || !shardUpdatedNodeIds.isEmpty()) {
                log.info(".. updating");
                NodeMetaDataParameters nmdp = new NodeMetaDataParameters();
                List<Long> nodeIds = new LinkedList<>();
                nodeIds.addAll(updatedNodeIds);
                nodeIds.addAll(unknownNodeIds);
                nodeIds.addAll(shardUpdatedNodeIds);
                nmdp.setNodeIds(nodeIds);

                // Fetches bulk metadata
                List<NodeMetaData> nodeMetaDatas = repositoryClient.getNodesMetaData(nmdp, Integer.MAX_VALUE);

                NEXT_NODE: for (NodeMetaData nodeMetaData : nodeMetaDatas) {
                    long start = System.nanoTime();

                    Node node = nodeIdsToNodes.get(nodeMetaData.getId());
                    if (nodeMetaData.getTxnId() > node.getTxnId()) {
                        // the node has moved on to a later transaction
                        // it will be indexed later
                        continue;
                    }

                    // All do potential cascade
                    if (mayHaveChildren(nodeMetaData)) {
                        cascadeUpdate(nodeMetaData, overwrite, request, processor);
                    }

                    // NON_SHARD_UPDATED do not index just cascade
                    if (nodeIdsToNodes.get(nodeMetaData.getId())
                            .getStatus() == SolrApiNodeStatus.NON_SHARD_UPDATED) {
                        continue;
                    }

                    AddUpdateCommand addDocCmd = new AddUpdateCommand(request);
                    addDocCmd.overwrite = overwrite;

                    // check index control
                    Map<QName, PropertyValue> properties = nodeMetaData.getProperties();
                    StringPropertyValue pValue = (StringPropertyValue) properties.get(ContentModel.PROP_IS_INDEXED);
                    if (pValue != null) {
                        Boolean isIndexed = Boolean.valueOf(pValue.getValue());
                        if (!isIndexed.booleanValue()) {
                            if (log.isDebugEnabled()) {
                                log.debug(".. clearing unindexed");
                            }
                            deleteNode(processor, request, node);

                            SolrInputDocument doc = createNewDoc(nodeMetaData, DOC_TYPE_UNINDEXED_NODE);
                            storeDocOnSolrContentStore(nodeMetaData, doc);
                            addDocCmd.solrDoc = doc;
                            if (recordUnindexedNodes) {
                                processor.processAdd(addDocCmd);
                            }

                            long end = System.nanoTime();
                            this.trackerStats.addNodeTime(end - start);
                            continue NEXT_NODE;
                        }
                    }

                    // Make sure any unindexed or error doc is removed.
                    if (log.isDebugEnabled()) {
                        log.debug(".. deleting node " + node.getId());
                    }
                    deleteNode(processor, request, node);

                    SolrInputDocument doc = createNewDoc(nodeMetaData, DOC_TYPE_NODE);
                    addToNewDocAndCache(nodeMetaData, doc);
                    addDocCmd.solrDoc = doc;
                    processor.processAdd(addDocCmd);

                    long end = System.nanoTime();
                    this.trackerStats.addNodeTime(end - start);
                } // Ends iteration over nodeMetadatas
            } // Ends checking for the existence of updated or unknown node ids 

        } catch (Exception e) {
            // Bulk version failed, so do one at a time.
            for (Node node : nodes) {
                this.indexNode(node, true);
            }
        } finally {
            if (processor != null) {
                processor.finish();
            }
            if (request != null) {
                request.close();
            }
        }

    }

    private void addToNewDocAndCache(NodeMetaData nodeMetaData, SolrInputDocument newDoc)
            throws IOException, AuthenticationException {
        addFieldsToDoc(nodeMetaData, newDoc);
        SolrInputDocument cachedDoc = null;
        boolean isContentIndexedForNode = isContentIndexedForNode(nodeMetaData.getProperties());
        String fixedTenantDomain = AlfrescoSolrDataModel.getTenantId(nodeMetaData.getTenantDomain());
        if (isContentIndexedForNode) {
            cachedDoc = retrieveDocFromSolrContentStore(fixedTenantDomain, nodeMetaData.getId());
        }
        Map<QName, PropertyValue> properties = nodeMetaData.getProperties();
        addPropertiesToDoc(properties, isContentIndexedForNode, newDoc, cachedDoc, transformContent);

        // Now that the new doc is fully updated and ready to go to the Solr index, cache it.
        storeDocOnSolrContentStore(fixedTenantDomain, nodeMetaData.getId(), newDoc);

    }

    private void addFieldsToDoc(NodeMetaData nodeMetaData, SolrInputDocument doc) {
        doc.addField(FIELD_TYPE, nodeMetaData.getType().toString());
        for (QName aspect : nodeMetaData.getAspects()) {
            doc.addField(FIELD_ASPECT, aspect.toString());
            if (aspect.equals(ContentModel.ASPECT_GEOGRAPHIC)) {
                StringPropertyValue latProp = ((StringPropertyValue) nodeMetaData.getProperties()
                        .get(ContentModel.PROP_LATITUDE));
                StringPropertyValue lonProp = ((StringPropertyValue) nodeMetaData.getProperties()
                        .get(ContentModel.PROP_LONGITUDE));

                if ((latProp != null) && (lonProp != null)) {
                    String lat = latProp.getValue();
                    String lon = lonProp.getValue();

                    // Only add if the data is valid

                    if ((lat != null) && (lon != null)) {
                        try {
                            double dLat = Double.parseDouble(lat);
                            double dLon = Double.parseDouble(lon);

                            if ((-90d <= dLat) && (dLat <= 90d) && (-180d <= dLon) && (dLon <= 180d)) {
                                doc.addField(FIELD_GEO, lat + ", " + lon);
                            }
                        } catch (NumberFormatException nfe) {
                            log.info("Skipping invalid geo data on node " + nodeMetaData.getId() + " -> (" + lat
                                    + ", " + lon + ")");
                        }
                    } else {
                        log.info("Skipping missing geo data on " + nodeMetaData.getId());
                    }
                } else {
                    log.info("Skipping missing geo data on " + nodeMetaData.getId());
                }
            }
        }
        doc.addField(FIELD_ISNODE, "T");
        // FIELD_FTSSTATUS is set when adding content properties to indicate whether or not the cache is clean.

        doc.addField(FIELD_TENANT, AlfrescoSolrDataModel.getTenantId(nodeMetaData.getTenantDomain()));

        updatePathRelatedFields(nodeMetaData, doc);
        updateNamePathRelatedFields(nodeMetaData, doc);
        updateAncestorRelatedFields(nodeMetaData, doc);
        doc.addField(FIELD_PARENT_ASSOC_CRC, nodeMetaData.getParentAssocsCrc());

        if (nodeMetaData.getOwner() != null) {
            doc.addField(FIELD_OWNER, nodeMetaData.getOwner());
        }

        StringBuilder qNameBuffer = new StringBuilder(64);
        StringBuilder assocTypeQNameBuffer = new StringBuilder(64);
        if (nodeMetaData.getParentAssocs() != null) {
            for (ChildAssociationRef childAssocRef : nodeMetaData.getParentAssocs()) {
                if (qNameBuffer.length() > 0) {
                    qNameBuffer.append(";/");
                    assocTypeQNameBuffer.append(";/");
                }
                qNameBuffer.append(ISO9075.getXPathName(childAssocRef.getQName()));
                assocTypeQNameBuffer.append(ISO9075.getXPathName(childAssocRef.getTypeQName()));
                doc.addField(FIELD_PARENT, childAssocRef.getParentRef().toString());

                if (childAssocRef.isPrimary()) {
                    if (doc.getField(FIELD_PRIMARYPARENT) == null) {
                        doc.addField(FIELD_PRIMARYPARENT, childAssocRef.getParentRef().toString());
                        doc.addField(FIELD_PRIMARYASSOCTYPEQNAME,
                                ISO9075.getXPathName(childAssocRef.getTypeQName()));
                        doc.addField(FIELD_PRIMARYASSOCQNAME, ISO9075.getXPathName(childAssocRef.getQName()));
                    } else {
                        log.warn("Duplicate primary parent for node id " + nodeMetaData.getId());
                    }
                }
            }
            doc.addField(FIELD_ASSOCTYPEQNAME, assocTypeQNameBuffer.toString());
            doc.addField(FIELD_QNAME, qNameBuffer.toString());
        }

    }

    /**
     * @param nodeMetaData
     * @param doc
     */
    private void updateAncestorRelatedFields(NodeMetaData nodeMetaData, SolrInputDocument doc) {
        doc.removeField(FIELD_ANCESTOR);
        if (nodeMetaData.getAncestors() != null) {
            for (NodeRef ancestor : nodeMetaData.getAncestors()) {
                doc.addField(FIELD_ANCESTOR, ancestor.toString());
            }
        }
    }

    /**
     * @param nodeMetaData
     * @param doc
     */
    private void updateNamePathRelatedFields(NodeMetaData nodeMetaData, SolrInputDocument doc) {
        doc.removeField(FIELD_NPATH);
        doc.removeField(FIELD_PNAME);

        for (List<String> namePath : nodeMetaData.getNamePaths()) {
            StringBuilder builder = new StringBuilder();
            int i = 0;
            for (String element : namePath) {
                builder.append('/').append(element);
                doc.addField(FIELD_NPATH, "" + i++ + builder.toString());
            }
            if (builder.length() > 0) {
                doc.addField(FIELD_NPATH, "F" + builder.toString());
            }

            builder = new StringBuilder();
            for (int j = 0; j < namePath.size() - 1; j++) {
                String element = namePath.get(namePath.size() - 2 - j);
                builder.insert(0, element);
                builder.insert(0, '/');
                doc.addField(FIELD_PNAME, "" + j + builder.toString());
            }
            if (builder.length() > 0) {
                doc.addField(FIELD_PNAME, "F" + builder.toString());
            }
        }
    }

    /**
     * @param nodeMetaData
     * @param doc
     */
    private void updatePathRelatedFields(NodeMetaData nodeMetaData, SolrInputDocument doc) {
        doc.removeField(FIELD_PATH);
        doc.removeField(FIELD_SITE);
        doc.removeField(FIELD_TAG);
        doc.removeField(FIELD_TAG_SUGGEST);

        boolean repoOnly = true;
        for (Pair<String, QName> path : nodeMetaData.getPaths()) {
            doc.addField(FIELD_PATH, path.getFirst());

            Matcher matcher = CAPTURE_SITE.matcher(path.getFirst());
            if (matcher.find()) {
                repoOnly = false;
                doc.addField(FIELD_SITE, ISO9075.decode(matcher.group(1)));
            }

            matcher = CAPTURE_SHARED_FILES.matcher(path.getFirst());
            if (matcher.find()) {
                repoOnly = false;
                doc.addField(FIELD_SITE, SHARED_FILES);
            }

            matcher = CAPTURE_TAG.matcher(path.getFirst());
            if (matcher.find()) {
                String tag = ISO9075.decode(matcher.group(1));
                doc.addField(FIELD_TAG, tag);
                doc.addField(FIELD_TAG_SUGGEST, tag);
            }

        }

        if (repoOnly) {
            doc.addField(FIELD_SITE, NO_SITE);
        }
    }

    static void addPropertiesToDoc(Map<QName, PropertyValue> properties, boolean isContentIndexedForNode,
            SolrInputDocument newDoc, SolrInputDocument cachedDoc, boolean transformContentFlag)
            throws IOException {
        for (QName propertyQName : properties.keySet()) {
            newDoc.addField(FIELD_PROPERTIES, propertyQName.toString());
            newDoc.addField(FIELD_PROPERTIES, propertyQName.getPrefixString());

            PropertyValue value = properties.get(propertyQName);
            if (value != null) {
                if (value instanceof StringPropertyValue) {
                    for (FieldInstance field : AlfrescoSolrDataModel.getInstance()
                            .getIndexedFieldNamesForProperty(propertyQName).getFields()) {
                        addStringPropertyToDoc(newDoc, field, (StringPropertyValue) value, properties);
                    }
                } else if (value instanceof MLTextPropertyValue) {
                    for (FieldInstance field : AlfrescoSolrDataModel.getInstance()
                            .getIndexedFieldNamesForProperty(propertyQName).getFields()) {
                        addMLTextPropertyToDoc(newDoc, field, (MLTextPropertyValue) value);
                    }

                } else if (value instanceof ContentPropertyValue) {
                    if (isContentIndexedForNode) {
                        addContentPropertyToDocUsingCache(newDoc, cachedDoc, propertyQName,
                                (ContentPropertyValue) value, transformContentFlag);
                    }
                } else if (value instanceof MultiPropertyValue) {
                    MultiPropertyValue typedValue = (MultiPropertyValue) value;
                    for (PropertyValue singleValue : typedValue.getValues()) {
                        if (singleValue instanceof StringPropertyValue) {
                            for (FieldInstance field : AlfrescoSolrDataModel.getInstance()
                                    .getIndexedFieldNamesForProperty(propertyQName).getFields()) {
                                addStringPropertyToDoc(newDoc, field, (StringPropertyValue) singleValue,
                                        properties);
                            }
                        } else if (singleValue instanceof MLTextPropertyValue) {
                            for (FieldInstance field : AlfrescoSolrDataModel.getInstance()
                                    .getIndexedFieldNamesForProperty(propertyQName).getFields()) {
                                addMLTextPropertyToDoc(newDoc, field, (MLTextPropertyValue) singleValue);
                            }

                        } else if (singleValue instanceof ContentPropertyValue) {
                            if (isContentIndexedForNode) {
                                addContentPropertyToDocUsingCache(newDoc, cachedDoc, propertyQName,
                                        (ContentPropertyValue) singleValue, transformContentFlag);
                            }
                        }
                    }
                }
            } else {
                // NULL property
                newDoc.addField(FIELD_NULLPROPERTIES, propertyQName.toString());
            }
        }
    }

    private void deleteNode(UpdateRequestProcessor processor, SolrQueryRequest request, Node node)
            throws IOException {
        String errorDocId = PREFIX_ERROR + node.getId();
        DeleteUpdateCommand delErrorDocCmd = new DeleteUpdateCommand(request);
        delErrorDocCmd.setId(errorDocId);
        processor.processDelete(delErrorDocCmd);
        // MNT-13767 fix, remove by node DBID.
        deleteNode(processor, request, node.getId());
    }

    private void deleteNode(UpdateRequestProcessor processor, SolrQueryRequest request, long dbid)
            throws IOException {
        DeleteUpdateCommand delDocCmd = new DeleteUpdateCommand(request);
        delDocCmd.setQuery(FIELD_DBID + ":" + dbid);
        processor.processDelete(delDocCmd);
    }

    private boolean isContentIndexedForNode(Map<QName, PropertyValue> properties) {
        boolean isContentIndexed = true;
        if (properties.containsKey(ContentModel.PROP_IS_CONTENT_INDEXED)) {
            StringPropertyValue pValue = (StringPropertyValue) properties.get(ContentModel.PROP_IS_CONTENT_INDEXED);
            if (pValue != null) {
                Boolean isIndexed = Boolean.valueOf(pValue.getValue());
                if ((isIndexed != null) && (isIndexed.booleanValue() == false)) {
                    isContentIndexed = false;
                }
            }
        }
        return isContentIndexed;
    }

    /**
     * @param list List<Long>
     * @return List<Long>
     */
    private List<Long> mapNullToEmptyList(List<Long> list) {
        return list == null ? Collections.<Long>emptyList() : list;
    }

    private void categorizeNodes(List<Node> nodes, Map<Long, Node> nodeIdsToNodes,
            EnumMap<SolrApiNodeStatus, List<Long>> nodeStatusToNodeIds) {
        for (Node node : nodes) {
            nodeIdsToNodes.put(node.getId(), node);

            List<Long> nodeIds = nodeStatusToNodeIds.get(node.getStatus());
            if (nodeIds == null) {
                nodeIds = new LinkedList<>();
                nodeStatusToNodeIds.put(node.getStatus(), nodeIds);
            }
            nodeIds.add(node.getId());
        }
    }

    /**
     * Gets the field name used in Solr for the specified content property.
     * Assumes that the first defined field in Solr is the "right one".
     * @param propertyQName the content property qualified name
     * @param type the content property field type, i.e. DOCID
     * @return a String representing the name of the field in Solr or null if not found
     */
    private static String getSolrFieldNameForContentPropertyMetadata(QName propertyQName,
            AlfrescoSolrDataModel.ContentFieldType type) {
        IndexedField indexedField = AlfrescoSolrDataModel.getInstance()
                .getIndexedFieldForContentPropertyMetadata(propertyQName, type);
        List<FieldInstance> fields = indexedField.getFields();
        String fieldName = null;
        if (fields != null && !fields.isEmpty()) {
            FieldInstance instance = fields.get(0);
            if (instance != null) {
                fieldName = instance.getField();
            }
        }
        return fieldName;
    }

    private void addContentPropertyMetadata(SolrInputDocument doc, QName propertyQName,
            AlfrescoSolrDataModel.ContentFieldType type, GetTextContentResponse textContentResponse) {
        IndexedField indexedField = AlfrescoSolrDataModel.getInstance()
                .getIndexedFieldForContentPropertyMetadata(propertyQName, type);
        for (FieldInstance fieldInstance : indexedField.getFields()) {
            doc.removeField(fieldInstance.getField());
            switch (type) {
            case TRANSFORMATION_EXCEPTION:
                doc.addField(fieldInstance.getField(), textContentResponse.getTransformException());
                break;
            case TRANSFORMATION_STATUS:
                doc.addField(fieldInstance.getField(), textContentResponse.getStatus());
                break;
            case TRANSFORMATION_TIME:
                doc.addField(fieldInstance.getField(), textContentResponse.getTransformDuration());
                break;
            // Skips the ones that require the ContentPropertyValue
            default:
                break;
            }
        }
    }

    private static void addContentPropertyMetadata(SolrInputDocument doc, QName propertyQName,
            ContentPropertyValue contentPropertyValue, AlfrescoSolrDataModel.ContentFieldType type) {
        IndexedField indexedField = AlfrescoSolrDataModel.getInstance()
                .getIndexedFieldForContentPropertyMetadata(propertyQName, type);
        for (FieldInstance fieldInstance : indexedField.getFields()) {
            switch (type) {
            case DOCID:
                doc.addField(fieldInstance.getField(), contentPropertyValue.getId());
                break;
            case ENCODING:
                doc.addField(fieldInstance.getField(), contentPropertyValue.getEncoding());
                break;
            case LOCALE:
                doc.addField(fieldInstance.getField(), contentPropertyValue.getLocale().toString());
                break;
            case MIMETYPE:
                doc.addField(fieldInstance.getField(), contentPropertyValue.getMimetype());
                break;
            case SIZE:
                doc.addField(fieldInstance.getField(), contentPropertyValue.getLength());
                break;
            // Skips the ones that require the text content response
            default:
                break;
            }
        }
    }

    private static void addContentPropertyToDocUsingCache(SolrInputDocument newDoc, SolrInputDocument cachedDoc,
            QName propertyQName, ContentPropertyValue contentPropertyValue, boolean transformContentFlag)
            throws IOException {
        addContentPropertyMetadata(newDoc, propertyQName, contentPropertyValue,
                AlfrescoSolrDataModel.ContentFieldType.DOCID);
        addContentPropertyMetadata(newDoc, propertyQName, contentPropertyValue,
                AlfrescoSolrDataModel.ContentFieldType.SIZE);
        addContentPropertyMetadata(newDoc, propertyQName, contentPropertyValue,
                AlfrescoSolrDataModel.ContentFieldType.LOCALE);
        addContentPropertyMetadata(newDoc, propertyQName, contentPropertyValue,
                AlfrescoSolrDataModel.ContentFieldType.MIMETYPE);
        addContentPropertyMetadata(newDoc, propertyQName, contentPropertyValue,
                AlfrescoSolrDataModel.ContentFieldType.ENCODING);

        if (false == transformContentFlag) {
            // Marks it as Clean so we do not get the actual content
            markFTSStatus(newDoc, FTSStatus.Clean);
            return;
        }

        if (cachedDoc != null) {
            // Builds up the new solr doc from the cached content regardless of whether or not it is current
            List<FieldInstance> fields = AlfrescoSolrDataModel.getInstance()
                    .getIndexedFieldNamesForProperty(propertyQName).getFields();
            for (FieldInstance field : fields) {
                String fieldName = field.getField();
                Object cachedFieldValue = cachedDoc.getFieldValue(fieldName);
                newDoc.addField(fieldName, cachedFieldValue);
                addFieldIfNotSet(newDoc, field);
            }

            String transformationStatusFieldName = getSolrFieldNameForContentPropertyMetadata(propertyQName,
                    AlfrescoSolrDataModel.ContentFieldType.TRANSFORMATION_STATUS);
            if (transformationStatusFieldName != null) {
                newDoc.addField(transformationStatusFieldName,
                        cachedDoc.getFieldValue(transformationStatusFieldName));
            }
            String transformationExceptionFieldName = getSolrFieldNameForContentPropertyMetadata(propertyQName,
                    AlfrescoSolrDataModel.ContentFieldType.TRANSFORMATION_EXCEPTION);
            if (transformationExceptionFieldName != null) {
                newDoc.addField(transformationExceptionFieldName,
                        cachedDoc.getFieldValue(transformationExceptionFieldName));
            }
            String transformationTimeFieldName = getSolrFieldNameForContentPropertyMetadata(propertyQName,
                    AlfrescoSolrDataModel.ContentFieldType.TRANSFORMATION_TIME);
            if (transformationTimeFieldName != null) {
                newDoc.addField(transformationTimeFieldName, cachedDoc.getFieldValue(transformationTimeFieldName));
            }

            // Gets the new content docid and compares to that of the cachedDoc to mark the content as clean/dirty
            String fldName = getSolrFieldNameForContentPropertyMetadata(propertyQName,
                    AlfrescoSolrDataModel.ContentFieldType.DOCID);

            if (newDoc.getFieldValue(FIELD_FTSSTATUS) == null) {
                newDoc.addField(FIELD_FTSSTATUS, cachedDoc.getFieldValue(FIELD_FTSSTATUS));
            }

            if (cachedDoc.getFieldValue(fldName) != null) {
                long cachedDocContentDocid = Long.valueOf(String.valueOf(cachedDoc.getFieldValue(fldName)));
                long currentContentDocid = contentPropertyValue.getId();
                // If we have used out of date content we mark it as dirty
                // Otherwise we leave it alone - it could already be marked as dirty/New and require an update

                if (cachedDocContentDocid != currentContentDocid) {
                    // The cached content is out of date
                    markFTSStatus(newDoc, FTSStatus.Dirty);
                }
            } else {
                markFTSStatus(newDoc, FTSStatus.Dirty);
            }
        } else {
            // There is not a SolrInputDocument in the solrContentStore, so no content is added now to the new solr doc
            markFTSStatus(newDoc, FTSStatus.New);
        }
    }

    @Override
    public void updateContentToIndexAndCache(long dbId, String tenant) throws Exception {
        SolrQueryRequest request = null;
        UpdateRequestProcessor processor = null;
        try {
            request = getLocalSolrQueryRequest();
            processor = this.core.getUpdateProcessingChain(null).createProcessor(request, new SolrQueryResponse());

            SolrInputDocument doc = retrieveDocFromSolrContentStore(tenant, dbId);
            if (doc == null) {
                log.warn("There is no cached doc in the Solr content store with tenant [" + tenant + "] and dbId ["
                        + dbId + "].\n"
                        + "This should only happen if the content has been removed from the Solr content store.\n"
                        + "Recreating cached doc ... ");
                doc = recreateSolrDoc(dbId, tenant);

                // if we did not build it again it has been deleted
                // We do the delete here to avoid doing this again if it for some reason persists in teh index
                // This is a work around for ACE-3228/ACE-3258 and the way stores are expunged when deleting a tenant
                if (doc == null) {
                    deleteNode(processor, request, dbId);
                }
            }

            if (doc != null) {
                addContentToDoc(doc, dbId);
                // Marks as clean since the doc's content is now up to date
                markFTSStatus(doc, FTSStatus.Clean);
                storeDocOnSolrContentStore(tenant, dbId, doc);

                // Add to index
                AddUpdateCommand addDocCmd = new AddUpdateCommand(request);
                addDocCmd.overwrite = true;
                addDocCmd.solrDoc = doc;
                processor.processAdd(addDocCmd);
            }
        } finally {
            if (processor != null) {
                processor.finish();
            }
            if (request != null) {
                request.close();
            }
        }
    }

    private SolrInputDocument recreateSolrDoc(long dbId, String tenant)
            throws AuthenticationException, IOException, JSONException {
        NodeMetaDataParameters nmdp = new NodeMetaDataParameters();
        nmdp.setFromNodeId(dbId);
        nmdp.setToNodeId(dbId);
        List<NodeMetaData> nodeMetaDatas = repositoryClient.getNodesMetaData(nmdp, Integer.MAX_VALUE);
        SolrInputDocument newDoc = null;
        if (!nodeMetaDatas.isEmpty()) {
            NodeMetaData nodeMetaData = nodeMetaDatas.get(0);
            newDoc = createNewDoc(nodeMetaData, DOC_TYPE_NODE);
            addFieldsToDoc(nodeMetaData, newDoc);
            boolean isContentIndexedForNode = isContentIndexedForNode(nodeMetaData.getProperties());
            Map<QName, PropertyValue> properties = nodeMetaData.getProperties();
            // Cached doc is null for this method because it is a recreate after a cache miss.
            addPropertiesToDoc(properties, isContentIndexedForNode, newDoc, null, transformContent);
        } else {
            // we get an empty list if a node is deleted
            if (log.isDebugEnabled()) {
                log.debug("Failed to recreate Solr doc with tenant [" + tenant + "] and dbId [" + dbId + "], "
                        + "because node not found in repository.");
            }
        }
        return newDoc;
    }

    private static void markFTSStatus(SolrInputDocument doc, FTSStatus status) {
        doc.removeField(FIELD_FTSSTATUS);
        doc.addField(FIELD_FTSSTATUS, status.toString());
    }

    private void addContentToDoc(SolrInputDocument doc, long dbId)
            throws UnsupportedEncodingException, AuthenticationException, IOException {
        Collection<String> fieldNames = doc.deepCopy().getFieldNames();
        for (String fieldName : fieldNames) {
            if (fieldName.startsWith(AlfrescoSolrDataModel.CONTENT_S_LOCALE_PREFIX)) {
                String locale = String.valueOf(doc.getFieldValue(fieldName));
                String qNamePart = fieldName.substring(AlfrescoSolrDataModel.CONTENT_S_LOCALE_PREFIX.length());
                QName propertyQName = QName.createQName(qNamePart);
                addContentPropertyToDocUsingAlfrescoRepository(doc, propertyQName, dbId, locale);
            }
            // Could update multi content but it is broken ....
        }
    }

    private void addContentPropertyToDocUsingAlfrescoRepository(SolrInputDocument doc, QName propertyQName,
            long dbId, String locale) throws AuthenticationException, IOException, UnsupportedEncodingException {
        long start = System.nanoTime();

        // Expensive call to be done with ContentTracker
        GetTextContentResponse response = repositoryClient.getTextContent(dbId, propertyQName, null);

        addContentPropertyMetadata(doc, propertyQName, AlfrescoSolrDataModel.ContentFieldType.TRANSFORMATION_STATUS,
                response);
        addContentPropertyMetadata(doc, propertyQName,
                AlfrescoSolrDataModel.ContentFieldType.TRANSFORMATION_EXCEPTION, response);
        addContentPropertyMetadata(doc, propertyQName, AlfrescoSolrDataModel.ContentFieldType.TRANSFORMATION_TIME,
                response);

        InputStream ris = response.getContent();
        String textContent = "";
        try {
            if (ris != null) {
                // Get and copy content
                byte[] bytes = FileCopyUtils.copyToByteArray(new BoundedInputStream(ris, contentStreamLimit));
                textContent = new String(bytes, "UTF8");
            }
        } finally {
            // release the response only when the content has been read
            response.release();
        }

        long end = System.nanoTime();
        this.getTrackerStats().addDocTransformationTime(end - start);

        StringBuilder builder = new StringBuilder(textContent.length() + 16);
        builder.append("\u0000").append(locale).append("\u0000");
        builder.append(textContent);
        String localisedText = builder.toString();

        for (FieldInstance field : AlfrescoSolrDataModel.getInstance()
                .getIndexedFieldNamesForProperty(propertyQName).getFields()) {
            doc.removeField(field.getField());
            if (field.isLocalised()) {
                doc.addField(field.getField(), localisedText);
            } else {
                doc.addField(field.getField(), textContent);
            }
            addFieldIfNotSet(doc, field);
        }
    }

    private void removeDocFromContentStore(NodeMetaData nodeMetaData) {
        String fixedTenantDomain = AlfrescoSolrDataModel.getTenantId(nodeMetaData.getTenantDomain());
        String contentUrl = SolrContentUrlBuilder.start().add(SolrContentUrlBuilder.KEY_TENANT, fixedTenantDomain)
                .add(SolrContentUrlBuilder.KEY_DB_ID, String.valueOf(nodeMetaData.getId())).getContentContext()
                .getContentUrl();
        this.solrContentStore.delete(contentUrl);
    }

    private void storeDocOnSolrContentStore(NodeMetaData nodeMetaData, SolrInputDocument doc) throws IOException {
        String fixedTenantDomain = AlfrescoSolrDataModel.getTenantId(nodeMetaData.getTenantDomain());
        storeDocOnSolrContentStore(fixedTenantDomain, nodeMetaData.getId(), doc);
    }

    private void storeDocOnSolrContentStore(String tenant, long dbId, SolrInputDocument doc) throws IOException {
        ContentContext contentContext = SolrContentUrlBuilder.start().add(SolrContentUrlBuilder.KEY_TENANT, tenant)
                .add(SolrContentUrlBuilder.KEY_DB_ID, String.valueOf(dbId)).getContentContext();
        this.solrContentStore.delete(contentContext.getContentUrl());
        ContentWriter writer = this.solrContentStore.getWriter(contentContext);
        if (log.isDebugEnabled()) {
            log.debug("Writing doc to " + contentContext.getContentUrl());
        }
        try (OutputStream contentOutputStream = writer.getContentOutputStream();
                // Compresses the document
                GZIPOutputStream gzip = new GZIPOutputStream(contentOutputStream);) {
            JavaBinCodec codec = new JavaBinCodec(resolver);
            codec.marshal(doc, gzip);
        } catch (Exception e) {
            // A failure to write to the store is acceptable as long as it's logged
            log.warn("Failed to write to store using URL: " + contentContext.getContentUrl(), e);
        }
    }

    private SolrInputDocument retrieveDocFromSolrContentStore(String tenant, long dbId) throws IOException {
        String contentUrl = SolrContentUrlBuilder.start().add(SolrContentUrlBuilder.KEY_TENANT, tenant)
                .add(SolrContentUrlBuilder.KEY_DB_ID, String.valueOf(dbId)).get();
        ContentReader reader = this.solrContentStore.getReader(contentUrl);
        SolrInputDocument cachedDoc = null;
        if (reader.exists()) {
            // try-with-resources statement closes all these InputStreams
            try (InputStream contentInputStream = reader.getContentInputStream();
                    // Uncompresses the document
                    GZIPInputStream gzip = new GZIPInputStream(contentInputStream);) {
                cachedDoc = (SolrInputDocument) new JavaBinCodec(resolver).unmarshal(gzip);
            } catch (Exception e) {
                // Don't fail for this
                log.warn("Failed to get doc from store using URL: " + contentUrl, e);
                return null;
            }
        }
        return cachedDoc;
    }

    private static void addMLTextPropertyToDoc(SolrInputDocument doc, FieldInstance field,
            MLTextPropertyValue mlTextPropertyValue) throws IOException {
        if (field.isLocalised()) {
            StringBuilder sort = new StringBuilder(128);
            for (Locale locale : mlTextPropertyValue.getLocales()) {
                final String propValue = mlTextPropertyValue.getValue(locale);
                if (log.isDebugEnabled()) {
                    log.debug("ML " + field.getField() + " in " + locale + " of " + propValue);
                }

                if ((locale == null) || (propValue == null)) {
                    continue;
                }

                StringBuilder builder = new StringBuilder(propValue.length() + 16);
                builder.append("\u0000").append(locale.toString()).append("\u0000").append(propValue);

                if (!field.isSort()) {
                    doc.addField(field.getField(), builder.toString());
                }

                if (sort.length() > 0) {
                    sort.append("\u0000");
                }
                sort.append(builder.toString());
            }
            if (field.isSort()) {
                doc.addField(field.getField(), sort.toString());
            }
        } else {
            for (Locale locale : mlTextPropertyValue.getLocales()) {
                doc.addField(field.getField(), mlTextPropertyValue.getValue(locale));
            }
        }
        addFieldIfNotSet(doc, field);
    }

    private static void addStringPropertyToDoc(SolrInputDocument doc, FieldInstance field,
            StringPropertyValue stringPropertyValue, Map<QName, PropertyValue> properties) throws IOException {
        if (field.isLocalised()) {
            Locale locale = null;

            PropertyValue localePropertyValue = properties.get(ContentModel.PROP_LOCALE);
            if (localePropertyValue != null) {
                locale = DefaultTypeConverter.INSTANCE.convert(Locale.class,
                        ((StringPropertyValue) localePropertyValue).getValue());
            }

            if (locale == null) {
                locale = I18NUtil.getLocale();
            }

            StringBuilder builder;
            builder = new StringBuilder();
            builder.append("\u0000").append(locale.toString()).append("\u0000")
                    .append(stringPropertyValue.getValue());

            doc.addField(field.getField(), builder.toString());
        } else {
            doc.addField(field.getField(), stringPropertyValue.getValue());
        }
        addFieldIfNotSet(doc, field);
    }

    private static void addFieldIfNotSet(SolrInputDocument doc, FieldInstance field) {
        Collection<Object> values = doc.getFieldValues(FIELD_FIELDS);
        if (values != null) {
            for (Object o : values) {
                if (o instanceof String) {
                    if (((String) o).equals(field.getField())) {
                        return;
                    }

                }
            }
        }
        doc.addField(FIELD_FIELDS, field.getField());
    }

    private boolean mayHaveChildren(NodeMetaData nodeMetaData) {
        // 1) Does the type support children?
        TypeDefinition nodeTypeDef = dataModel.getDictionaryService(CMISStrictDictionaryService.DEFAULT)
                .getType(nodeMetaData.getType());
        if ((nodeTypeDef != null) && (nodeTypeDef.getChildAssociations().size() > 0)) {
            return true;
        }
        // 2) Do any of the applied aspects support children?
        for (QName aspect : nodeMetaData.getAspects()) {
            AspectDefinition aspectDef = dataModel.getDictionaryService(CMISStrictDictionaryService.DEFAULT)
                    .getAspect(aspect);
            if ((aspectDef != null) && (aspectDef.getChildAssociations().size() > 0)) {
                return true;
            }
        }
        return false;
    }

    @Override
    public void indexTransaction(Transaction info, boolean overwrite) throws IOException {
        canUpdate();
        SolrQueryRequest request = null;
        UpdateRequestProcessor processor = null;
        try {
            request = getLocalSolrQueryRequest();
            processor = this.core.getUpdateProcessingChain(null).createProcessor(request, new SolrQueryResponse());

            AddUpdateCommand cmd = new AddUpdateCommand(request);
            cmd.overwrite = overwrite;
            SolrInputDocument input = new SolrInputDocument();
            input.addField(FIELD_SOLR4_ID, AlfrescoSolrDataModel.getTransactionDocumentId(info.getId()));
            input.addField(FIELD_VERSION, 0);
            input.addField(FIELD_TXID, info.getId());
            input.addField(FIELD_INTXID, info.getId());
            input.addField(FIELD_TXCOMMITTIME, info.getCommitTimeMs());
            input.addField(FIELD_DOC_TYPE, DOC_TYPE_TX);
            cmd.solrDoc = input;
            processor.processAdd(cmd);

            putTransactionState(processor, request, info);
        } finally {
            if (processor != null) {
                processor.finish();
            }
            if (request != null) {
                request.close();
            }
        }
    }

    @Override
    public Transaction getMaxTransactionIdAndCommitTimeInIndex() {
        SolrQueryRequest request = null;
        try {
            request = getLocalSolrQueryRequest();
            SolrDocument txState = getState(request, "TRACKER!STATE!TX");
            Transaction maxTransaction = null;
            if (txState != null) {
                long id = this.getFieldValueLong(txState, FIELD_S_TXID);
                long commitTime = this.getFieldValueLong(txState, FIELD_S_TXCOMMITTIME);
                maxTransaction = new Transaction();
                maxTransaction.setId(id);
                maxTransaction.setCommitTimeMs(commitTime);
            } else {
                maxTransaction = new Transaction();
            }
            return maxTransaction;
        } finally {
            if (request != null) {
                request.close();
            }
        }
    }

    /**
     * Puts the latest transaction state onto the index
     * @param processor UpdateRequestProcessor
     * @param request SolrQueryRequest
     * @param tx Transaction
     * @throws IOException
     */
    public void putTransactionState(UpdateRequestProcessor processor, SolrQueryRequest request, Transaction tx)
            throws IOException {
        String version;
        SolrDocument txState = getState(request, "TRACKER!STATE!TX");
        if (txState != null) {
            long txCommitTime = this.getFieldValueLong(txState, FIELD_S_TXCOMMITTIME);
            long txId = this.getFieldValueLong(txState, FIELD_S_TXID);
            // Transactions are ordered by commit time and tie-broken by tx id
            if (tx.getCommitTimeMs() > txCommitTime || tx.getCommitTimeMs() == txCommitTime && tx.getId() > txId) {
                // Uses optimistic concurrency 
                version = this.getFieldValueString(txState, FIELD_VERSION);
            } else {
                version = null; // Should not update in this case
            }
        } else {
            version = "0";
        }

        if (version != null) {
            AddUpdateCommand cmd = new AddUpdateCommand(request);
            cmd.overwrite = true;
            SolrInputDocument input = new SolrInputDocument();
            input.addField(FIELD_SOLR4_ID, "TRACKER!STATE!TX");
            input.addField(FIELD_VERSION, version);
            input.addField(FIELD_S_TXID, tx.getId());
            input.addField(FIELD_S_INTXID, tx.getId());
            input.addField(FIELD_S_TXCOMMITTIME, tx.getCommitTimeMs());
            input.addField(FIELD_DOC_TYPE, DOC_TYPE_STATE);
            cmd.solrDoc = input;
            processor.processAdd(cmd);
        }
    }

    /**
     * @param id String
     */
    @Override
    public boolean isInIndex(String id) throws IOException {
        Boolean found = isIdIndexCache.get(id);
        if (found != null) {
            return found;
        } else {
            boolean isInIndex = isInIndexImpl(id);
            if (isInIndex) {
                isIdIndexCache.put(id, Boolean.TRUE);
            }
            return isInIndex;
        }
    }

    private boolean isInIndexImpl(String ids) throws IOException {
        SolrQueryRequest request = null;
        try {
            request = getLocalSolrQueryRequest();

            SolrRequestHandler handler = core.getRequestHandler(REQUEST_HANDLER_GET);
            SolrQueryResponse rsp = new SolrQueryResponse();

            ModifiableSolrParams newParams = new ModifiableSolrParams(request.getParams());
            newParams.set("ids", ids);
            request.setParams(newParams);

            handler.handleRequest(request, rsp);

            @SuppressWarnings("rawtypes")
            NamedList values = rsp.getValues();
            SolrDocumentList response = (SolrDocumentList) values.get(RESPONSE_DEFAULT_IDS);
            return response.getNumFound() > 0;
        } finally {
            if (request != null) {
                request.close();
            }
        }
    }

    @Override
    public boolean putModel(M2Model model) {
        return this.dataModel.putModel(model);
    }

    @Override
    public void rollback() throws IOException {
        commitAndRollbackLock.writeLock().lock();
        try {
            activeTrackerThreadsLock.writeLock().lock();
            try {
                activeTrackerThreads.clear();

                SolrQueryRequest request = null;
                UpdateRequestProcessor processor = null;
                try {
                    request = getLocalSolrQueryRequest();
                    processor = this.core.getUpdateProcessingChain(null).createProcessor(request,
                            new SolrQueryResponse());
                    processor.processRollback(new RollbackUpdateCommand(request));
                } finally {
                    if (processor != null) {
                        processor.finish();
                    }
                    if (request != null) {
                        request.close();
                    }
                }
            } finally {
                activeTrackerThreadsLock.writeLock().unlock();
            }

        } finally {
            commitAndRollbackLock.writeLock().unlock();
        }

    }

    private void canUpdate() {
        activeTrackerThreadsLock.readLock().lock();
        try {
            if (activeTrackerThreads.contains(Thread.currentThread().getId())) {
                return;
            } else {
                throw new TrackerStateException("The trackers work was rolled back by another tracker error");
            }
        } finally {
            activeTrackerThreadsLock.readLock().unlock();
        }

    }

    /* (non-Javadoc)
     * @see org.alfresco.solr.InformationServer#registerTrackerThread()
     */
    @Override
    public void registerTrackerThread() {
        activeTrackerThreadsLock.writeLock().lock();
        try {
            activeTrackerThreads.add(Thread.currentThread().getId());
        } finally {
            activeTrackerThreadsLock.writeLock().unlock();
        }

    }

    /* (non-Javadoc)
     * @see org.alfresco.solr.InformationServer#unregisterTrackerThread()
     */
    @Override
    public void unregisterTrackerThread() {
        activeTrackerThreadsLock.writeLock().lock();
        try {
            activeTrackerThreads.remove(Thread.currentThread().getId());
        } finally {
            activeTrackerThreadsLock.writeLock().unlock();
        }
    }

    /* (non-Javadoc)
     * @see org.alfresco.solr.InformationServer#reindexNodeByQuery(java.lang.String)
     */
    @Override
    public void reindexNodeByQuery(String query) throws IOException, AuthenticationException, JSONException {
        SolrQueryRequest request = null;
        RefCounted<SolrIndexSearcher> refCounted = null;
        try {
            refCounted = core.getSearcher(false, true, null);
            SolrIndexSearcher solrIndexSearcher = refCounted.get();

            request = getLocalSolrQueryRequest();

            NumericDocValues dbidDocValues = solrIndexSearcher.getAtomicReader()
                    .getNumericDocValues(QueryConstants.FIELD_DBID);

            ArrayList<Node> batch = new ArrayList<Node>(200);
            DocList docList = cloud.getDocList(nativeRequestHandler, request,
                    query.startsWith("{") ? query : "{!afts}" + query);
            for (DocIterator it = docList.iterator(); it.hasNext(); /**/) {
                int docID = it.nextDoc();
                // Obtain the ACL ID for this ACL doc.
                long dbid = dbidDocValues.get(docID);

                Node node = new Node();
                node.setId(dbid);
                node.setStatus(SolrApiNodeStatus.UNKNOWN);
                node.setTxnId(Long.MAX_VALUE);

                batch.add(node);

                if (batch.size() >= 200) {
                    indexNodes(batch, true);
                    batch.clear();
                }
            }
            if (batch.size() > 0) {
                indexNodes(batch, true);
                batch.clear();
            }
        } finally {
            if (request != null) {
                request.close();
            }

            if (refCounted != null) {
                refCounted.decref();
            }
        }

    }

    /* (non-Javadoc)
     * @see org.alfresco.solr.InformationServer#getUrl()
     */
    @Override
    public String getBaseUrl() {
        return baseUrl;
    }

    /* (non-Javadoc)
     * @see org.alfresco.solr.InformationServer#getPort()
     */
    @Override
    public int getPort() {
        return port;
    }

    /* (non-Javadoc)
     * @see org.alfresco.solr.InformationServer#getHostName()
     */
    @Override
    public String getHostName() {
        return hostName;
    }

    private String getHttpPort(String defaultPort) {
        try {
            MBeanServer mBeanServer = MBeanServerFactory.findMBeanServer(null).get(0);
            QueryExp query = Query.and(Query.eq(Query.attr("scheme"), Query.value("http")),
                    Query.eq(Query.attr("protocol"), Query.value("HTTP/1.1")));
            Set<ObjectName> objectNames = mBeanServer.queryNames(null, query);

            if (objectNames != null && objectNames.size() > 0) {
                for (ObjectName objectName : objectNames) {
                    String name = objectName.toString();
                    if (name.indexOf("port=") > -1) {
                        String[] parts = name.split("port=");
                        String port = parts[1];
                        try {
                            Integer.parseInt(port);
                            return port;
                        } catch (NumberFormatException e) {
                            log.error("Error parsing http port:" + port);
                            return defaultPort;
                        }
                    }
                }
            }
        } catch (Throwable t) {
            log.error("Error getting https port:", t);
        }

        return defaultPort;
    }
}