org.opencms.search.CmsSearchIndex.java Source code

Java tutorial

Introduction

Here is the source code for org.opencms.search.CmsSearchIndex.java

Source

/*
 * This library is part of OpenCms -
 * the Open Source Content Management System
 *
 * Copyright (c) Alkacon Software GmbH (http://www.alkacon.com)
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * Lesser General Public License for more details.
 *
 * For further information about Alkacon Software GmbH, please see the
 * company website: http://www.alkacon.com
 *
 * For further information about OpenCms, please see the
 * project website: http://www.opencms.org
 * 
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */

package org.opencms.search;

import org.opencms.configuration.CmsParameterConfiguration;
import org.opencms.configuration.I_CmsConfigurationParameterHandler;
import org.opencms.file.CmsObject;
import org.opencms.file.CmsPropertyDefinition;
import org.opencms.file.CmsResource;
import org.opencms.file.CmsResourceFilter;
import org.opencms.i18n.CmsLocaleManager;
import org.opencms.i18n.CmsMessageContainer;
import org.opencms.main.CmsException;
import org.opencms.main.CmsIllegalArgumentException;
import org.opencms.main.CmsLog;
import org.opencms.main.OpenCms;
import org.opencms.report.I_CmsReport;
import org.opencms.search.documents.I_CmsDocumentFactory;
import org.opencms.search.documents.I_CmsTermHighlighter;
import org.opencms.search.fields.CmsSearchField;
import org.opencms.search.fields.CmsSearchFieldConfiguration;
import org.opencms.util.CmsFileUtil;
import org.opencms.util.CmsStringUtil;

import java.io.File;
import java.io.IOException;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;

import org.apache.commons.logging.Log;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.DateTools;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.FieldSelector;
import org.apache.lucene.document.FieldSelectorResult;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.FilterIndexReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.LogByteSizeMergePolicy;
import org.apache.lucene.index.LogMergePolicy;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanFilter;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.CachingWrapperFilter;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.FilterClause;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TermsFilter;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;

/**
 * Implements the search within an index and the management of the index configuration.<p>
 * 
 * @since 6.0.0 
 */
public class CmsSearchIndex implements I_CmsConfigurationParameterHandler {

    /**
     * Lucene filter index reader implementation that will ensure the OpenCms default search index fields
     * {@link CmsSearchField#FIELD_CONTENT} and {@link CmsSearchField#FIELD_CONTENT_BLOB}
     * are lazy loaded.<p>
     * 
     * This is to optimize performance - these 2 fields will be rather large especially for extracted
     * binary documents like PDF, MS Office etc. By using lazy fields the data is only read when it is 
     * actually used.<p>
     */
    protected class LazyContentReader extends FilterIndexReader {

        /** The initial index reader. */
        private IndexReader m_reader;

        /**
         * Create a new lazy content reader.<p>
         * 
         * @param indexReader the index reader to use this lazy content reader with
         */
        public LazyContentReader(IndexReader indexReader) {

            super(indexReader);
            m_reader = indexReader;
        }

        /**
         * @see org.apache.lucene.index.IndexReader#document(int)
         */
        @Override
        public Document document(int n) throws CorruptIndexException, IOException {

            return super.document(n, CONTENT_SELECTOR);
        }

        /**
         * @see org.apache.lucene.index.IndexReader#reopen()
         * 
         * @deprecated since Lucene 3.5 but kept for backward compatibility
         */
        @Override
        @Deprecated
        public synchronized IndexReader reopen() throws CorruptIndexException, IOException {

            return m_reader.reopen();
        }

        /**
         * @see org.apache.lucene.index.FilterIndexReader#doClose()
         */
        @Override
        protected void doClose() throws IOException {

            super.doClose();
            m_reader.close();
        }

        /**
         * @see org.apache.lucene.index.IndexReader#doOpenIfChanged()
         */
        @Override
        protected IndexReader doOpenIfChanged() throws CorruptIndexException, IOException {

            IndexReader result = IndexReader.openIfChanged(m_reader);
            if (result != null) {
                result = new LazyContentReader(result);
            }
            return result;
        }

        /**
         * @see org.apache.lucene.index.IndexReader#doOpenIfChanged(boolean)
         */
        @Override
        protected IndexReader doOpenIfChanged(boolean openReadOnly) throws CorruptIndexException, IOException {

            IndexReader result = IndexReader.openIfChanged(m_reader, openReadOnly);
            if (result != null) {
                result = new LazyContentReader(result);
            }
            return result;
        }
    }

    /** Constant for additional parameter to enable optimized full index regeneration (default: false). */
    public static final String BACKUP_REINDEXING = CmsSearchIndex.class.getName() + ".useBackupReindexing";

    /** Look table to quickly zero-pad days / months in date Strings. */
    public static final String[] DATES = new String[] { "00", "01", "02", "03", "04", "05", "06", "07", "08", "09",
            "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26",
            "27", "28", "29", "30", "31" };

    /** Constant for a field list that contains the "meta" field as well as the "content" field. */
    public static final String[] DOC_META_FIELDS = new String[] { CmsSearchField.FIELD_META,
            CmsSearchField.FIELD_CONTENT };

    /** Constant for additional parameter to enable excerpt creation (default: true). */
    public static final String EXCERPT = CmsSearchIndex.class.getName() + ".createExcerpt";

    /** Constant for additional parameter for index content extraction. */
    public static final String EXTRACT_CONTENT = CmsSearchIndex.class.getName() + ".extractContent";

    /** Constant for additional parameter for the Lucene index setting. */
    public static final String LUCENE_AUTO_COMMIT = "lucene.AutoCommit";

    /** Constant for additional parameter for the Lucene index setting. */
    public static final String LUCENE_MAX_MERGE_DOCS = "lucene.MaxMergeDocs";

    /** Constant for additional parameter for the Lucene index setting. */
    public static final String LUCENE_MERGE_FACTOR = "lucene.MergeFactor";

    /** Constant for additional parameter for the Lucene index setting. */
    public static final String LUCENE_RAM_BUFFER_SIZE_MB = "lucene.RAMBufferSizeMB";

    /** Constant for additional parameter for the Lucene index setting. */
    public static final String LUCENE_USE_COMPOUND_FILE = "lucene.UseCompoundFile";

    /** The Lucene Version used to create Query parsers and such. */
    public static final Version LUCENE_VERSION = Version.LUCENE_35;

    /** Constant for additional parameter for controlling how many hits are loaded at maximum (default: 1000). */
    public static final String MAX_HITS = CmsSearchIndex.class.getName() + ".maxHits";

    /** Indicates how many hits are loaded at maximum by default. */
    public static final int MAX_HITS_DEFAULT = 5000;

    /** Constant for years max range span in document search. */
    public static final int MAX_YEAR_RANGE = 12;

    /** Constant for additional parameter to enable permission checks (default: true). */
    public static final String PERMISSIONS = CmsSearchIndex.class.getName() + ".checkPermissions";

    /** Constant for additional parameter to set the thread priority during search. */
    public static final String PRIORITY = CmsSearchIndex.class.getName() + ".priority";

    /** Special value for the search.exclude property. */
    public static final String PROPERTY_SEARCH_EXCLUDE_VALUE_ALL = "all";

    /** Special value for the search.exclude property. */
    public static final String PROPERTY_SEARCH_EXCLUDE_VALUE_GALLERY = "gallery";

    /** Automatic ("auto") index rebuild mode. */
    public static final String REBUILD_MODE_AUTO = "auto";

    /** Manual ("manual") index rebuild mode. */
    public static final String REBUILD_MODE_MANUAL = "manual";

    /** Offline ("offline") index rebuild mode. */
    public static final String REBUILD_MODE_OFFLINE = "offline";

    /** Constant for additional parameter to enable time range checks (default: true). */
    public static final String TIME_RANGE = CmsSearchIndex.class.getName() + ".checkTimeRange";

    /** The use all locale. */
    public static final String USE_ALL_LOCALE = "all";

    /**
     * Field selector for Lucene that that will ensure the OpenCms default search index fields
     * {@link CmsSearchField#FIELD_CONTENT} and {@link CmsSearchField#FIELD_CONTENT_BLOB}
     * are lazy loaded.<p>
     * 
     * This is to optimize performance - these 2 fields will be rather large especially for extracted
     * binary documents like PDF, MS Office etc. By using lazy fields the data is only read when it is 
     * actually used.<p>
     */
    protected static final FieldSelector CONTENT_SELECTOR = new FieldSelector() {

        /** Required for safe serialization. */
        private static final long serialVersionUID = 2785064181424297998L;

        /**
         * Makes the content fields lazy.<p>
         * 
         * @see org.apache.lucene.document.FieldSelector#accept(java.lang.String)
         */
        public FieldSelectorResult accept(String fieldName) {

            if (CmsSearchField.FIELD_CONTENT.equals(fieldName)
                    || CmsSearchField.FIELD_CONTENT_BLOB.equals(fieldName)) {
                return FieldSelectorResult.LAZY_LOAD;
            }
            return FieldSelectorResult.LOAD;
        }
    };

    /** The log object for this class. */
    private static final Log LOG = CmsLog.getLog(CmsSearchIndex.class);

    /** Controls if a resource requires view permission to be displayed in the result list. */
    protected boolean m_requireViewPermission;

    /** The list of configured index sources. */
    protected List<CmsSearchIndexSource> m_sources;

    /** The configured Lucene analyzer used for this index. */
    private Analyzer m_analyzer;

    /** Indicates if backup re-indexing is used by this index. */
    private boolean m_backupReindexing;

    /** The permission check mode for this index. */
    private boolean m_checkPermissions;

    /** The time range check mode for this index. */
    private boolean m_checkTimeRange;

    /** The excerpt mode for this index. */
    private boolean m_createExcerpt;

    /** Map of display query filters to use. */
    private Map<String, Filter> m_displayFilters;

    /** Document types of folders/channels. */
    private Map<String, List<String>> m_documenttypes;

    /** An internal enabled flag, used to disable the index if for instance the configured project does not exist. */
    private boolean m_enabled;

    /** The content extraction mode for this index. */
    private boolean m_extractContent;

    /** The search field configuration of this index. */
    private CmsSearchFieldConfiguration m_fieldConfiguration;

    /** The name of the search field configuration used by this index. */
    private String m_fieldConfigurationName;

    /** The Lucene index searcher to use. */
    private IndexSearcher m_indexSearcher;

    /** The Lucene index writer to use. */
    private I_CmsIndexWriter m_indexWriter;

    /** The locale of this index. */
    private Locale m_locale;

    /** The Lucene index merge factor setting, see {@link IndexWriter#setMaxMergeDocs(int)}. */
    private Integer m_luceneMaxMergeDocs;

    /** The Lucene index merge factor setting, see {@link IndexWriter#setMergeFactor(int)}. */
    private Integer m_luceneMergeFactor;

    /** The Lucene index RAM buffer size, see {@link IndexWriter#setRAMBufferSizeMB(double)}. */
    private Double m_luceneRAMBufferSizeMB;

    /** The Lucene index setting that controls, see {@link IndexWriter#setUseCompoundFile(boolean)}.  */
    private Boolean m_luceneUseCompoundFile;

    /** Indicates how many hits are loaded at maximum. */
    private int m_maxHits;

    /** The name of this index. */
    private String m_name;

    /** The path where this index stores it's data in the "real" file system. */
    private String m_path;

    /** The thread priority for a search. */
    private int m_priority;

    /** The project of this index. */
    private String m_project;

    /** The rebuild mode for this index. */
    private String m_rebuild;

    /** The configured sources for this index. */
    private List<String> m_sourceNames;

    /**
     * Default constructor only intended to be used by the XML configuration. <p>
     * 
     * It is recommended to use the constructor <code>{@link #CmsSearchIndex(String)}</code> 
     * as it enforces the mandatory name argument. <p>
     * 
     */
    public CmsSearchIndex() {

        m_sourceNames = new ArrayList<String>();
        m_documenttypes = new HashMap<String, List<String>>();
        m_createExcerpt = true;
        m_extractContent = true;
        m_checkTimeRange = false;
        m_checkPermissions = true;
        m_enabled = true;
        m_priority = -1;
        m_maxHits = MAX_HITS_DEFAULT;
    }

    /**
     * Creates a new CmsSearchIndex with the given name.<p>
     * 
     * @param name the system-wide unique name for the search index 
     * 
     * @throws CmsIllegalArgumentException if the given name is null, empty or already taken by another search index 
     * 
     */
    public CmsSearchIndex(String name) throws CmsIllegalArgumentException {

        this();
        setName(name);
    }

    /**
     * Generates a list of date terms for the optimized date range search with "daily" granularity level.<p> 
     * 
     * How this works:<ul>
     * <li>For each document, terms are added for the year, the month and the day the document
     * was modified or created) in. So for example if a document is modified at February 02, 2009, 
     * then the following terms are stored for this document:
     * "20090202", "200902" and "2009".</li>
     * <li>In case a date range search is done, then all possible matches for the
     * provided rage are created as search terms and matched with the document terms.</li>
     * <li>Consider the following use case: You want to find out if a resource has been changed
     * in the time between November 29, 2007 and March 01, 2009.
     * One term to match is simply "2008" because if a document 
     * was modified in 2008, then it is clearly in the date range.
     * Other terms are "200712", "200901" and "200902", because all documents 
     * modified in these months are also a certain matches.
     * Finally we need to add terms for "20071129", "20071130" and "20090301" to match the days in the 
     * starting and final month.</li>
     * </ul>
     * 
     * @param startDate start date of the range to search in
     * @param endDate end date of the range to search in
     * 
     * @return a list of date terms for the optimized date range search
     */
    public static List<String> getDateRangeSpan(long startDate, long endDate) {

        if (startDate > endDate) {
            // switch so that the end is always before the start
            long temp = endDate;
            endDate = startDate;
            startDate = temp;
        }

        List<String> result = new ArrayList<String>(100);

        // initialize calendars from the time value
        Calendar calStart = Calendar.getInstance(OpenCms.getLocaleManager().getTimeZone());
        Calendar calEnd = Calendar.getInstance(calStart.getTimeZone());
        calStart.setTimeInMillis(startDate);
        calEnd.setTimeInMillis(endDate);

        // get the required info to build the date range from the calendars
        int startDay = calStart.get(Calendar.DAY_OF_MONTH);
        int endDay = calEnd.get(Calendar.DAY_OF_MONTH);
        int maxDayInStartMonth = calStart.getActualMaximum(Calendar.DAY_OF_MONTH);
        int startMonth = calStart.get(Calendar.MONTH) + 1;
        int endMonth = calEnd.get(Calendar.MONTH) + 1;
        int startYear = calStart.get(Calendar.YEAR);
        int endYear = calEnd.get(Calendar.YEAR);

        // first add all full years in the date range
        result.addAll(getYearSpan(startYear + 1, endYear - 1));

        if (startYear != endYear) {
            // different year, different month
            result.addAll(getMonthSpan(startMonth + 1, 12, startYear));
            result.addAll(getMonthSpan(1, endMonth - 1, endYear));
            result.addAll(getDaySpan(startDay, maxDayInStartMonth, startMonth, startYear));
            result.addAll(getDaySpan(1, endDay, endMonth, endYear));
        } else {
            if (startMonth != endMonth) {
                // same year, different month
                result.addAll(getMonthSpan(startMonth + 1, endMonth - 1, startYear));
                result.addAll(getDaySpan(startDay, maxDayInStartMonth, startMonth, startYear));
                result.addAll(getDaySpan(1, endDay, endMonth, endYear));
            } else {
                // same year, same month
                result.addAll(getDaySpan(startDay, endDay, endMonth, endYear));
            }
        }

        // sort the result, makes the range better readable in the debugger
        Collections.sort(result);
        return result;
    }

    /**
     * Calculate a span of days in the given year and month for the optimized date range search.<p>
     *  
     * The result will contain dates formatted like "yyyyMMDD", for example "20080131".<p> 
     *  
     * @param startDay the start day
     * @param endDay the end day
     * @param month the month
     * @param year the year
     * 
     * @return a span of days in the given year and month for the optimized date range search
     */
    private static List<String> getDaySpan(int startDay, int endDay, int month, int year) {

        List<String> result = new ArrayList<String>();
        String yearMonthStr = String.valueOf(year) + DATES[month];
        for (int i = startDay; i <= endDay; i++) {
            String dateStr = yearMonthStr + DATES[i];
            result.add(dateStr);
        }
        return result;
    }

    /**
     * Calculate a span of months in the given year for the optimized date range search.<p>
     *  
     * The result will contain dates formatted like "yyyyMM", for example "200801".<p> 
     *  
     * @param startMonth the start month
     * @param endMonth the end month
     * @param year the year
     * 
     * @return a span of months in the given year for the optimized date range search
     */
    private static List<String> getMonthSpan(int startMonth, int endMonth, int year) {

        List<String> result = new ArrayList<String>();
        String yearStr = String.valueOf(year);
        for (int i = startMonth; i <= endMonth; i++) {
            String dateStr = yearStr + DATES[i];
            result.add(dateStr);
        }
        return result;
    }

    /**
     * Calculate a span of years for the optimized date range search.<p>
     *  
     * The result will contain dates formatted like "yyyy", for example "2008".<p> 
     *  
     * @param startYear the start year
     * @param endYear the end year
     * 
     * @return a span of years for the optimized date range search
     */
    private static List<String> getYearSpan(int startYear, int endYear) {

        List<String> result = new ArrayList<String>();
        for (int i = startYear; i <= endYear; i++) {
            String dateStr = String.valueOf(i);
            result.add(dateStr);
        }
        return result;
    }

    /**
     * Adds a parameter.<p>
     * 
     * @param key the key/name of the parameter
     * @param value the value of the parameter
     */
    public void addConfigurationParameter(String key, String value) {

        if (PERMISSIONS.equals(key)) {
            m_checkPermissions = Boolean.valueOf(value).booleanValue();
        } else if (TIME_RANGE.equals(key)) {
            m_checkTimeRange = Boolean.valueOf(value).booleanValue();
        } else if (EXCERPT.equals(key)) {
            m_createExcerpt = Boolean.valueOf(value).booleanValue();
        } else if (EXTRACT_CONTENT.equals(key)) {
            m_extractContent = Boolean.valueOf(value).booleanValue();
        } else if (BACKUP_REINDEXING.equals(key)) {
            m_backupReindexing = Boolean.valueOf(value).booleanValue();
        } else if (MAX_HITS.equals(key)) {
            try {
                m_maxHits = Integer.parseInt(value);
            } catch (NumberFormatException e) {
                LOG.error(Messages.get().getBundle().key(Messages.LOG_INVALID_PARAM_3, value, key, getName()));
            }
            if (m_maxHits < (MAX_HITS_DEFAULT / 100)) {
                m_maxHits = MAX_HITS_DEFAULT;
                LOG.error(Messages.get().getBundle().key(Messages.LOG_INVALID_PARAM_3, value, key, getName()));
            }
        } else if (PRIORITY.equals(key)) {
            m_priority = Integer.parseInt(value);
            if (m_priority < Thread.MIN_PRIORITY) {
                m_priority = Thread.MIN_PRIORITY;
                LOG.error(Messages.get().getBundle().key(Messages.LOG_SEARCH_PRIORITY_TOO_LOW_2, value,
                        new Integer(Thread.MIN_PRIORITY)));

            } else if (m_priority > Thread.MAX_PRIORITY) {
                m_priority = Thread.MAX_PRIORITY;
                LOG.debug(Messages.get().getBundle().key(Messages.LOG_SEARCH_PRIORITY_TOO_HIGH_2, value,
                        new Integer(Thread.MAX_PRIORITY)));

            }
        } else if (LUCENE_MAX_MERGE_DOCS.equals(key)) {
            try {
                m_luceneMaxMergeDocs = Integer.valueOf(value);
            } catch (NumberFormatException e) {
                LOG.error(Messages.get().getBundle().key(Messages.LOG_INVALID_PARAM_3, value, key, getName()));
            }
        } else if (LUCENE_MERGE_FACTOR.equals(key)) {
            try {
                m_luceneMergeFactor = Integer.valueOf(value);
            } catch (NumberFormatException e) {
                LOG.error(Messages.get().getBundle().key(Messages.LOG_INVALID_PARAM_3, value, key, getName()));
            }
        } else if (LUCENE_RAM_BUFFER_SIZE_MB.equals(key)) {
            try {
                m_luceneRAMBufferSizeMB = Double.valueOf(value);
            } catch (NumberFormatException e) {
                LOG.error(Messages.get().getBundle().key(Messages.LOG_INVALID_PARAM_3, value, key, getName()));
            }
        } else if (LUCENE_USE_COMPOUND_FILE.equals(key)) {
            m_luceneUseCompoundFile = Boolean.valueOf(value);
        }
    }

    /**
     * Adds am index source to this search index.<p>
     * 
     * @param sourceName the index source name to add
     */
    public void addSourceName(String sourceName) {

        m_sourceNames.add(sourceName);
    }

    /**
     * Checks is this index has been configured correctly.<p>
     * 
     * In case the check fails, the <code>enabled</code> property
     * is set to <code>false</code>
     * 
     * @param cms a OpenCms user context to perform the checks with (should have "Administrator" permissions)
     *
     * @return <code>true</code> in case the index is correctly configured and enabled after the check
     * 
     * @see #isEnabled()
     */
    public boolean checkConfiguration(CmsObject cms) {

        if (isEnabled()) {
            // check if the project for the index exists        
            try {
                cms.readProject(getProject());
                setEnabled(true);
            } catch (CmsException e) {
                // the project does not exist, disable the index
                setEnabled(false);
                if (LOG.isErrorEnabled()) {
                    LOG.error(Messages.get().getBundle().key(Messages.LOG_SEARCHINDEX_CREATE_BAD_PROJECT_2,
                            getProject(), getName()));
                }
            }
        } else {
            if (LOG.isInfoEnabled()) {
                LOG.info(Messages.get().getBundle().key(Messages.LOG_SEARCHINDEX_DISABLED_1, getName()));
            }
        }

        return isEnabled();
    }

    /**
     * @see java.lang.Object#equals(java.lang.Object)
     */
    @Override
    public boolean equals(Object obj) {

        if (obj == this) {
            return true;
        }
        if (obj instanceof CmsSearchIndex) {
            return ((CmsSearchIndex) obj).m_name.equals(m_name);
        }
        return false;
    }

    /**
     * Returns the Lucene analyzer used for this index.<p>
     *
     * @return the Lucene analyzer used for this index
     */
    public Analyzer getAnalyzer() {

        return m_analyzer;
    }

    /**
     * @see org.opencms.configuration.I_CmsConfigurationParameterHandler#getConfiguration()
     */
    public CmsParameterConfiguration getConfiguration() {

        CmsParameterConfiguration result = new CmsParameterConfiguration();
        if (getPriority() > 0) {
            result.put(PRIORITY, String.valueOf(m_priority));
        }
        if (!isCreatingExcerpt()) {
            result.put(EXCERPT, String.valueOf(m_createExcerpt));
        }
        if (!isExtractingContent()) {
            result.put(EXTRACT_CONTENT, String.valueOf(m_extractContent));
        }
        if (!isCheckingPermissions()) {
            result.put(PERMISSIONS, String.valueOf(m_checkPermissions));
        }
        // always write time range check parameter because of logic change in OpenCms 8.0
        result.put(TIME_RANGE, String.valueOf(m_checkTimeRange));
        if (isBackupReindexing()) {
            result.put(BACKUP_REINDEXING, String.valueOf(m_backupReindexing));
        }
        if (getMaxHits() != MAX_HITS_DEFAULT) {
            result.put(MAX_HITS, String.valueOf(getMaxHits()));
        }
        // set the index writer parameter if required 
        if (m_luceneMaxMergeDocs != null) {
            result.put(LUCENE_MAX_MERGE_DOCS, String.valueOf(m_luceneMaxMergeDocs));
        }
        if (m_luceneMergeFactor != null) {
            result.put(LUCENE_MERGE_FACTOR, String.valueOf(m_luceneMergeFactor));
        }
        if (m_luceneRAMBufferSizeMB != null) {
            result.put(LUCENE_RAM_BUFFER_SIZE_MB, String.valueOf(m_luceneRAMBufferSizeMB));
        }
        if (m_luceneUseCompoundFile != null) {
            result.put(LUCENE_USE_COMPOUND_FILE, String.valueOf(m_luceneUseCompoundFile));
        }
        return result;
    }

    /**
     * Returns the Lucene document with the given root path from the index.<p>
     * 
     * @param rootPath the root path of the document to get 
     * 
     * @return the Lucene document with the given root path from the index
     * 
     * @deprecated Use {@link #getDocument(String, String)} instead and provide {@link CmsSearchField#FIELD_PATH} as field to search in
     */
    @Deprecated
    public Document getDocument(String rootPath) {

        return getDocument(CmsSearchField.FIELD_PATH, rootPath);
    }

    /**
     * Returns the first Lucene document where the given term matches the selected index field.<p>
     * 
     * Use this method to search for documents which have unique field values, like a unique id.<p>
     * 
     * @param field the field to search in
     * @param term the term to search for
     * 
     * @return the first Lucene document where the given term matches the selected index field
     */
    public synchronized Document getDocument(String field, String term) {

        Document result = null;
        IndexSearcher searcher = getSearcher();
        if (searcher != null) {
            // search for an exact match on the selected field
            Term resultTerm = new Term(field, term);
            try {
                TopDocs hits = searcher.search(new TermQuery(resultTerm), 1);
                if (hits.scoreDocs.length > 0) {
                    result = searcher.doc(hits.scoreDocs[0].doc);
                }
            } catch (IOException e) {
                // ignore, return null and assume document was not found
            }
        }
        return result;
    }

    /**
     * Returns the document type factory used for the given resource in this index, or <code>null</code>  
     * in case the resource is not indexed by this index.<p>
     * 
     * A resource is indexed if the following is all true: <ol>
     * <li>The index contains at last one index source matching the root path of the given resource.
     * <li>For this matching index source, the document type factory needed by the resource is also configured.
     * </ol>
     * 
     * @param res the resource to check
     * 
     * @return he document type factory used for the given resource in this index, or <code>null</code>  
     * in case the resource is not indexed by this index
     */
    public I_CmsDocumentFactory getDocumentFactory(CmsResource res) {

        if ((res != null) && (m_sources != null)) {
            // the result can only be null or the type configured for the resource
            I_CmsDocumentFactory result = OpenCms.getSearchManager().getDocumentFactory(res);
            if (result != null) {
                // check the path of the resource if it matches with one (or more) of the configured index sources
                Iterator<CmsSearchIndexSource> i = m_sources.iterator();
                while (i.hasNext()) {
                    CmsSearchIndexSource source = i.next();
                    if (source.isIndexing(res.getRootPath(), result.getName())) {
                        // we found an index source that indexes the resource
                        return result;
                    }
                }
            }
        }
        return null;
    }

    /**
     * Returns the search field configuration of this index.<p>
     * 
     * @return the search field configuration of this index
     */
    public CmsSearchFieldConfiguration getFieldConfiguration() {

        return m_fieldConfiguration;
    }

    /**
     * Returns the name of the field configuration used for this index.<p>
     * 
     * @return the name of the field configuration used for this index
     */
    public String getFieldConfigurationName() {

        return m_fieldConfigurationName;
    }

    /**
     * Returns a new index writer for this index.<p>
     * 
     * @param report the report to write error messages on
     * @param create if <code>true</code> a whole new index is created, if <code>false</code> an existing index is updated
     * 
     * @return a new instance of IndexWriter
     * @throws CmsIndexException if the index can not be opened
     */
    public I_CmsIndexWriter getIndexWriter(I_CmsReport report, boolean create) throws CmsIndexException {

        // note - create will be: 
        //   true if the index is to be fully rebuild, 
        //   false if the index is to be incrementally updated

        if (m_indexWriter != null) {
            if (!create) {
                /// re-use existing index writer
                return m_indexWriter;
            } else {
                // need to close the index writer if create is "true"
                try {
                    m_indexWriter.close();
                    m_indexWriter = null;
                } catch (IOException e) {
                    // if we can't close the index we are busted!
                    throw new CmsIndexException(
                            Messages.get().container(Messages.LOG_IO_INDEX_WRITER_CLOSE_2, getPath(), getName()),
                            e);
                }
            }
        }

        indexWriterUnlock(report);
        // now create is true of false, but the index writer is definitely null / closed
        I_CmsIndexWriter indexWriter = indexWriterCreate(create);

        if (!create) {
            // if not create we re-use the index writer for the next incremental updates
            m_indexWriter = indexWriter;
        }

        return indexWriter;
    }

    /**
     * Returns the language locale of this index.<p>
     * 
     * @return the language locale of this index, for example "en"
     */
    public Locale getLocale() {

        return m_locale;
    }

    /**
     * Returns the language locale for the given resource in this index.<p>
     * 
     * @param cms the current OpenCms user context
     * @param resource the resource to check
     * @param availableLocales a list of locales supported by the resource
     * 
     * @return the language locale for the given resource in this index
     */
    public Locale getLocaleForResource(CmsObject cms, CmsResource resource, List<Locale> availableLocales) {

        Locale result;
        List<Locale> defaultLocales = OpenCms.getLocaleManager().getDefaultLocales(cms, resource);
        List<Locale> locales = availableLocales;
        if ((locales == null) || (locales.size() == 0)) {
            locales = defaultLocales;
        }
        result = OpenCms.getLocaleManager().getBestMatchingLocale(getLocale(), defaultLocales, locales);
        return result;
    }

    /**
    * Returns the language locale of the index as a String.<p>
    * 
    * @return the language locale of the index as a String
    * 
    * @see #getLocale()
    */
    public String getLocaleString() {

        return getLocale().toString();
    }

    /**
     * Indicates the number of how many hits are loaded at maximum.<p> 
     * 
     * Since Lucene 2.4, the number of maximum documents to load from the index
     * must be specified. The default of this setting is {@link CmsSearchIndex#MAX_HITS_DEFAULT} (5000).
     * This means that at maximum 5000 results are returned from the index.
     * Please note that this number may be reduced further because of OpenCms read permissions 
     * or per-user file visibility settings not controlled in the index.<p>
     * 
     * @return the number of how many hits are loaded at maximum
     * 
     * @since 7.5.1
     */
    public int getMaxHits() {

        return m_maxHits;
    }

    /**
     * Gets the name of this index.<p>
     * 
     * @return the name of the index
     */
    public String getName() {

        return m_name;
    }

    /**
     * Returns the path where this index stores it's data in the "real" file system.<p>
     * 
     * @return the path where this index stores it's data in the "real" file system
     */
    public String getPath() {

        return m_path;
    }

    /**
     * Returns the Thread priority for this search index.<p>
     *
     * @return the Thread priority for this search index
     */
    public int getPriority() {

        return m_priority;
    }

    /**
     * Gets the project of this index.<p>
     * 
     * @return the project of the index, i.e. "online"
     */
    public String getProject() {

        return m_project;
    }

    /**
     * Get the rebuild mode of this index.<p>
     * 
     * @return the current rebuild mode
     */
    public String getRebuildMode() {

        return m_rebuild;
    }

    /**
     * Returns the Lucene index searcher used for this search index.<p>
     *
     * @return the Lucene index searcher used for this search index
     */
    public IndexSearcher getSearcher() {

        return m_indexSearcher;
    }

    /**
     * Returns all configured sources names of this search index.<p>
     * 
     * @return a list with all configured sources names of this search index
     */
    public List<String> getSourceNames() {

        return m_sourceNames;
    }

    /**
     * Returns all configured index sources of this search index.<p>
     * 
     * @return all configured index sources of this search index
     */
    public List<CmsSearchIndexSource> getSources() {

        return m_sources;
    }

    /**
     * @see java.lang.Object#hashCode()
     */
    @Override
    public int hashCode() {

        return m_name != null ? m_name.hashCode() : 0;
    }

    /**
     * @see org.opencms.configuration.I_CmsConfigurationParameterHandler#initConfiguration()
     */
    public void initConfiguration() {

        // noting to do here
    }

    /**
     * Initializes the search index.<p>
     * 
     * @throws CmsSearchException if the index source association failed
     */
    public void initialize() throws CmsSearchException {

        if (!isEnabled()) {
            // index is disabled, no initialization is required
            return;
        }

        String sourceName = null;
        CmsSearchIndexSource indexSource = null;
        List<String> searchIndexSourceDocumentTypes = null;
        List<String> resourceNames = null;
        String resourceName = null;
        m_sources = new ArrayList<CmsSearchIndexSource>();

        m_path = OpenCms.getSystemInfo()
                .getAbsoluteRfsPathRelativeToWebInf(OpenCms.getSearchManager().getDirectory() + "/" + m_name);

        for (int i = 0, n = m_sourceNames.size(); i < n; i++) {

            try {
                sourceName = m_sourceNames.get(i);
                indexSource = OpenCms.getSearchManager().getIndexSource(sourceName);
                m_sources.add(indexSource);

                resourceNames = indexSource.getResourcesNames();
                searchIndexSourceDocumentTypes = indexSource.getDocumentTypes();
                for (int j = 0, m = resourceNames.size(); j < m; j++) {

                    resourceName = resourceNames.get(j);
                    m_documenttypes.put(resourceName, searchIndexSourceDocumentTypes);
                }
            } catch (Exception e) {
                // mark this index as disabled
                setEnabled(false);
                throw new CmsSearchException(
                        Messages.get().container(Messages.ERR_INDEX_SOURCE_ASSOCIATION_1, sourceName), e);
            }
        }

        // initialize the search field configuration
        if (m_fieldConfigurationName == null) {
            // if not set, use standard field configuration
            m_fieldConfigurationName = CmsSearchFieldConfiguration.STR_STANDARD;
        }
        m_fieldConfiguration = OpenCms.getSearchManager().getFieldConfiguration(m_fieldConfigurationName);
        if (m_fieldConfiguration == null) {
            // we must have a valid field configuration to continue
            throw new CmsSearchException(Messages.get().container(Messages.ERR_FIELD_CONFIGURATION_UNKNOWN_2,
                    m_name, m_fieldConfigurationName));
        }

        // get the configured analyzer and apply the the field configuration analyzer wrapper
        Analyzer baseAnalyzer = OpenCms.getSearchManager().getAnalyzer(getLocale());
        setAnalyzer(m_fieldConfiguration.getAnalyzer(baseAnalyzer));

        // initialize the index searcher instance
        indexSearcherOpen(m_path);
    }

    /**
     * Returns <code>true</code> if backup re-indexing is done by this index.<p>
     *
     * This is an optimization method by which the old extracted content is 
     * reused in order to save performance when re-indexing.<p>
     *
     * @return  <code>true</code> if backup re-indexing is done by this index
     * 
     * @since 7.5.1
     */
    public boolean isBackupReindexing() {

        return m_backupReindexing;
    }

    /**
     * Returns <code>true</code> if permissions are checked for search results by this index.<p>
     *
     * If permission checks are not required, they can be turned off in the index search configuration parameters
     * in <code>opencms-search.xml</code>. Not checking permissions will improve performance.<p> 
     * 
     * This is can be of use in scenarios when you know that all search results are always readable,
     * which is usually true for public websites that do not have personalized accounts.<p>
     * 
     * Please note that even if a result is returned where the current user has no read permissions, 
     * the user can not actually access this document. It will only appear in the search result list,
     * but if the user clicks the link to open the document he will get an error.<p>
     * 
     *
     * @return <code>true</code> if permissions are checked for search results by this index
     */
    public boolean isCheckingPermissions() {

        return m_checkPermissions;
    }

    /**
     * Returns <code>true</code> if the document time range is checked with a granularity level of seconds
     * for search results by this index.<p>
     * 
     * Since OpenCms 8.0, time range checks are always done if {@link CmsSearchParameters#setMinDateLastModified(long)}
     * or any of the corresponding methods are used. 
     * This is done very efficiently using optimized Lucene filers.
     * However, the granularity of these checks are done only on a daily
     * basis, which means that you can only find "changes made yesterday" but not "changes made last hour". 
     * For normal limitation of search results, a daily granularity should be enough.<p> 
     * 
     * If time range checks with a granularity level of seconds are required, 
     * they can be turned on in the index search configuration parameters
     * in <code>opencms-search.xml</code>. 
     * Not checking the time range  with a granularity level of seconds will improve performance.<p>
     * 
     * By default the granularity level of seconds is turned off since OpenCms 8.0<p> 
     * 
     * @return <code>true</code> if the document time range is checked  with a granularity level of seconds for search results by this index
     */
    public boolean isCheckingTimeRange() {

        return m_checkTimeRange;
    }

    /**
     * Returns <code>true</code> if an excerpt is generated by this index.<p>
     *
     * If no except is required, generation can be turned off in the index search configuration parameters
     * in <code>opencms-search.xml</code>. Not generating an excerpt will improve performance.<p> 
     *
     * @return <code>true</code> if an excerpt is generated by this index
     */
    public boolean isCreatingExcerpt() {

        return m_createExcerpt;
    }

    /**
     * Returns <code>true</code> if this index is currently disabled.<p>
     * 
     * @return <code>true</code> if this index is currently disabled
     */
    public boolean isEnabled() {

        return m_enabled;
    }

    /**
     * Returns <code>true</code> if full text is extracted by this index.<p>
     *
     * Full text content extraction can be turned off in the index search configuration parameters
     * in <code>opencms-search.xml</code>. 
     * Not extraction the full text information will highly improve performance.<p> 
     *
     * @return <code>true</code> if full text is extracted by this index
     */
    public boolean isExtractingContent() {

        return m_extractContent;
    }

    /**
     * Returns <code>true</code> if a resource requires read permission to be incuded in the result list.<p>
     * 
     * @return <code>true</code> if a resource requires read permission to be incuded in the result list
     */
    public boolean isRequireViewPermission() {

        return m_requireViewPermission;
    }

    /**
     * Returns <code>true</code> in case this index is updated incremental.<p>
     * 
     * An index is updated incremental if the index rebuild mode as defined by 
     * {@link #getRebuildMode()} is either set to {@value #REBUILD_MODE_AUTO} or 
     * {@value #REBUILD_MODE_OFFLINE}. Moreover, at least one update must have 
     * been written to the index already.
     * 
     * @return <code>true</code> in case this index is updated incremental
     */
    public boolean isUpdatedIncremental() {

        return m_indexWriter != null;
    }

    /**
     * Removes an index source from this search index.<p>
     * 
     * @param sourceName the index source name to remove
     */
    public void removeSourceName(String sourceName) {

        m_sourceNames.remove(sourceName);
    }

    /**
     * Performs a search on the index within the given fields.<p>
     * 
     * The result is returned as List with entries of type I_CmsSearchResult.<p>
     * @param cms the current user's Cms object
     * @param params the parameters to use for the search
     * @return the List of results found or an empty list
     * @throws CmsSearchException if something goes wrong
     */
    public synchronized CmsSearchResultList search(CmsObject cms, CmsSearchParameters params)
            throws CmsSearchException {

        long timeTotal = -System.currentTimeMillis();
        long timeLucene;
        long timeResultProcessing;

        if (LOG.isDebugEnabled()) {
            LOG.debug(Messages.get().getBundle().key(Messages.LOG_SEARCH_PARAMS_2, params, getName()));
        }

        // the hits found during the search
        TopDocs hits;

        // storage for the results found
        CmsSearchResultList searchResults = new CmsSearchResultList();

        int previousPriority = Thread.currentThread().getPriority();

        try {
            // copy the user OpenCms context
            CmsObject searchCms = OpenCms.initCmsObject(cms);

            if (getPriority() > 0) {
                // change thread priority in order to reduce search impact on overall system performance
                Thread.currentThread().setPriority(getPriority());
            }

            // change the project     
            searchCms.getRequestContext().setCurrentProject(searchCms.readProject(getProject()));

            timeLucene = -System.currentTimeMillis();

            // several search options are searched using filters
            BooleanFilter filter = new BooleanFilter();
            // append root path filter
            filter = appendPathFilter(searchCms, filter, params.getRoots());
            // append category filter
            filter = appendCategoryFilter(searchCms, filter, params.getCategories());
            // append resource type filter
            filter = appendResourceTypeFilter(searchCms, filter, params.getResourceTypes());

            // append date last modified filter
            filter = appendDateLastModifiedFilter(filter, params.getMinDateLastModified(),
                    params.getMaxDateLastModified());
            // append date created filter
            filter = appendDateCreatedFilter(filter, params.getMinDateCreated(), params.getMaxDateCreated());

            // the search query to use, will be constructed in the next lines 
            Query query = null;
            // store separate fields query for excerpt highlighting  
            Query fieldsQuery = null;

            // get an index searcher that is certainly up to date
            indexSearcherUpdate();
            IndexSearcher searcher = getSearcher();

            if (!params.isIgnoreQuery()) {
                // since OpenCms 8 the query can be empty in which case only filters are used for the result
                if (params.getParsedQuery() != null) {
                    // the query was already build, re-use it 
                    QueryParser p = new QueryParser(LUCENE_VERSION, CmsSearchField.FIELD_CONTENT, getAnalyzer());
                    fieldsQuery = p.parse(params.getParsedQuery());
                } else if (params.getFieldQueries() != null) {
                    // each field has an individual query
                    BooleanQuery mustOccur = null;
                    BooleanQuery shouldOccur = null;
                    for (CmsSearchParameters.CmsSearchFieldQuery fq : params.getFieldQueries()) {
                        // add one sub-query for each defined field
                        QueryParser p = new QueryParser(LUCENE_VERSION, fq.getFieldName(), getAnalyzer());
                        // first generate the combined keyword query
                        Query keywordQuery = null;
                        if (fq.getSearchTerms().size() == 1) {
                            // this is just a single size keyword list
                            keywordQuery = p.parse(fq.getSearchTerms().get(0));
                        } else {
                            // multiple size keyword list
                            BooleanQuery keywordListQuery = new BooleanQuery();
                            for (String keyword : fq.getSearchTerms()) {
                                keywordListQuery.add(p.parse(keyword), fq.getTermOccur());
                            }
                            keywordQuery = keywordListQuery;
                        }
                        if (BooleanClause.Occur.SHOULD.equals(fq.getOccur())) {
                            if (shouldOccur == null) {
                                shouldOccur = new BooleanQuery();
                            }
                            shouldOccur.add(keywordQuery, fq.getOccur());
                        } else {
                            if (mustOccur == null) {
                                mustOccur = new BooleanQuery();
                            }
                            mustOccur.add(keywordQuery, fq.getOccur());
                        }
                    }
                    BooleanQuery booleanFieldsQuery = new BooleanQuery();
                    if (mustOccur != null) {
                        booleanFieldsQuery.add(mustOccur, BooleanClause.Occur.MUST);
                    }
                    if (shouldOccur != null) {
                        booleanFieldsQuery.add(shouldOccur, BooleanClause.Occur.MUST);
                    }
                    fieldsQuery = searcher.rewrite(booleanFieldsQuery);
                } else if ((params.getFields() != null) && (params.getFields().size() > 0)) {
                    // no individual field queries have been defined, so use one query for all fields 
                    BooleanQuery booleanFieldsQuery = new BooleanQuery();
                    // this is a "regular" query over one or more fields
                    // add one sub-query for each of the selected fields, e.g. "content", "title" etc.
                    for (int i = 0; i < params.getFields().size(); i++) {
                        QueryParser p = new QueryParser(LUCENE_VERSION, params.getFields().get(i), getAnalyzer());
                        booleanFieldsQuery.add(p.parse(params.getQuery()), BooleanClause.Occur.SHOULD);
                    }
                    fieldsQuery = searcher.rewrite(booleanFieldsQuery);
                } else {
                    // if no fields are provided, just use the "content" field by default
                    QueryParser p = new QueryParser(LUCENE_VERSION, CmsSearchField.FIELD_CONTENT, getAnalyzer());
                    fieldsQuery = searcher.rewrite(p.parse(params.getQuery()));
                }

                // finally set the main query to the fields query
                // please note that we still need both variables in case the query is a MatchAllDocsQuery - see below
                query = fieldsQuery;
            }

            if (LOG.isDebugEnabled()) {
                LOG.debug(Messages.get().getBundle().key(Messages.LOG_BASE_QUERY_1, query));
            }

            if (query == null) {
                // if no text query is set, then we match all documents 
                query = new MatchAllDocsQuery();
            } else {
                // store the parsed query for page browsing
                params.setParsedQuery(query.toString(CmsSearchField.FIELD_CONTENT));
            }

            // collect the categories
            CmsSearchCategoryCollector categoryCollector;
            if (params.isCalculateCategories()) {
                // USE THIS OPTION WITH CAUTION
                // this may slow down searched by an order of magnitude
                categoryCollector = new CmsSearchCategoryCollector(searcher);
                // perform a first search to collect the categories
                searcher.search(query, filter, categoryCollector);
                // store the result
                searchResults.setCategories(categoryCollector.getCategoryCountResult());
            }

            // perform the search operation          
            if ((params.getSort() == null) || (params.getSort() == CmsSearchParameters.SORT_DEFAULT)) {
                // apparently scoring is always enabled by Lucene if no sort order is provided
                hits = searcher.search(query, filter, m_maxHits);
            } else {
                // if  a sort order is provided, we must check if scoring must be calculated by the searcher
                prepareSortScoring(searcher, params.getSort());
                hits = searcher.search(query, filter, m_maxHits, params.getSort());
            }

            timeLucene += System.currentTimeMillis();
            timeResultProcessing = -System.currentTimeMillis();

            Document doc;
            CmsSearchResult searchResult;

            if (hits != null) {
                int hitCount = hits.totalHits > hits.scoreDocs.length ? hits.scoreDocs.length : hits.totalHits;
                int page = params.getSearchPage();
                int start = -1, end = -1;
                if ((params.getMatchesPerPage() > 0) && (page > 0) && (hitCount > 0)) {
                    // calculate the final size of the search result
                    start = params.getMatchesPerPage() * (page - 1);
                    end = start + params.getMatchesPerPage();
                    // ensure that both i and n are inside the range of foundDocuments.size()
                    start = (start > hitCount) ? hitCount : start;
                    end = (end > hitCount) ? hitCount : end;
                } else {
                    // return all found documents in the search result
                    start = 0;
                    end = hitCount;
                }

                int visibleHitCount = hitCount;
                for (int i = 0, cnt = 0; (i < hitCount) && (cnt < end); i++) {
                    try {
                        doc = searcher.doc(hits.scoreDocs[i].doc);
                        if ((isInTimeRange(doc, params)) && (hasReadPermission(searchCms, doc))) {
                            // user has read permission
                            if (cnt >= start) {
                                // do not use the resource to obtain the raw content, read it from the lucene document!
                                String excerpt = null;
                                if (isCreatingExcerpt() && (fieldsQuery != null)) {
                                    I_CmsTermHighlighter highlighter = OpenCms.getSearchManager().getHighlighter();
                                    excerpt = highlighter.getExcerpt(doc, this, params, fieldsQuery, getAnalyzer());
                                }
                                searchResult = new CmsSearchResult(
                                        Math.round((hits.scoreDocs[i].score / hits.getMaxScore()) * 100f), doc,
                                        excerpt);
                                searchResults.add(searchResult);
                            }
                            cnt++;
                        } else {
                            visibleHitCount--;
                        }
                    } catch (Exception e) {
                        // should not happen, but if it does we want to go on with the next result nevertheless                        
                        if (LOG.isWarnEnabled()) {
                            LOG.warn(Messages.get().getBundle().key(Messages.LOG_RESULT_ITERATION_FAILED_0), e);
                        }
                    }
                }

                // save the total count of search results
                searchResults.setHitCount(visibleHitCount);
            } else {
                searchResults.setHitCount(0);
            }

            timeResultProcessing += System.currentTimeMillis();
        } catch (RuntimeException e) {
            throw new CmsSearchException(Messages.get().container(Messages.ERR_SEARCH_PARAMS_1, params), e);
        } catch (Exception e) {
            throw new CmsSearchException(Messages.get().container(Messages.ERR_SEARCH_PARAMS_1, params), e);
        } finally {

            // re-set thread to previous priority
            Thread.currentThread().setPriority(previousPriority);
        }

        if (LOG.isDebugEnabled()) {
            timeTotal += System.currentTimeMillis();
            Object[] logParams = new Object[] { new Integer(hits == null ? 0 : hits.totalHits), new Long(timeTotal),
                    new Long(timeLucene), new Long(timeResultProcessing) };
            LOG.debug(Messages.get().getBundle().key(Messages.LOG_STAT_RESULTS_TIME_4, logParams));
        }

        return searchResults;
    }

    /**
     * Sets the Lucene analyzer used for this index.<p>
     *
     * @param analyzer the Lucene analyzer to set
     */
    public void setAnalyzer(Analyzer analyzer) {

        m_analyzer = analyzer;
    }

    /**
     * Can be used to enable / disable this index.<p>
     * 
     * @param enabled the state of the index to set
     */
    public void setEnabled(boolean enabled) {

        m_enabled = enabled;
    }

    /**
     * Sets the field configuration used for this index.<p>
     * 
     * @param fieldConfiguration the field configuration to set
     */
    public void setFieldConfiguration(CmsSearchFieldConfiguration fieldConfiguration) {

        m_fieldConfiguration = fieldConfiguration;
    }

    /**
     * Sets the name of the field configuration used for this index.<p>
     * 
     * @param fieldConfigurationName the name of the field configuration to set
     */
    public void setFieldConfigurationName(String fieldConfigurationName) {

        m_fieldConfigurationName = fieldConfigurationName;
    }

    /**
     * Sets the locale to index resources.<p>
     * 
     * @param locale the locale to index resources
     */
    public void setLocale(Locale locale) {

        m_locale = locale;
    }

    /**
     * Sets the locale to index resources as a String.<p>
     * 
     * @param locale the locale to index resources
     * 
     * @see #setLocale(Locale)
     */
    public void setLocaleString(String locale) {

        setLocale(CmsLocaleManager.getLocale(locale));
    }

    /**
     * Sets the number of how many hits are loaded at maximum.<p> 
     * 
     * This must be set at least to 50, or this setting is ignored.<p>
     * 
     * @param maxHits the number of how many hits are loaded at maximum to set
     * 
     * @see #getMaxHits()
     * 
     * @since 7.5.1
     */
    public void setMaxHits(int maxHits) {

        if (m_maxHits >= (MAX_HITS_DEFAULT / 100)) {
            m_maxHits = maxHits;
        }
    }

    /**
     * Sets the logical key/name of this search index.<p>
     * 
     * @param name the logical key/name of this search index 
     * 
     * @throws CmsIllegalArgumentException if the given name is null, empty or already taken by another search index 
     */
    public void setName(String name) throws CmsIllegalArgumentException {

        if (CmsStringUtil.isEmptyOrWhitespaceOnly(name)) {
            throw new CmsIllegalArgumentException(
                    Messages.get().container(Messages.ERR_SEARCHINDEX_CREATE_MISSING_NAME_0));
        } else {
            // check if already used, but only if the name was modified: 
            // this is important as unmodifiable DisplayWidgets will also invoke this...
            if (!name.equals(m_name)) {
                // don't mess with XML configuration
                if (OpenCms.getRunLevel() > OpenCms.RUNLEVEL_2_INITIALIZING) {
                    // not needed at startup and additionally getSearchManager may return null
                    Iterator<String> itIdxNames = OpenCms.getSearchManager().getIndexNames().iterator();
                    while (itIdxNames.hasNext()) {
                        if (itIdxNames.next().equals(name)) {
                            throw new CmsIllegalArgumentException(
                                    Messages.get().container(Messages.ERR_SEARCHINDEX_CREATE_INVALID_NAME_1, name));
                        }
                    }
                }
            }
        }
        m_name = name;
    }

    /**
     * Sets the name of the project used to index resources.<p>
     * 
     * A duplicate method of <code>{@link #setProjectName(String)}</code> that allows 
     * to use instances of this class as a widget object (bean convention, 
     * cp.: <code>{@link #getProject()}</code>.<p> 
     * 
     * @param projectName the name of the project used to index resources
     */
    public void setProject(String projectName) {

        setProjectName(projectName);
    }

    /**
     * Sets the name of the project used to index resources.<p>
     * 
     * @param projectName the name of the project used to index resources
     */
    public void setProjectName(String projectName) {

        m_project = projectName;
    }

    /**
     * Sets the rebuild mode of this search index.<p>
     * 
     * @param rebuildMode the rebuild mode of this search index {auto|manual}
     */
    public void setRebuildMode(String rebuildMode) {

        m_rebuild = rebuildMode;
    }

    /**
     * Controls if a resource requires view permission to be displayed in the result list.<p>
     * 
     * By default this is <code>false</code>.<p>
     * 
     * @param requireViewPermission controls if a resource requires view permission to be displayed in the result list
     */
    public void setRequireViewPermission(boolean requireViewPermission) {

        m_requireViewPermission = requireViewPermission;
    }

    /**
     * Shuts down the search index.<p>
     * 
     * This will close the local Lucene index searcher instance.<p>
     */
    public void shutDown() {

        // close the index writer
        if (m_indexWriter != null) {
            try {
                m_indexWriter.commit();
                m_indexWriter.close();
            } catch (IOException e) {
                LOG.error(
                        Messages.get().getBundle().key(Messages.LOG_IO_INDEX_WRITER_CLOSE_2, getPath(), getName()),
                        e);
            }
        }
        indexSearcherClose();
        if (CmsLog.INIT.isInfoEnabled()) {
            CmsLog.INIT.info(Messages.get().getBundle().key(Messages.INIT_SHUTDOWN_INDEX_1, getName()));
        }
    }

    /**
     * Returns the name (<code>{@link #getName()}</code>) of this search index.<p>
     *  
     * @return the name (<code>{@link #getName()}</code>) of this search index
     * 
     * @see java.lang.Object#toString()
     */
    @Override
    public String toString() {

        return getName();
    }

    /**
     * Appends the a category filter to the given filter clause that matches all given categories.<p>
     * 
     * In case the provided List is null or empty, the original filter is left unchanged.<p>
     * 
     * The original filter parameter is extended and also provided as return value.<p> 
     * 
     * @param cms the current OpenCms search context
     * @param filter the filter to extend
     * @param categories the categories that will compose the filter
     * 
     * @return the extended filter clause
     */
    protected BooleanFilter appendCategoryFilter(CmsObject cms, BooleanFilter filter, List<String> categories) {

        if ((categories != null) && (categories.size() > 0)) {
            // add query categories (if required)

            // categories are indexed as lower-case strings 
            // @see org.opencms.search.fields.CmsSearchFieldConfiguration#appendCategories
            List<String> lowerCaseCategories = new ArrayList<String>();
            for (String category : categories) {
                lowerCaseCategories.add(category.toLowerCase());
            }
            filter.add(new FilterClause(getMultiTermQueryFilter(CmsSearchField.FIELD_CATEGORY, lowerCaseCategories),
                    BooleanClause.Occur.MUST));
        }

        return filter;
    }

    /**
     * Appends a date of creation filter to the given filter clause that matches the
     * given time range.<p>
     * 
     * If the start time is equal to {@link Long#MIN_VALUE} and the end time is equal to {@link Long#MAX_VALUE}  
     * than the original filter is left unchanged.<p>
     * 
     * The original filter parameter is extended and also provided as return value.<p> 
     * 
     * @param filter the filter to extend
     * @param startTime start time of the range to search in
     * @param endTime end time of the range to search in
     * 
     * @return the extended filter clause
     */
    protected BooleanFilter appendDateCreatedFilter(BooleanFilter filter, long startTime, long endTime) {

        // create special optimized sub-filter for the date last modified search
        Filter dateFilter = createDateRangeFilter(CmsSearchField.FIELD_DATE_CREATED_LOOKUP, startTime, endTime);
        if (dateFilter != null) {
            // extend main filter with the created date filter
            filter.add(new FilterClause(dateFilter, BooleanClause.Occur.MUST));
        }

        return filter;
    }

    /**
     * Appends a date of last modification filter to the given filter clause that matches the
     * given time range.<p>
     * 
     * If the start time is equal to {@link Long#MIN_VALUE} and the end time is equal to {@link Long#MAX_VALUE}  
     * than the original filter is left unchanged.<p>
     * 
     * The original filter parameter is extended and also provided as return value.<p> 
     * 
     * @param filter the filter to extend
     * @param startTime start time of the range to search in
     * @param endTime end time of the range to search in
     * 
     * @return the extended filter clause
     */
    protected BooleanFilter appendDateLastModifiedFilter(BooleanFilter filter, long startTime, long endTime) {

        // create special optimized sub-filter for the date last modified search
        Filter dateFilter = createDateRangeFilter(CmsSearchField.FIELD_DATE_LASTMODIFIED_LOOKUP, startTime,
                endTime);
        if (dateFilter != null) {
            // extend main filter with the created date filter
            filter.add(new FilterClause(dateFilter, BooleanClause.Occur.MUST));
        }

        return filter;
    }

    /**
     * Appends the a VFS path filter to the given filter clause that matches all given root paths.<p>
     * 
     * In case the provided List is null or empty, the current request context site root is appended.<p>
     * 
     * The original filter parameter is extended and also provided as return value.<p> 
     * 
     * @param cms the current OpenCms search context
     * @param filter the filter to extend
     * @param roots the VFS root paths that will compose the filter
     * 
     * @return the extended filter clause
     */
    protected BooleanFilter appendPathFilter(CmsObject cms, BooleanFilter filter, List<String> roots) {

        // complete the search root
        TermsFilter pathFilter = new TermsFilter();
        if ((roots != null) && (roots.size() > 0)) {
            // add the all configured search roots with will request context
            for (int i = 0; i < roots.size(); i++) {
                String searchRoot = cms.getRequestContext().addSiteRoot(roots.get(i));
                extendPathFilter(pathFilter, searchRoot);
            }
        } else {
            // use the current site root as the search root
            extendPathFilter(pathFilter, cms.getRequestContext().getSiteRoot());
            // also add the shared folder (v 8.0)
            if (OpenCms.getSiteManager().getSharedFolder() != null) {
                extendPathFilter(pathFilter, OpenCms.getSiteManager().getSharedFolder());
            }
        }

        // add the calculated path filter for the root path
        filter.add(new FilterClause(pathFilter, BooleanClause.Occur.MUST));
        return filter;
    }

    /**
     * Appends the a resource type filter to the given filter clause that matches all given resource types.<p>
     * 
     * In case the provided List is null or empty, the original filter is left unchanged.<p>
     * 
     * The original filter parameter is extended and also provided as return value.<p> 
     * 
     * @param cms the current OpenCms search context
     * @param filter the filter to extend
     * @param resourceTypes the resource types that will compose the filter
     * 
     * @return the extended filter clause
     */
    protected BooleanFilter appendResourceTypeFilter(CmsObject cms, BooleanFilter filter,
            List<String> resourceTypes) {

        if ((resourceTypes != null) && (resourceTypes.size() > 0)) {
            // add query resource types (if required)
            filter.add(new FilterClause(getMultiTermQueryFilter(CmsSearchField.FIELD_TYPE, resourceTypes),
                    BooleanClause.Occur.MUST));
        }

        return filter;
    }

    /**
     * Creates an optimized date range filter for the date of last modification or creation.<p>
     * 
     * If the start date is equal to {@link Long#MIN_VALUE} and the end date is equal to {@link Long#MAX_VALUE}  
     * than <code>null</code> is returned.<p>
     * 
     * @param fieldName the name of the field to search
     * @param startTime start time of the range to search in
     * @param endTime end time of the range to search in
     * 
     * @return an optimized date range filter for the date of last modification or creation
     */
    protected Filter createDateRangeFilter(String fieldName, long startTime, long endTime) {

        TermsFilter filter = null;
        if ((startTime != Long.MIN_VALUE) || (endTime != Long.MAX_VALUE)) {
            // a date range has been set for this LGT document search
            if (startTime == Long.MIN_VALUE) {
                // default start will always be "yyyy1231" in order to reduce term size                    
                Calendar cal = Calendar.getInstance(OpenCms.getLocaleManager().getTimeZone());
                cal.setTimeInMillis(endTime);
                cal.set(cal.get(Calendar.YEAR) - MAX_YEAR_RANGE, 11, 31, 0, 0, 0);
                startTime = cal.getTimeInMillis();
            } else if (endTime == Long.MAX_VALUE) {
                // default end will always be "yyyy0101" in order to reduce term size                    
                Calendar cal = Calendar.getInstance(OpenCms.getLocaleManager().getTimeZone());
                cal.setTimeInMillis(startTime);
                cal.set(cal.get(Calendar.YEAR) + MAX_YEAR_RANGE, 0, 1, 0, 0, 0);
                endTime = cal.getTimeInMillis();
            }
            // create the filter for the date
            filter = new TermsFilter();
            // get the list of all possible date range options
            List<String> dateRage = getDateRangeSpan(startTime, endTime);
            Iterator<String> i = dateRage.iterator();
            while (i.hasNext()) {
                // generate one term query per date range option
                filter.addTerm(new Term(fieldName, i.next()));
            }
        }
        return filter;
    }

    /**
     * Creates a backup of this index for optimized re-indexing of the whole content.<p>
     * 
     * @return the path to the backup folder, or <code>null</code> in case no backup was created
     */
    protected String createIndexBackup() {

        if (!isBackupReindexing()) {
            // if no backup is generated we don't need to do anything
            return null;
        }

        // check if the target directory already exists
        File file = new File(getPath());
        if (!file.exists()) {
            // index does not exist yet, so we can't backup it
            return null;
        }
        String backupPath = getPath() + "_backup";
        try {
            // open file directory for Lucene
            FSDirectory oldDir = FSDirectory.open(file);
            FSDirectory newDir = FSDirectory.open(new File(backupPath));
            for (String fileName : oldDir.listAll()) {
                oldDir.copy(newDir, fileName, fileName);
            }
        } catch (Exception e) {
            LOG.error(Messages.get().getBundle().key(Messages.LOG_IO_INDEX_BACKUP_CREATE_3, getName(), getPath(),
                    backupPath), e);
            backupPath = null;
        }
        return backupPath;
    }

    /**
     * Checks if the provided resource should be excluded from this search index.<p> 
     * 
     * @param cms the OpenCms context used for building the search index
     * @param resource the resource to index
     * 
     * @return true if the resource should be excluded, false if it should be included in this index
     */
    protected boolean excludeFromIndex(CmsObject cms, CmsResource resource) {

        // check if this resource should be excluded from the index, if so skip it
        boolean excludeFromIndex = false;

        if (resource.isInternal() || resource.isFolder() || resource.isTemporaryFile()
                || (resource.getDateExpired() <= System.currentTimeMillis())) {
            // don't index internal resources, folders or temporary files or resources with expire date in the past
            return true;
        }

        try {
            // do property lookup with folder search
            String propValue = cms.readPropertyObject(resource, CmsPropertyDefinition.PROPERTY_SEARCH_EXCLUDE, true)
                    .getValue();
            excludeFromIndex = Boolean.valueOf(propValue).booleanValue();
            if (!excludeFromIndex && (propValue != null)) {
                // property value was neither "true" nor null, must check for "all"
                excludeFromIndex = PROPERTY_SEARCH_EXCLUDE_VALUE_ALL.equalsIgnoreCase(propValue.trim());
            }
        } catch (CmsException e) {
            if (LOG.isDebugEnabled()) {
                LOG.debug(Messages.get().getBundle().key(Messages.LOG_UNABLE_TO_READ_PROPERTY_1,
                        resource.getRootPath()));
            }
        }
        if (!excludeFromIndex && !USE_ALL_LOCALE.equalsIgnoreCase(getLocale().getLanguage())) {
            // check if any resource default locale has a match with the index locale, if not skip resource
            List<Locale> locales = OpenCms.getLocaleManager().getDefaultLocales(cms, resource);
            Locale match = OpenCms.getLocaleManager().getFirstMatchingLocale(Collections.singletonList(getLocale()),
                    locales);
            excludeFromIndex = (match == null);
        }

        return excludeFromIndex;
    }

    /**
     * Extends the given path query with another term for the given search root element.<p>
     * 
     * @param pathFilter the path filter to extend
     * @param searchRoot the search root to add to the path query
     */
    protected void extendPathFilter(TermsFilter pathFilter, String searchRoot) {

        if (!CmsResource.isFolder(searchRoot)) {
            searchRoot += "/";
        }
        pathFilter.addTerm(new Term(CmsSearchField.FIELD_PARENT_FOLDERS, searchRoot));
    }

    /**
     * Returns a cached Lucene term query filter for the given field and terms.<p>
     * 
     * @param field the field to use
     * @param terms the term to use
     * 
     * @return a cached Lucene term query filter for the given field and terms
     */
    protected Filter getMultiTermQueryFilter(String field, List<String> terms) {

        return getMultiTermQueryFilter(field, null, terms);
    }

    /**
     * Returns a cached Lucene term query filter for the given field and terms.<p>
     * 
     * @param field the field to use
     * @param terms the term to use
     * 
     * @return a cached Lucene term query filter for the given field and terms
     */
    protected Filter getMultiTermQueryFilter(String field, String terms) {

        return getMultiTermQueryFilter(field, terms, null);
    }

    /**
     * Returns a cached Lucene term query filter for the given field and terms.<p>
     * 
     * @param field the field to use
     * @param termsStr the terms to use as a String separated by a space ' ' char
     * @param termsList the list of terms to use
     * 
     * @return a cached Lucene term query filter for the given field and terms
     */
    protected Filter getMultiTermQueryFilter(String field, String termsStr, List<String> termsList) {

        if (termsStr == null) {
            StringBuffer buf = new StringBuffer(64);
            for (int i = 0; i < termsList.size(); i++) {
                if (i > 0) {
                    buf.append(' ');
                }
                buf.append(termsList.get(i));
            }
            termsStr = buf.toString();
        }
        Filter result = m_displayFilters
                .get((new StringBuffer(64)).append(field).append('|').append(termsStr).toString());
        if (result == null) {
            TermsFilter filter = new TermsFilter();
            if (termsList == null) {
                termsList = CmsStringUtil.splitAsList(termsStr, ' ');
            }
            for (int i = 0; i < termsList.size(); i++) {
                filter.addTerm(new Term(field, termsList.get(i)));
            }
            result = new CachingWrapperFilter(filter);
            m_displayFilters.put(field + termsStr, result);
        }
        return result;
    }

    /**
     * Returns a cached Lucene term query filter for the given field and term.<p>
     * 
     * @param field the field to use
     * @param term the term to use
     * 
     * @return a cached Lucene term query filter for the given field and term
     */
    protected Filter getTermQueryFilter(String field, String term) {

        return getMultiTermQueryFilter(field, term, Collections.singletonList(term));
    }

    /**
     * Checks if the OpenCms resource referenced by the result document can be read 
     * be the user of the given OpenCms context.<p>
     * 
     * @param cms the OpenCms user context to use for permission testing
     * @param doc the search result document to check
     * @return <code>true</code> if the user has read permissions to the resource
     */
    protected boolean hasReadPermission(CmsObject cms, Document doc) {

        if (!isCheckingPermissions()) {
            // no permission check is performed at all
            return true;
        }

        Fieldable typeField = doc.getFieldable(CmsSearchField.FIELD_TYPE);
        Fieldable pathField = doc.getFieldable(CmsSearchField.FIELD_PATH);
        if ((typeField == null) || (pathField == null)) {
            // permission check needs only to be performed for VFS documents that contain both fields
            return true;
        }

        String type = typeField.stringValue();
        if (!CmsSearchFieldConfiguration.VFS_DOCUMENT_KEY_PREFIX.equals(type)
                && !OpenCms.getResourceManager().hasResourceType(type)) {
            // this is an unknown VFS resource type (also not the generic "VFS" type of OpenCms before 7.0)
            return true;
        }

        // check if the resource exits in the VFS, 
        // this will implicitly check read permission and if the resource was deleted
        String contextPath = cms.getRequestContext().removeSiteRoot(pathField.stringValue());

        CmsResourceFilter filter = CmsResourceFilter.DEFAULT;
        if (isRequireViewPermission()) {
            filter = CmsResourceFilter.DEFAULT_ONLY_VISIBLE;
        }
        return cms.existsResource(contextPath, filter);
    }

    /**
     * Closes the Lucene index searcher for this index.<p>
     * 
     * @see #indexSearcherOpen(String)
     */
    protected synchronized void indexSearcherClose() {

        indexSearcherClose(m_indexSearcher);
    }

    /**
     * Closes the given Lucene index searcher.<p>
     * 
     * @param searcher the searcher to close
     */
    protected synchronized void indexSearcherClose(IndexSearcher searcher) {

        // in case there is an index searcher available close it
        if ((searcher != null) && (searcher.getIndexReader() != null)) {
            try {
                searcher.getIndexReader().close();
                searcher.close();
            } catch (Exception e) {
                LOG.error(Messages.get().getBundle().key(Messages.ERR_INDEX_SEARCHER_CLOSE_1, getName()), e);
            }
        }
    }

    /**
     * Initializes the Lucene index searcher for this index.<p>
     * 
     * Use {@link #getSearcher()} in order to obtain the searcher that has been opened.<p>
     * 
     * In case there is an index searcher still open, it is closed first.<p>
     * 
     * For performance reasons, one instance of the Lucene index searcher should be kept 
     * for all searches. However, if the index is updated or changed 
     * this searcher instance needs to be re-initialized.<p>
     * 
     * @param path the path to the index directory
     */
    protected synchronized void indexSearcherOpen(String path) {

        IndexSearcher oldSearcher = null;
        try {
            Directory indexDirectory = FSDirectory.open(new File(path));
            if (IndexReader.indexExists(indexDirectory)) {
                IndexReader reader = new LazyContentReader(IndexReader.open(indexDirectory, true));
                if (m_indexSearcher != null) {
                    // store old searcher instance to close it later
                    oldSearcher = m_indexSearcher;
                }
                m_indexSearcher = new IndexSearcher(reader);
                m_displayFilters = new HashMap<String, Filter>();
            }
        } catch (IOException e) {
            LOG.error(Messages.get().getBundle().key(Messages.ERR_INDEX_SEARCHER_1, getName()), e);
        }
        if (oldSearcher != null) {
            // close the old searcher if required
            indexSearcherClose(oldSearcher);
        }
    }

    /**
     * Reopens the Lucene index search reader for this index, required after the index has been changed.<p>
     * 
     * @see #indexSearcherOpen(String)
     */
    protected synchronized void indexSearcherUpdate() {

        IndexSearcher oldSearcher = m_indexSearcher;
        if ((oldSearcher != null) && (oldSearcher.getIndexReader() != null)) {
            // in case there is an index searcher available close it
            try {
                IndexReader newReader = IndexReader.openIfChanged(oldSearcher.getIndexReader(), true);
                if (newReader != null) {
                    m_indexSearcher = new IndexSearcher(newReader);
                    indexSearcherClose(oldSearcher);
                }
            } catch (Exception e) {
                LOG.error(Messages.get().getBundle().key(Messages.ERR_INDEX_SEARCHER_REOPEN_1, getName()), e);
            }
        } else {
            // make sure we end up with an open index searcher / reader           
            indexSearcherOpen(m_path);
        }
    }

    /**
     * Creates a new index writer.<p>
     * 
     * @param create if <code>true</code> a whole new index is created, if <code>false</code> an existing index is updated
     * 
     * @return the created new index writer
     * 
     * @throws CmsIndexException in case the writer could not be created
     * 
     * @see #getIndexWriter(I_CmsReport, boolean)
     */
    protected I_CmsIndexWriter indexWriterCreate(boolean create) throws CmsIndexException {

        IndexWriter indexWriter;
        try {
            // check if the target directory already exists
            File f = new File(m_path);
            if (!f.exists()) {
                // index does not exist yet
                f = f.getParentFile();
                if ((f != null) && !f.exists()) {
                    // create the parent folders if required
                    f.mkdirs();
                }
                // create must be true if the directory does not exist
                create = true;
            }

            // open file directory for Lucene
            FSDirectory dir = FSDirectory.open(new File(m_path));
            // create Lucene merge policy
            LogMergePolicy mergePolicy = new LogByteSizeMergePolicy();
            if (m_luceneMaxMergeDocs != null) {
                mergePolicy.setMaxMergeDocs(m_luceneMaxMergeDocs.intValue());
            }
            if (m_luceneMergeFactor != null) {
                mergePolicy.setMergeFactor(m_luceneMergeFactor.intValue());
            }
            if (m_luceneUseCompoundFile != null) {
                mergePolicy.setUseCompoundFile(m_luceneUseCompoundFile.booleanValue());
            }
            // create a new Lucene index configuration
            IndexWriterConfig indexConfig = new IndexWriterConfig(LUCENE_VERSION, getAnalyzer());
            // set the index configuration parameters if required 
            if (m_luceneRAMBufferSizeMB != null) {
                indexConfig.setRAMBufferSizeMB(m_luceneRAMBufferSizeMB.doubleValue());
            }
            if (create) {
                indexConfig.setOpenMode(OpenMode.CREATE);
            } else {
                indexConfig.setOpenMode(OpenMode.CREATE_OR_APPEND);
            }
            // create the index
            indexWriter = new IndexWriter(dir, indexConfig);
        } catch (Exception e) {
            throw new CmsIndexException(
                    Messages.get().container(Messages.ERR_IO_INDEX_WRITER_OPEN_2, m_path, m_name), e);
        }
        return new CmsLuceneIndexWriter(indexWriter, this);
    }

    /**
     * Unlocks the Lucene index writer of this index if required.<p>
     * 
     * @param report the report to write error messages on
     * 
     * @throws CmsIndexException if unlocking of the index is impossible for any reason
     */
    protected void indexWriterUnlock(I_CmsReport report) throws CmsIndexException {

        File indexPath = new File(getPath());
        boolean indexLocked = true;
        // check if the target index path already exists
        if (indexPath.exists()) {
            Directory indexDirectory = null;
            // get the lock state of the given index            
            try {
                indexDirectory = FSDirectory.open(indexPath);
                indexLocked = IndexWriter.isLocked(indexDirectory);
            } catch (Exception e) {
                LOG.error(Messages.get().getBundle().key(Messages.LOG_IO_INDEX_READER_OPEN_2, getPath(), getName()),
                        e);
            }

            // if index is locked try unlocking
            if (indexLocked) {
                try {
                    // try to force unlock on the index
                    IndexWriter.unlock(indexDirectory);
                } catch (Exception e) {
                    // unable to force unlock of Lucene index, we can't continue this way
                    CmsMessageContainer msg = Messages.get().container(Messages.ERR_INDEX_LOCK_FAILED_1, getName());
                    report.println(msg, I_CmsReport.FORMAT_ERROR);
                    throw new CmsIndexException(msg, e);
                }
            }
        }
    }

    /**
     * Checks if the document is in the time range specified in the search parameters.<p>
     * 
     * The creation date and/or the last modification date are checked.<p>
     * 
     * @param doc the document to check the dates against the given time range
     * @param params the search parameters where the time ranges are specified
     * 
     * @return true if document is in time range or not time range set otherwise false
     */
    protected boolean isInTimeRange(Document doc, CmsSearchParameters params) {

        if (!isCheckingTimeRange()) {
            // time range check disabled
            return true;
        }

        try {
            // check the creation date of the document against the given time range
            Date dateCreated = DateTools
                    .stringToDate(doc.getFieldable(CmsSearchField.FIELD_DATE_CREATED).stringValue());
            if (dateCreated.getTime() < params.getMinDateCreated()) {
                return false;
            }
            if (dateCreated.getTime() > params.getMaxDateCreated()) {
                return false;
            }

            // check the last modification date of the document against the given time range
            Date dateLastModified = DateTools
                    .stringToDate(doc.getFieldable(CmsSearchField.FIELD_DATE_LASTMODIFIED).stringValue());
            if (dateLastModified.getTime() < params.getMinDateLastModified()) {
                return false;
            }
            if (dateLastModified.getTime() > params.getMaxDateLastModified()) {
                return false;
            }

        } catch (ParseException ex) {
            // date could not be parsed -> doc is in time range
        }

        return true;
    }

    /**
     * Checks if the score for the results must be calculated based on the provided sort option.<p>  
     * 
     * Since Lucene 3 apparently the score is no longer calculated by default, but only if the 
     * searcher is explicitly told so. This methods checks if, based on the given sort, 
     * the score must be calculated.<p> 
     * 
     * @param searcher the index searcher to prepare 
     * @param sort the sort option to use
     */
    protected void prepareSortScoring(IndexSearcher searcher, Sort sort) {

        boolean doScoring = false;
        if (sort != null) {
            if ((sort == CmsSearchParameters.SORT_DEFAULT) || (sort == CmsSearchParameters.SORT_TITLE)) {
                // these default sorts do need score calculation
                doScoring = true;
            } else if ((sort == CmsSearchParameters.SORT_DATE_CREATED)
                    || (sort == CmsSearchParameters.SORT_DATE_LASTMODIFIED)) {
                // these default sorts don't need score calculation
                doScoring = false;
            } else {
                // for all non-defaults: check if the score field is present, in that case we must calculate the score
                SortField[] fields = sort.getSort();
                for (SortField field : fields) {
                    if (field == SortField.FIELD_SCORE) {
                        doScoring = true;
                        break;
                    }
                }
            }
        }
        searcher.setDefaultFieldSortScoring(doScoring, doScoring);
    }

    /**
     * Removes the given backup folder of this index.<p>
     * 
     * @param path the backup folder to remove
     * 
     * @see #isBackupReindexing()
     */
    protected void removeIndexBackup(String path) {

        if (!isBackupReindexing()) {
            // if no backup is generated we don't need to do anything
            return;
        }

        // check if the target directory already exists
        File file = new File(path);
        if (!file.exists()) {
            // index does not exist yet
            return;
        }
        try {
            FSDirectory dir = FSDirectory.open(file);
            dir.close();
            CmsFileUtil.purgeDirectory(file);
        } catch (Exception e) {
            LOG.error(Messages.get().getBundle().key(Messages.LOG_IO_INDEX_BACKUP_REMOVE_2, getName(), path), e);
        }
    }
}