com.zimbra.cs.index.AbstractIndexStoreTest.java Source code

Java tutorial

Introduction

Here is the source code for com.zimbra.cs.index.AbstractIndexStoreTest.java

Source

/*
 * ***** BEGIN LICENSE BLOCK *****
 * Zimbra Collaboration Suite Server
 * Copyright (C) 2012, 2013, 2014, 2016 Synacor, Inc.
 *
 * This program is free software: you can redistribute it and/or modify it under
 * the terms of the GNU General Public License as published by the Free Software Foundation,
 * version 2 of the License.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
 * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 * See the GNU General Public License for more details.
 * You should have received a copy of the GNU General Public License along with this program.
 * If not, see <https://www.gnu.org/licenses/>.
 * ***** END LICENSE BLOCK *****
 */
package com.zimbra.cs.index;

import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;

import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.MultiPhraseQuery;
import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.search.PrefixQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TermRangeQuery;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;

import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.io.Closeables;
import com.zimbra.common.mailbox.ContactConstants;
import com.zimbra.common.service.ServiceException;
import com.zimbra.common.util.ZimbraLog;
import com.zimbra.cs.account.Account;
import com.zimbra.cs.account.MockProvisioning;
import com.zimbra.cs.account.Provisioning;
import com.zimbra.cs.index.ZimbraIndexReader.TermFieldEnumeration;
import com.zimbra.cs.mailbox.Contact;
import com.zimbra.cs.mailbox.Folder;
import com.zimbra.cs.mailbox.Mailbox;
import com.zimbra.cs.mailbox.MailboxManager;
import com.zimbra.cs.mailbox.MailboxTestUtil;
import com.zimbra.cs.mime.ParsedContact;

public abstract class AbstractIndexStoreTest {
    static String originalIndexStoreFactory;

    protected abstract String getIndexStoreFactory();

    /**
     * Override this for any Index Store specific cleanup.  Note that for Mock Provisioning, deleting an account
     * does not currently cleanup the index.
     */
    protected void cleanupForIndexStore() {
    }

    /**
     * Override this for any Index Store which might not be available;
     */
    protected boolean indexStoreAvailable() {
        return true;
    }

    static Provisioning prov;
    static Account testAcct;

    @BeforeClass
    public static void init() throws Exception {
        System.setProperty("log4j.configuration", "log4j-test.properties");
        MailboxTestUtil.initServer();
        prov = Provisioning.getInstance();
        testAcct = prov.createAccount("test@zimbra.com", "secret", new HashMap<String, Object>());
        originalIndexStoreFactory = IndexStore.getFactory().getClass().getName();
    }

    @AfterClass
    public static void destroy() {
        try {
            prov.deleteAccount(testAcct.getId());
        } catch (ServiceException e) {
            ZimbraLog.test.error("Problem cleaning up test@zimbra.com account", e);
        }
        IndexStore.getFactory().destroy();
        IndexStore.setFactory(originalIndexStoreFactory);
    }

    @After
    public void teardown() throws Exception {
        IndexStore.getFactory().destroy();
        cleanupForIndexStore();
        MailboxTestUtil.clearData();
    }

    @Before
    public void setup() throws Exception {
        IndexStore.setFactory(getIndexStoreFactory());
        Assert.assertTrue("Index Store NEEDS to be configured and available", indexStoreAvailable());
        MailboxTestUtil.clearData();
        cleanupForIndexStore();
    }

    @Test
    public void termQuery() throws Exception {
        ZimbraLog.test.debug("--->TEST termQuery");
        Mailbox mbox = MailboxManager.getInstance().getMailboxByAccountId(MockProvisioning.DEFAULT_ACCOUNT_ID);
        Contact contact = createContact(mbox, "First", "Last", "test@zimbra.com");
        createContact(mbox, "a", "bc", "abc@zimbra.com");
        createContact(mbox, "j", "k", "j.k@zimbra.com");
        createContact(mbox, "Matilda", "Higgs-Bozon", "matilda.higgs.bozon@zimbra.com");
        mbox.index.indexDeferredItems();

        // Stick with just one IndexStore - the one cached in Mailbox:
        //    IndexStore index = IndexStore.getFactory().getIndexStore(mbox);
        IndexStore index = mbox.index.getIndexStore();
        ZimbraIndexSearcher searcher = index.openSearcher();
        ZimbraTopDocs result = searcher
                .search(new TermQuery(new Term(LuceneFields.L_CONTACT_DATA, "none@zimbra.com")), 100);
        Assert.assertNotNull("searcher.search result object - searching for none@zimbra.com", result);
        ZimbraLog.test.debug("Result for search for 'none@zimbra.com'\n" + result.toString());
        Assert.assertEquals("Number of hits searching for none@zimbra.com", 0, result.getTotalHits());

        result = searcher.search(new TermQuery(new Term(LuceneFields.L_CONTACT_DATA, "test@zimbra.com")), 100);
        Assert.assertNotNull("searcher.search result object - searching for test@zimbra.com", result);
        ZimbraLog.test.debug("Result for search for 'test@zimbra.com'\n" + result.toString());
        Assert.assertEquals("Number of hits searching for test@zimbra.com", 1, result.getTotalHits());
        Assert.assertEquals(String.valueOf(contact.getId()), getBlobIdForResultDoc(searcher, result, 0));
        Assert.assertEquals(4, searcher.getIndexReader().numDocs());
        searcher.close();
    }

    @Test
    public void filteredTermQuery() throws Exception {
        ZimbraLog.test.debug("--->TEST filteredTermQuery");
        Mailbox mbox = MailboxManager.getInstance().getMailboxByAccountId(MockProvisioning.DEFAULT_ACCOUNT_ID);
        Folder folder = mbox.getFolderById(null, Mailbox.ID_FOLDER_CONTACTS);
        Contact contact1 = createContact(mbox, "a", "bc", "abc@zimbra.com");
        Contact contact2 = createContact(mbox, "a", "bcd", "abcd@zimbra.com");
        Contact contact3 = createContact(mbox, "x", "y", "xy@zimbra.com");
        Contact contact4 = createContact(mbox, "x", "yz", "xyz@zimbra.com");
        Contact contact5 = createContact(mbox, "x", "yz", "xyz@zimbra.com");
        mbox.index.indexDeferredItems(); // Make sure we don't index items after the deleteIndex() below

        IndexStore index = mbox.index.getIndexStore();
        index.deleteIndex();
        Indexer indexer = index.openIndexer();
        indexer.addDocument(folder, contact1, contact1.generateIndexData());
        indexer.addDocument(folder, contact2, contact2.generateIndexData());
        indexer.addDocument(folder, contact3, contact3.generateIndexData());
        indexer.addDocument(folder, contact4, contact4.generateIndexData());
        // Note: NOT indexed contact5
        indexer.close();

        List<Term> terms = Lists.newArrayList();
        terms.add(new Term(LuceneFields.L_MAILBOX_BLOB_ID, String.valueOf(contact2.getId())));
        terms.add(new Term(LuceneFields.L_MAILBOX_BLOB_ID, String.valueOf(contact4.getId())));
        terms.add(new Term(LuceneFields.L_MAILBOX_BLOB_ID, String.valueOf(contact5.getId())));
        ZimbraTermsFilter filter = new ZimbraTermsFilter(terms);
        ZimbraIndexSearcher searcher = index.openSearcher();
        ZimbraTopDocs result;
        result = searcher.search(new TermQuery(new Term(LuceneFields.L_CONTACT_DATA, "zimbra.com")), filter, 100);
        Assert.assertNotNull("searcher.search result object - searching for zimbra.com", result);
        ZimbraLog.test.debug("Result for search for 'zimbra.com', filtering by IDs\n%s", result.toString());
        Assert.assertEquals("Number of hits", 2, result.getTotalHits());
        List<String> expecteds = Lists.newArrayList();
        List<String> matches = Lists.newArrayList();
        matches.add(getBlobIdForResultDoc(searcher, result, 0));
        matches.add(getBlobIdForResultDoc(searcher, result, 1));
        expecteds.add(String.valueOf(contact2.getId()));
        expecteds.add(String.valueOf(contact4.getId()));
        Collections.sort(matches);
        Collections.sort(expecteds);
        Assert.assertEquals("Match Blob ID", expecteds.get(0), matches.get(0));
        Assert.assertEquals("Match Blob ID", expecteds.get(1), matches.get(1));
        searcher.close();
    }

    @Test
    public void sortedFilteredTermQuery() throws Exception {
        ZimbraLog.test.debug("--->TEST sortedFilteredTermQuery");
        Mailbox mbox = MailboxManager.getInstance().getMailboxByAccountId(MockProvisioning.DEFAULT_ACCOUNT_ID);
        Folder folder = mbox.getFolderById(null, Mailbox.ID_FOLDER_CONTACTS);
        Contact con1 = createContact(mbox, "a", "bc", "abc@zimbra.com");
        Contact con2 = createContact(mbox, "abcd@zimbra.com");
        Contact con3 = createContact(mbox, "xy@zimbra.com");
        Thread.sleep(1001); // To ensure different sort date
        Contact con4 = createContact(mbox, "xyz@zimbra.com");
        Contact con5 = createContact(mbox, "xyz@zimbra.com");
        mbox.index.indexDeferredItems(); // Make sure we don't index items after the deleteIndex() below

        IndexStore index = mbox.index.getIndexStore();
        index.deleteIndex();
        Indexer indexer = index.openIndexer();
        indexer.addDocument(folder, con1, con1.generateIndexData());
        indexer.addDocument(folder, con2, con2.generateIndexData());
        indexer.addDocument(folder, con3, con3.generateIndexData());
        indexer.addDocument(folder, con4, con4.generateIndexData());
        // Note: NOT indexed contact5
        indexer.close();

        List<Term> terms = Lists.newArrayList();
        terms.add(new Term(LuceneFields.L_MAILBOX_BLOB_ID, String.valueOf(con2.getId())));
        terms.add(new Term(LuceneFields.L_MAILBOX_BLOB_ID, String.valueOf(con4.getId())));
        terms.add(new Term(LuceneFields.L_MAILBOX_BLOB_ID, String.valueOf(con5.getId())));
        ZimbraTermsFilter filter = new ZimbraTermsFilter(terms);
        ZimbraIndexSearcher srchr = index.openSearcher();
        ZimbraTopDocs result;
        Sort sort = new Sort(new SortField(LuceneFields.L_SORT_DATE, SortField.STRING, false));
        result = srchr.search(new TermQuery(new Term(LuceneFields.L_CONTACT_DATA, "zimbra.com")), filter, 100,
                sort);
        Assert.assertNotNull("searcher.search result object - searching for zimbra.com", result);
        ZimbraLog.test.debug("Result for search for 'zimbra.com', filtering by IDs 2,4 & 5\n%s", result.toString());
        Assert.assertEquals("Number of hits", 2, result.getTotalHits());
        Assert.assertEquals("Match Blob ID 1", String.valueOf(con2.getId()),
                getBlobIdForResultDoc(srchr, result, 0));
        Assert.assertEquals("Match Blob ID 2", String.valueOf(con4.getId()),
                getBlobIdForResultDoc(srchr, result, 1));
        // Repeat but with a reverse sort this time
        sort = new Sort(new SortField(LuceneFields.L_SORT_DATE, SortField.STRING, true));
        result = srchr.search(new TermQuery(new Term(LuceneFields.L_CONTACT_DATA, "zimbra.com")), filter, 100,
                sort);
        Assert.assertNotNull("searcher.search result object - searching for zimbra.com", result);
        ZimbraLog.test.debug("Result for search for 'zimbra.com' sorted reverse, filter by IDs\n%s",
                result.toString());
        Assert.assertEquals("Number of hits", 2, result.getTotalHits());
        Assert.assertEquals("Match Blob ID 1", String.valueOf(con4.getId()),
                getBlobIdForResultDoc(srchr, result, 0));
        Assert.assertEquals("Match Blob ID 2", String.valueOf(con2.getId()),
                getBlobIdForResultDoc(srchr, result, 1));
        srchr.close();
    }

    @Test
    public void leadingWildcardQuery() throws Exception {
        ZimbraLog.test.debug("--->TEST leadingWildcardQuery");
        Mailbox mbox = MailboxManager.getInstance().getMailboxByAccountId(MockProvisioning.DEFAULT_ACCOUNT_ID);
        Contact contact = createContact(mbox, "First", "Last", "f.last@zimbra.com", "Leading Wildcard");
        createContact(mbox, "Grand", "Piano", "grand@vmware.com");
        mbox.index.indexDeferredItems(); // Make sure all indexing has been done

        IndexStore index = mbox.index.getIndexStore();
        ZimbraIndexSearcher searcher = index.openSearcher();
        // This seems to be the supported way of enabling leading wildcard queries for Lucene
        QueryParser queryParser = new QueryParser(LuceneIndex.VERSION, LuceneFields.L_CONTACT_DATA,
                new StandardAnalyzer(LuceneIndex.VERSION));
        queryParser.setAllowLeadingWildcard(true);
        Query query = queryParser.parse("*irst");
        ZimbraTopDocs result = searcher.search(query, 100);
        Assert.assertNotNull("searcher.search result object - searching for *irst", result);
        ZimbraLog.test.debug("Result for search for '*irst'\n" + result.toString());
        Assert.assertEquals("Number of hits searching for *irst", 1, result.getTotalHits());
        String expected1Id = String.valueOf(contact.getId());
        String match1Id = searcher.doc(result.getScoreDoc(0).getDocumentID()).get(LuceneFields.L_MAILBOX_BLOB_ID);
        Assert.assertEquals("Mailbox Blob ID of match", expected1Id, match1Id);
    }

    @Test
    public void booleanQuery() throws Exception {
        ZimbraLog.test.debug("--->TEST booleanQuery");
        Mailbox mbox = MailboxManager.getInstance().getMailboxByAccountId(MockProvisioning.DEFAULT_ACCOUNT_ID);
        Contact contact = createContact(mbox, "First", "Last", "f.last@zimbra.com",
                "Software Development Engineer");
        createContact(mbox, "Given", "Surname", "GiV.SurN@zimbra.com");
        mbox.index.indexDeferredItems(); // Make sure all indexing has been done

        IndexStore index = mbox.index.getIndexStore();
        ZimbraIndexSearcher searcher = index.openSearcher();
        // This seems to be the supported way of enabling leading wildcard queries
        QueryParser queryParser = new QueryParser(LuceneIndex.VERSION, LuceneFields.L_CONTACT_DATA,
                new StandardAnalyzer(LuceneIndex.VERSION));
        queryParser.setAllowLeadingWildcard(true);
        Query wquery = queryParser.parse("*irst");
        Query tquery = new TermQuery(new Term(LuceneFields.L_CONTACT_DATA, "absent"));
        Query tquery2 = new TermQuery(new Term(LuceneFields.L_CONTACT_DATA, "Last"));
        BooleanQuery bquery = new BooleanQuery();
        bquery.add(wquery, Occur.MUST);
        bquery.add(tquery, Occur.MUST_NOT);
        bquery.add(tquery2, Occur.SHOULD);
        ZimbraTopDocs result = searcher.search(bquery, 100);
        Assert.assertNotNull("searcher.search result object", result);
        ZimbraLog.test.debug("Result for search [hits=%d]:%s", result.getTotalHits(), result.toString());
        Assert.assertEquals("Number of hits", 1, result.getTotalHits());
        String expected1Id = String.valueOf(contact.getId());
        String match1Id = searcher.doc(result.getScoreDoc(0).getDocumentID()).get(LuceneFields.L_MAILBOX_BLOB_ID);
        Assert.assertEquals("Mailbox Blob ID of match", expected1Id, match1Id);
    }

    @Test
    public void phraseQuery() throws Exception {
        ZimbraLog.test.debug("--->TEST phraseQuery");
        Mailbox mbox = MailboxManager.getInstance().getMailboxByAccountId(MockProvisioning.DEFAULT_ACCOUNT_ID);
        createContact(mbox, "Non", "Match", "nOn.MaTchiNg@zimbra.com");
        Contact contact2 = createContact(mbox, "First", "Last", "f.last@zimbra.com",
                "Software Development Engineer");
        createContact(mbox, "Given", "Surname", "GiV.SurN@zimbra.com");
        mbox.index.indexDeferredItems(); // Make sure all indexing has been done

        IndexStore index = mbox.index.getIndexStore();
        ZimbraIndexSearcher searcher = index.openSearcher();
        PhraseQuery pquery = new PhraseQuery();
        // Lower case required for each term for Lucene
        pquery.add(new Term(LuceneFields.L_CONTENT, "software"));
        pquery.add(new Term(LuceneFields.L_CONTENT, "development"));
        pquery.add(new Term(LuceneFields.L_CONTENT, "engineer"));
        ZimbraTopDocs result = searcher.search(pquery, 100);
        Assert.assertNotNull("searcher.search result object", result);
        ZimbraLog.test.debug("Result for search [hits=%d]:%s", result.getTotalHits(), result.toString());
        Assert.assertEquals("Number of hits", 1, result.getTotalHits());
        String expected1Id = String.valueOf(contact2.getId());
        String match1Id = getBlobIdForResultDoc(searcher, result, 0);
        Assert.assertEquals("Mailbox Blob ID of match", expected1Id, match1Id);
        pquery = new PhraseQuery();
        // Try again with words out of order
        pquery.add(new Term(LuceneFields.L_CONTENT, "development"));
        pquery.add(new Term(LuceneFields.L_CONTENT, "software"));
        pquery.add(new Term(LuceneFields.L_CONTENT, "engineer"));
        result = searcher.search(pquery, 100);
        Assert.assertNotNull("searcher.search result object", result);
        ZimbraLog.test.debug("Result for search [hits=%d]:%s", result.getTotalHits(), result.toString());
        Assert.assertEquals("Number of hits", 0, result.getTotalHits());
    }

    @Test
    public void phraseQueryWithStopWord() throws Exception {
        ZimbraLog.test.debug("--->TEST phraseQueryWithStopWord");
        Mailbox mbox = MailboxManager.getInstance().getMailboxByAccountId(MockProvisioning.DEFAULT_ACCOUNT_ID);
        createContact(mbox, "Non", "Match", "nOn.MaTchiNg@zimbra.com");
        Contact contact2 = createContact(mbox, "First", "Last", "f.last@zimbra.com",
                "1066 and all that with William the conqueror and others");
        createContact(mbox, "Given", "Surname", "GiV.SurN@zimbra.com");
        mbox.index.indexDeferredItems(); // Make sure all indexing has been done

        IndexStore index = mbox.index.getIndexStore();
        ZimbraIndexSearcher searcher = index.openSearcher();
        PhraseQuery pquery = new PhraseQuery();
        // Lower case required for each term for Lucene
        pquery.add(new Term(LuceneFields.L_CONTENT, "william"));
        // pquery.add(new Term(LuceneFields.L_CONTENT, "the")); - excluded because it is a stop word
        pquery.add(new Term(LuceneFields.L_CONTENT, "conqueror"));
        ZimbraTopDocs result = searcher.search(pquery, 100);
        Assert.assertNotNull("searcher.search result object", result);
        ZimbraLog.test.debug("Result for search [hits=%d]:%s", result.getTotalHits(), result.toString());
        Assert.assertEquals("Number of hits", 1, result.getTotalHits());
        String expected1Id = String.valueOf(contact2.getId());
        String match1Id = getBlobIdForResultDoc(searcher, result, 0);
        Assert.assertEquals("Mailbox Blob ID of match", expected1Id, match1Id);
    }

    @Test
    public void multiPhraseQuery() throws Exception {
        ZimbraLog.test.debug("--->TEST multiPhraseQuery");
        Mailbox mbox = MailboxManager.getInstance().getMailboxByAccountId(MockProvisioning.DEFAULT_ACCOUNT_ID);
        createContact(mbox, "Non", "Match", "nOn.MaTchiNg@zimbra.com");
        Contact contact1 = createContact(mbox, "Paul", "AA", "aa@example.net", "Software Development Engineer");
        createContact(mbox, "Jane", "BB", "bb@example.net", "Software Planning Engineer");
        Contact contact2 = createContact(mbox, "Peter", "CC", "cc@example.net", "Software Dev Engineer");
        createContact(mbox, "Avril", "DD", "dd@example.net", "Software Architectural Engineer");
        Contact contact3 = createContact(mbox, "Leo", "EE", "ee@example.net", "Software Developer Engineer");
        Contact contact4 = createContact(mbox, "Wow", "DD", "dd@example.net", "Softly Development Engineer");
        createContact(mbox, "Given", "Surname", "GiV.SurN@zimbra.com");
        mbox.index.indexDeferredItems(); // Make sure all indexing has been done

        IndexStore index = mbox.index.getIndexStore();
        ZimbraIndexSearcher searcher = index.openSearcher();
        MultiPhraseQuery pquery = new MultiPhraseQuery();
        // Lower case required for each term for Lucene
        Term[] firstWords = { new Term(LuceneFields.L_CONTENT, "softly"),
                new Term(LuceneFields.L_CONTENT, "software") };
        pquery.add(firstWords);
        Term[] secondWords = { new Term(LuceneFields.L_CONTENT, "dev"),
                new Term(LuceneFields.L_CONTENT, "development"), new Term(LuceneFields.L_CONTENT, "developer") };
        pquery.add(secondWords);
        pquery.add(new Term(LuceneFields.L_CONTENT, "engineer"));
        ZimbraTopDocs result = searcher.search(pquery, 100);
        Assert.assertNotNull("searcher.search result object", result);
        ZimbraLog.test.debug("Result for search [hits=%d]:%s", result.getTotalHits(), result.toString());
        Assert.assertEquals("Number of hits", 4, result.getTotalHits());
        List<String> expecteds = Lists.newArrayList();
        List<String> matches = Lists.newArrayList();
        matches.add(getBlobIdForResultDoc(searcher, result, 0));
        matches.add(getBlobIdForResultDoc(searcher, result, 1));
        matches.add(getBlobIdForResultDoc(searcher, result, 2));
        matches.add(getBlobIdForResultDoc(searcher, result, 3));
        expecteds.add(String.valueOf(contact1.getId()));
        expecteds.add(String.valueOf(contact2.getId()));
        expecteds.add(String.valueOf(contact3.getId()));
        expecteds.add(String.valueOf(contact4.getId()));
        Collections.sort(matches);
        Collections.sort(expecteds);
        for (int ndx = 0; ndx < 4; ndx++) {
            Assert.assertEquals("Match Blob ID", expecteds.get(0), matches.get(0));
        }
    }

    @Test
    public void prefixQuery() throws Exception {
        ZimbraLog.test.debug("--->TEST prefixQuery");
        Mailbox mbox = MailboxManager.getInstance().getMailboxByAccountId(MockProvisioning.DEFAULT_ACCOUNT_ID);
        Contact contact1 = createContact(mbox, "a", "bc", "abc@zimbra.com");
        Contact contact2 = createContact(mbox, "a", "bcd", "abcd@zimbra.com");
        createContact(mbox, "x", "Y", "xy@zimbra.com");
        createContact(mbox, "x", "Yz", "x.Yz@zimbra.com");
        mbox.index.indexDeferredItems(); // Make sure all indexing has been done
        IndexStore index = mbox.index.getIndexStore();
        ZimbraIndexSearcher searcher = index.openSearcher();
        ZimbraTopDocs result = searcher.search(new PrefixQuery(new Term(LuceneFields.L_CONTACT_DATA, "ab")), 100);
        Assert.assertNotNull("searcher.search result object - searching for 'ab' prefix", result);
        ZimbraLog.test.debug("Result for search for 'ab'\n" + result.toString());
        Assert.assertEquals("Number of hits searching for 'ab' prefix", 2, result.getTotalHits());
        String contact1Id = String.valueOf(contact1.getId());
        String contact2Id = String.valueOf(contact2.getId());
        String match1Id = getBlobIdForResultDoc(searcher, result, 0);
        String match2Id = getBlobIdForResultDoc(searcher, result, 1);
        ZimbraLog.test.debug("Contact1ID=%s Contact2ID=%s match1id=%s match2id=%s", contact1Id, contact2Id,
                match1Id, match2Id);
        if (contact1Id.equals(match1Id)) {
            Assert.assertEquals("2nd match isn't contact2's ID", contact2Id, match2Id);
        } else if (contact1Id.equals(match2Id)) {
            Assert.assertEquals("2nd match isn't contact1's ID", contact2Id, match1Id);
        } else {
            Assert.fail(String.format("Contact 1 ID [%s] doesn't match either [%s] or [%s]", contact1Id, match1Id,
                    match2Id));
        }
    }

    @Test
    public void termRangeQuery() throws Exception {
        ZimbraLog.test.debug("--->TEST termRangeQuery");
        Mailbox mbox = MailboxManager.getInstance().getMailboxByAccountId(MockProvisioning.DEFAULT_ACCOUNT_ID);
        Contact contact1 = createContact(mbox, "James", "Peters", "abc@zimbra.com");
        Contact contact2 = createContact(mbox, "a", "bcd", "abcd@zimbra.com");
        createContact(mbox, "aa", "bcd", "aaaa@zimbra.com");
        createContact(mbox, "aa", "bcd", "zzz@zimbra.com");

        mbox.index.indexDeferredItems(); // Make sure all indexing has been done
        IndexStore index = mbox.index.getIndexStore();
        ZimbraIndexSearcher searcher = index.openSearcher();
        TermRangeQuery query = new TermRangeQuery(LuceneFields.L_FIELD, "email:aba@zimbra.com",
                "email:abz@zimbra.com", false, true);
        ZimbraTopDocs result = searcher.search(query, 100);
        Assert.assertNotNull("searcher.search result object", result);
        ZimbraLog.test.debug("Result for search %s", result.toString());
        Assert.assertEquals("Number of hits", 2, result.getTotalHits());
        List<String> expecteds = Lists.newArrayList();
        List<String> matches = Lists.newArrayList();
        matches.add(getBlobIdForResultDoc(searcher, result, 0));
        matches.add(getBlobIdForResultDoc(searcher, result, 1));
        expecteds.add(String.valueOf(contact1.getId()));
        expecteds.add(String.valueOf(contact2.getId()));
        Collections.sort(matches);
        Collections.sort(expecteds);
        Assert.assertEquals("Match Blob ID", expecteds.get(0), matches.get(0));
        Assert.assertEquals("Match Blob ID", expecteds.get(1), matches.get(1));
    }

    @Test
    public void deleteDocument() throws Exception {
        ZimbraLog.test.debug("--->TEST deleteDocument");
        Mailbox mbox = MailboxManager.getInstance().getMailboxByAccountId(MockProvisioning.DEFAULT_ACCOUNT_ID);
        IndexStore index = mbox.index.getIndexStore();
        index.deleteIndex();
        Indexer indexer = index.openIndexer();
        Assert.assertEquals("maxDocs at start", 0, indexer.maxDocs());
        Contact contact1 = createContact(mbox, "James", "Peters", "test1@zimbra.com");
        createContact(mbox, "Emma", "Peters", "test2@zimbra.com");

        mbox.index.indexDeferredItems(); // Make sure all indexing has been done
        ZimbraIndexSearcher searcher = index.openSearcher();
        Assert.assertEquals("numDocs after 2 adds", 2, searcher.getIndexReader().numDocs());
        ZimbraTopDocs result = searcher.search(new TermQuery(new Term(LuceneFields.L_CONTACT_DATA, "@zimbra.com")),
                100);
        Assert.assertNotNull("searcher.search result object - searching for '@zimbra.com'", result);
        ZimbraLog.test.debug("Result for search for '@zimbra.com'\n" + result.toString());
        Assert.assertEquals("Total hits after 2 adds", 2, result.getTotalHits());
        searcher.close();

        indexer = index.openIndexer();
        indexer.deleteDocument(Collections.singletonList(contact1.getId()));
        indexer.close();

        searcher = index.openSearcher();
        Assert.assertEquals("numDocs after 2 adds/1 del", 1, searcher.getIndexReader().numDocs());
        result = searcher.search(new TermQuery(new Term(LuceneFields.L_CONTACT_DATA, "@zimbra.com")), 100);
        Assert.assertNotNull("searcher.search result object after 2 adds/1 del", result);
        ZimbraLog.test.debug("Result for search for '@zimbra.com'\n" + result.toString());
        Assert.assertEquals("Total hits after 2 adds/1 del", 1, result.getTotalHits());
        searcher.close();
    }

    @Test
    public void getCount() throws Exception {
        ZimbraLog.test.debug("--->TEST getCount");
        Mailbox mbox = MailboxManager.getInstance().getMailboxByAccountId(MockProvisioning.DEFAULT_ACCOUNT_ID);
        IndexStore index = mbox.index.getIndexStore();
        index.deleteIndex();
        Indexer indexer = index.openIndexer();
        Assert.assertEquals("maxDocs at start", 0, indexer.maxDocs());
        createContact(mbox, "Jane", "Peters", "test1@zimbra.com");
        createContact(mbox, "Emma", "Peters", "test2@zimbra.com");
        createContact(mbox, "Fiona", "Peters", "test3@zimbra.com");
        createContact(mbox, "Edward", "Peters", "test4@zimbra.com");
        mbox.index.indexDeferredItems(); // Make sure all indexing has been done

        Assert.assertEquals("maxDocs after adding 4 contacts", 4, indexer.maxDocs());
        indexer.close();

        ZimbraIndexSearcher searcher = index.openSearcher();
        Assert.assertEquals("numDocs after adding 4 contacts", 4, searcher.getIndexReader().numDocs());
        Assert.assertEquals("docs which match 'test1'", 1,
                searcher.docFreq(new Term(LuceneFields.L_CONTACT_DATA, "test1")));
        Assert.assertEquals("docs which match '@zimbra.com'", 4,
                searcher.docFreq(new Term(LuceneFields.L_CONTACT_DATA, "@zimbra.com")));
        searcher.close();
    }

    private void checkNextTerm(TermFieldEnumeration fields, Term term) {
        Assert.assertTrue("fields.hasMoreElements() value when expecting:" + term.toString(),
                fields.hasMoreElements());
        BrowseTerm browseTerm = fields.nextElement();
        Assert.assertNotNull("fields.nextElement() value when expecting:" + term.toString(), browseTerm);
        ZimbraLog.test.debug("Expecting %s=%s value is %s docFreq=%d", term.field(), term.text(),
                browseTerm.getText(), browseTerm.getFreq());
        Assert.assertEquals("field value", term.text(), browseTerm.getText());
    }

    private void checkNextTermFieldType(TermFieldEnumeration fields, String field) {
        Assert.assertTrue("fields.hasMoreElements() value when expecting:" + field, fields.hasMoreElements());
        BrowseTerm browseTerm = fields.nextElement();
        Assert.assertNotNull("fields.nextElement() value when expecting:" + field, browseTerm);
        ZimbraLog.test.debug("Expecting %s=?anyvalue? value is %s docFreq=%d", field, browseTerm.getText(),
                browseTerm.getFreq());
    }

    private void checkAtEnd(TermFieldEnumeration fields, String field) {
        Assert.assertFalse("fields.hasMoreElements() at end of list for field:" + field, fields.hasMoreElements());
        try {
            fields.nextElement();
            Assert.fail("fields.nextElement() at end of list for field:" + field + " contact data succeeded");
        } catch (NoSuchElementException ex) {
        }
    }

    /**
     * The result of getTermsForField can be good for seeing the effects of {@code ZimbraAnalyzer} on how fields get
     * tokenized. TODO:  Add tests for different types of tokenizers.
     * @throws Exception
     */
    @Test
    public void termEnum() throws Exception {
        ZimbraLog.test.debug("--->TEST termEnum");
        Mailbox mbox = MailboxManager.getInstance().getMailboxByAccountId(MockProvisioning.DEFAULT_ACCOUNT_ID);
        createContact(mbox, "teSt1@ziMBRA.com");
        createContact(mbox, "test2@zimbra.com");

        mbox.index.indexDeferredItems(); // Make sure all indexing has been done
        IndexStore index = mbox.index.getIndexStore();
        ZimbraIndexSearcher searcher = index.openSearcher();
        // Note that TermFieldEnumeration order is defined to be sorted
        TermFieldEnumeration fields = null;
        try {
            fields = searcher.getIndexReader().getTermsForField(LuceneFields.L_CONTACT_DATA, "");
            checkNextTerm(fields, new Term(LuceneFields.L_CONTACT_DATA, "@zimbra"));
            checkNextTerm(fields, new Term(LuceneFields.L_CONTACT_DATA, "@zimbra.com"));
            checkNextTerm(fields, new Term(LuceneFields.L_CONTACT_DATA, "test1"));
            checkNextTerm(fields, new Term(LuceneFields.L_CONTACT_DATA, "test1@zimbra.com"));
            checkNextTerm(fields, new Term(LuceneFields.L_CONTACT_DATA, "test2"));
            checkNextTerm(fields, new Term(LuceneFields.L_CONTACT_DATA, "test2@zimbra.com"));
            checkNextTerm(fields, new Term(LuceneFields.L_CONTACT_DATA, "zimbra"));
            checkNextTerm(fields, new Term(LuceneFields.L_CONTACT_DATA, "zimbra.com"));
            checkAtEnd(fields, LuceneFields.L_CONTACT_DATA);
        } finally {
            Closeables.closeQuietly(fields);
        }
        fields = null;
        try {
            // l.content values:
            // "test1@zimbra.com test1 @zimbra.com zimbra.com zimbra @zimbra  "
            // "test2@zimbra.com test2 @zimbra.com zimbra.com zimbra @zimbra  "
            fields = searcher.getIndexReader().getTermsForField(LuceneFields.L_CONTENT, "");
            checkNextTerm(fields, new Term(LuceneFields.L_CONTENT, "test1"));
            checkNextTerm(fields, new Term(LuceneFields.L_CONTENT, "test1@zimbra.com"));
            checkNextTerm(fields, new Term(LuceneFields.L_CONTENT, "test2"));
            checkNextTerm(fields, new Term(LuceneFields.L_CONTENT, "test2@zimbra.com"));
            checkNextTerm(fields, new Term(LuceneFields.L_CONTENT, "zimbra"));
            checkNextTerm(fields, new Term(LuceneFields.L_CONTENT, "zimbra.com"));
            checkAtEnd(fields, LuceneFields.L_CONTENT);
        } finally {
            Closeables.closeQuietly(fields);
        }
        fields = null;
        try {
            fields = searcher.getIndexReader().getTermsForField(LuceneFields.L_FIELD, "");
            checkNextTerm(fields, new Term(LuceneFields.L_FIELD, "email:test1@zimbra.com"));
            checkNextTerm(fields, new Term(LuceneFields.L_FIELD, "email:test2@zimbra.com"));
            checkAtEnd(fields, LuceneFields.L_FIELD);
        } finally {
            Closeables.closeQuietly(fields);
        }
        fields = null;
        try {
            fields = searcher.getIndexReader().getTermsForField(LuceneFields.L_PARTNAME, "");
            checkNextTerm(fields, new Term(LuceneFields.L_PARTNAME, "CONTACT"));
            checkAtEnd(fields, LuceneFields.L_PARTNAME);
        } finally {
            Closeables.closeQuietly(fields);
        }
        fields = null;
        try {
            fields = searcher.getIndexReader().getTermsForField(LuceneFields.L_H_TO, "");
            checkNextTerm(fields, new Term(LuceneFields.L_H_TO, "@zimbra"));
            checkNextTerm(fields, new Term(LuceneFields.L_H_TO, "@zimbra.com"));
            checkNextTerm(fields, new Term(LuceneFields.L_H_TO, "test1"));
            checkNextTerm(fields, new Term(LuceneFields.L_H_TO, "test1@zimbra.com"));
            checkNextTerm(fields, new Term(LuceneFields.L_H_TO, "test2"));
            checkNextTerm(fields, new Term(LuceneFields.L_H_TO, "test2@zimbra.com"));
            checkNextTerm(fields, new Term(LuceneFields.L_H_TO, "zimbra"));
            checkNextTerm(fields, new Term(LuceneFields.L_H_TO, "zimbra.com"));
            checkAtEnd(fields, LuceneFields.L_H_TO);
        } finally {
            Closeables.closeQuietly(fields);
        }
        fields = null;
        try {
            fields = searcher.getIndexReader().getTermsForField(LuceneFields.L_H_TO, "tess");
            checkNextTerm(fields, new Term(LuceneFields.L_H_TO, "test1"));
            checkNextTerm(fields, new Term(LuceneFields.L_H_TO, "test1@zimbra.com"));
            checkNextTerm(fields, new Term(LuceneFields.L_H_TO, "test2"));
            checkNextTerm(fields, new Term(LuceneFields.L_H_TO, "test2@zimbra.com"));
            checkNextTerm(fields, new Term(LuceneFields.L_H_TO, "zimbra"));
            checkNextTerm(fields, new Term(LuceneFields.L_H_TO, "zimbra.com"));
            checkAtEnd(fields, LuceneFields.L_H_TO + "(sublist)");
        } finally {
            Closeables.closeQuietly(fields);
        }
        fields = null;
        try {
            fields = searcher.getIndexReader().getTermsForField(LuceneFields.L_SORT_DATE, "");
            checkNextTermFieldType(fields, LuceneFields.L_SORT_DATE);
            // TODO:  ElasticSearch has more.  Not sure why and not sure it matters
            // checkAtEnd(fields, LuceneFields.L_SORT_DATE);
        } finally {
            Closeables.closeQuietly(fields);
        }
        fields = null;
        try {
            fields = searcher.getIndexReader().getTermsForField(LuceneFields.L_MAILBOX_BLOB_ID, "");
            checkNextTermFieldType(fields, LuceneFields.L_MAILBOX_BLOB_ID);
            checkNextTermFieldType(fields, LuceneFields.L_MAILBOX_BLOB_ID);
            // TODO:  ElasticSearch has more.  Investigate?  Believe it relates to fact that is a number field
            // Numbers have an associated precision step (number of terms generated for each number value)
            // which defaults to 4.
            // checkAtEnd(fields, LuceneFields.L_MAILBOX_BLOB_ID);
        } finally {
            Closeables.closeQuietly(fields);
        }
        searcher.close();
    }

    private Contact createContact(Mailbox mbox, String email) throws ServiceException {
        Folder folder = mbox.getFolderById(null, Mailbox.ID_FOLDER_CONTACTS);
        return mbox.createContact(null,
                new ParsedContact(Collections.singletonMap(ContactConstants.A_email, email)), folder.getId(), null);
    }

    private Contact createContact(Mailbox mbox, String firstName, String lastName, String email)
            throws ServiceException {
        Folder folder = mbox.getFolderById(null, Mailbox.ID_FOLDER_CONTACTS);
        Map<String, Object> fields;
        fields = ImmutableMap.<String, Object>of(ContactConstants.A_firstName, firstName,
                ContactConstants.A_lastName, lastName, ContactConstants.A_email, email);
        return mbox.createContact(null, new ParsedContact(fields), folder.getId(), null);
    }

    private Contact createContact(Mailbox mbox, String firstName, String lastName, String email, String jobTitle)
            throws ServiceException {
        Folder folder = mbox.getFolderById(null, Mailbox.ID_FOLDER_CONTACTS);
        Map<String, Object> fields;
        fields = ImmutableMap.<String, Object>of(ContactConstants.A_firstName, firstName,
                ContactConstants.A_lastName, lastName, ContactConstants.A_jobTitle, jobTitle,
                ContactConstants.A_email, email);
        return mbox.createContact(null, new ParsedContact(fields), folder.getId(), null);
    }

    private static String getBlobIdForResultDoc(ZimbraIndexSearcher searcher, ZimbraTopDocs result, int index)
            throws IOException {
        return searcher.doc(result.getScoreDoc(index).getDocumentID()).get(LuceneFields.L_MAILBOX_BLOB_ID);
    }

}