org.elasticsearch.test.unit.index.fielddata.StringFieldDataTests.java Source code

Java tutorial

Introduction

Here is the source code for org.elasticsearch.test.unit.index.fielddata.StringFieldDataTests.java

Source

/*
 * Licensed to ElasticSearch and Shay Banon under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership. ElasticSearch licenses this
 * file to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.elasticsearch.test.unit.index.fielddata;

import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.search.*;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.lucene.HashedBytesRef;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.fieldcomparator.SortMode;
import org.elasticsearch.index.fielddata.util.BytesRefArrayRef;
import org.elasticsearch.index.fielddata.util.StringArrayRef;
import org.testng.annotations.Test;

import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.nullValue;

/**
 */
@Test
public abstract class StringFieldDataTests extends AbstractFieldDataTests {

    protected String one() {
        return "1";
    }

    protected String two() {
        return "2";
    }

    protected String three() {
        return "3";
    }

    protected String four() {
        return "4";
    }

    private String toString(Object value) {
        if (value instanceof BytesRef) {
            return ((BytesRef) value).utf8ToString();
        }
        return value.toString();
    }

    protected void fillSingleValueAllSet() throws Exception {
        Document d = new Document();
        d.add(new StringField("_id", "1", Field.Store.NO));
        d.add(new StringField("value", "2", Field.Store.NO));
        writer.addDocument(d);

        d = new Document();
        d.add(new StringField("_id", "1", Field.Store.NO));
        d.add(new StringField("value", "1", Field.Store.NO));
        writer.addDocument(d);

        d = new Document();
        d.add(new StringField("_id", "3", Field.Store.NO));
        d.add(new StringField("value", "3", Field.Store.NO));
        writer.addDocument(d);
    }

    @Test
    public void testSingleValueAllSet() throws Exception {
        fillSingleValueAllSet();
        IndexFieldData indexFieldData = getForField("value");
        AtomicReaderContext readerContext = refreshReader();
        AtomicFieldData fieldData = indexFieldData.load(readerContext);

        assertThat(fieldData.getNumDocs(), equalTo(3));

        BytesValues bytesValues = fieldData.getBytesValues();

        assertThat(bytesValues.isMultiValued(), equalTo(false));

        assertThat(bytesValues.hasValue(0), equalTo(true));
        assertThat(bytesValues.hasValue(1), equalTo(true));
        assertThat(bytesValues.hasValue(2), equalTo(true));

        assertThat(bytesValues.getValue(0), equalTo(new BytesRef(two())));
        assertThat(bytesValues.getValue(1), equalTo(new BytesRef(one())));
        assertThat(bytesValues.getValue(2), equalTo(new BytesRef(three())));

        BytesRef bytesRef = new BytesRef();
        assertThat(bytesValues.getValueScratch(0, bytesRef), equalTo(new BytesRef(two())));
        assertThat(bytesRef, equalTo(new BytesRef(two())));
        assertThat(bytesValues.getValueScratch(1, bytesRef), equalTo(new BytesRef(one())));
        assertThat(bytesRef, equalTo(new BytesRef(one())));
        assertThat(bytesValues.getValueScratch(2, bytesRef), equalTo(new BytesRef(three())));
        assertThat(bytesRef, equalTo(new BytesRef(three())));

        BytesRefArrayRef bytesRefArrayRef = bytesValues.getValues(0);
        assertThat(bytesRefArrayRef.size(), equalTo(1));
        assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(two())));

        bytesRefArrayRef = bytesValues.getValues(1);
        assertThat(bytesRefArrayRef.size(), equalTo(1));
        assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(one())));

        bytesRefArrayRef = bytesValues.getValues(2);
        assertThat(bytesRefArrayRef.size(), equalTo(1));
        assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(three())));

        BytesValues.Iter bytesValuesIter = bytesValues.getIter(0);
        assertThat(bytesValuesIter.hasNext(), equalTo(true));
        assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two())));
        assertThat(bytesValuesIter.hasNext(), equalTo(false));

        bytesValues.forEachValueInDoc(0, new BytesValuesVerifierProc(0).addExpected(two()));
        bytesValues.forEachValueInDoc(1, new BytesValuesVerifierProc(1).addExpected(one()));
        bytesValues.forEachValueInDoc(2, new BytesValuesVerifierProc(2).addExpected(three()));

        HashedBytesValues hashedBytesValues = fieldData.getHashedBytesValues();

        assertThat(hashedBytesValues.hasValue(0), equalTo(true));
        assertThat(hashedBytesValues.hasValue(1), equalTo(true));
        assertThat(hashedBytesValues.hasValue(2), equalTo(true));

        assertThat(hashedBytesValues.getValue(0), equalTo(new HashedBytesRef(two())));
        assertThat(hashedBytesValues.getValue(1), equalTo(new HashedBytesRef(one())));
        assertThat(hashedBytesValues.getValue(2), equalTo(new HashedBytesRef(three())));

        HashedBytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0);
        assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
        assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(two())));
        assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));

        hashedBytesValues.forEachValueInDoc(0, new HashedBytesValuesVerifierProc(0).addExpected(two()));
        hashedBytesValues.forEachValueInDoc(1, new HashedBytesValuesVerifierProc(1).addExpected(one()));
        hashedBytesValues.forEachValueInDoc(2, new HashedBytesValuesVerifierProc(2).addExpected(three()));

        StringValues stringValues = fieldData.getStringValues();

        assertThat(stringValues.hasValue(0), equalTo(true));
        assertThat(stringValues.hasValue(1), equalTo(true));
        assertThat(stringValues.hasValue(2), equalTo(true));

        assertThat(stringValues.getValue(0), equalTo(two()));
        assertThat(stringValues.getValue(1), equalTo(one()));
        assertThat(stringValues.getValue(2), equalTo(three()));

        StringArrayRef stringArrayRef;
        stringArrayRef = stringValues.getValues(0);
        assertThat(stringArrayRef.size(), equalTo(1));
        assertThat(stringArrayRef.values[stringArrayRef.start], equalTo(two()));

        stringArrayRef = stringValues.getValues(1);
        assertThat(stringArrayRef.size(), equalTo(1));
        assertThat(stringArrayRef.values[stringArrayRef.start], equalTo(one()));

        stringArrayRef = stringValues.getValues(2);
        assertThat(stringArrayRef.size(), equalTo(1));
        assertThat(stringArrayRef.values[stringArrayRef.start], equalTo(three()));

        StringValues.Iter stringValuesIter = stringValues.getIter(0);
        assertThat(stringValuesIter.hasNext(), equalTo(true));
        assertThat(stringValuesIter.next(), equalTo(two()));
        assertThat(stringValuesIter.hasNext(), equalTo(false));

        stringValuesIter = stringValues.getIter(1);
        assertThat(stringValuesIter.hasNext(), equalTo(true));
        assertThat(stringValuesIter.next(), equalTo(one()));
        assertThat(stringValuesIter.hasNext(), equalTo(false));

        stringValuesIter = stringValues.getIter(2);
        assertThat(stringValuesIter.hasNext(), equalTo(true));
        assertThat(stringValuesIter.next(), equalTo(three()));
        assertThat(stringValuesIter.hasNext(), equalTo(false));

        stringValues.forEachValueInDoc(0, new StringValuesVerifierProc(0).addExpected(two()));
        stringValues.forEachValueInDoc(1, new StringValuesVerifierProc(1).addExpected(one()));
        stringValues.forEachValueInDoc(2, new StringValuesVerifierProc(2).addExpected(three()));

        IndexSearcher searcher = new IndexSearcher(readerContext.reader());
        TopFieldDocs topDocs;

        topDocs = searcher.search(new MatchAllDocsQuery(), 10,
                new Sort(new SortField("value", indexFieldData.comparatorSource(null, SortMode.MIN))));
        assertThat(topDocs.totalHits, equalTo(3));
        assertThat(topDocs.scoreDocs[0].doc, equalTo(1));
        assertThat(toString(((FieldDoc) topDocs.scoreDocs[0]).fields[0]), equalTo(one()));
        assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
        assertThat(toString(((FieldDoc) topDocs.scoreDocs[1]).fields[0]), equalTo(two()));
        assertThat(topDocs.scoreDocs[2].doc, equalTo(2));
        assertThat(toString(((FieldDoc) topDocs.scoreDocs[2]).fields[0]), equalTo(three()));

        topDocs = searcher.search(new MatchAllDocsQuery(), 10,
                new Sort(new SortField("value", indexFieldData.comparatorSource(null, SortMode.MAX), true)));
        assertThat(topDocs.totalHits, equalTo(3));
        assertThat(topDocs.scoreDocs[0].doc, equalTo(2));
        assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
        assertThat(topDocs.scoreDocs[2].doc, equalTo(1));
    }

    protected void fillSingleValueWithMissing() throws Exception {
        Document d = new Document();
        d.add(new StringField("_id", "1", Field.Store.NO));
        d.add(new StringField("value", "2", Field.Store.NO));
        writer.addDocument(d);

        d = new Document();
        d.add(new StringField("_id", "2", Field.Store.NO));
        //d.add(new StringField("value", one(), Field.Store.NO)); // MISSING....
        writer.addDocument(d);

        d = new Document();
        d.add(new StringField("_id", "3", Field.Store.NO));
        d.add(new StringField("value", "3", Field.Store.NO));
        writer.addDocument(d);
    }

    @Test
    public void testSingleValueWithMissing() throws Exception {
        fillSingleValueWithMissing();
        IndexFieldData indexFieldData = getForField("value");
        AtomicFieldData fieldData = indexFieldData.load(refreshReader());

        assertThat(fieldData.getNumDocs(), equalTo(3));

        BytesValues bytesValues = fieldData.getBytesValues();

        assertThat(bytesValues.isMultiValued(), equalTo(false));

        assertThat(bytesValues.hasValue(0), equalTo(true));
        assertThat(bytesValues.hasValue(1), equalTo(false));
        assertThat(bytesValues.hasValue(2), equalTo(true));

        assertThat(bytesValues.getValue(0), equalTo(new BytesRef(two())));
        assertThat(bytesValues.getValue(1), nullValue());
        assertThat(bytesValues.getValue(2), equalTo(new BytesRef(three())));

        BytesRef bytesRef = new BytesRef();
        assertThat(bytesValues.getValueScratch(0, bytesRef), equalTo(new BytesRef(two())));
        assertThat(bytesRef, equalTo(new BytesRef(two())));
        assertThat(bytesValues.getValueScratch(1, bytesRef), equalTo(new BytesRef()));
        assertThat(bytesRef, equalTo(new BytesRef()));
        assertThat(bytesValues.getValueScratch(2, bytesRef), equalTo(new BytesRef(three())));
        assertThat(bytesRef, equalTo(new BytesRef(three())));

        BytesRefArrayRef bytesRefArrayRef = bytesValues.getValues(0);
        assertThat(bytesRefArrayRef.size(), equalTo(1));
        assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(two())));

        bytesRefArrayRef = bytesValues.getValues(1);
        assertThat(bytesRefArrayRef.size(), equalTo(0));

        bytesRefArrayRef = bytesValues.getValues(2);
        assertThat(bytesRefArrayRef.size(), equalTo(1));
        assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(three())));

        BytesValues.Iter bytesValuesIter = bytesValues.getIter(0);
        assertThat(bytesValuesIter.hasNext(), equalTo(true));
        assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two())));
        assertThat(bytesValuesIter.hasNext(), equalTo(false));

        bytesValuesIter = bytesValues.getIter(1);
        assertThat(bytesValuesIter.hasNext(), equalTo(false));

        bytesValues.forEachValueInDoc(0, new BytesValuesVerifierProc(0).addExpected(two()));
        bytesValues.forEachValueInDoc(1, new BytesValuesVerifierProc(1).addMissing());
        bytesValues.forEachValueInDoc(2, new BytesValuesVerifierProc(2).addExpected(three()));

        HashedBytesValues hashedBytesValues = fieldData.getHashedBytesValues();

        assertThat(hashedBytesValues.hasValue(0), equalTo(true));
        assertThat(hashedBytesValues.hasValue(1), equalTo(false));
        assertThat(hashedBytesValues.hasValue(2), equalTo(true));

        assertThat(hashedBytesValues.getValue(0), equalTo(new HashedBytesRef(two())));
        assertThat(hashedBytesValues.getValue(1), nullValue());
        assertThat(hashedBytesValues.getValue(2), equalTo(new HashedBytesRef(three())));

        HashedBytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0);
        assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
        assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(two())));
        assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));

        hashedBytesValuesIter = hashedBytesValues.getIter(1);
        assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));

        hashedBytesValues.forEachValueInDoc(0, new HashedBytesValuesVerifierProc(0).addExpected(two()));
        hashedBytesValues.forEachValueInDoc(1, new HashedBytesValuesVerifierProc(1).addMissing());
        hashedBytesValues.forEachValueInDoc(2, new HashedBytesValuesVerifierProc(2).addExpected(three()));

        StringValues stringValues = fieldData.getStringValues();

        assertThat(stringValues.hasValue(0), equalTo(true));
        assertThat(stringValues.hasValue(1), equalTo(false));
        assertThat(stringValues.hasValue(2), equalTo(true));

        assertThat(stringValues.getValue(0), equalTo(two()));
        assertThat(stringValues.getValue(1), nullValue());
        assertThat(stringValues.getValue(2), equalTo(three()));

        StringArrayRef stringArrayRef;
        stringArrayRef = stringValues.getValues(0);
        assertThat(stringArrayRef.size(), equalTo(1));
        assertThat(stringArrayRef.values[stringArrayRef.start], equalTo(two()));

        stringArrayRef = stringValues.getValues(1);
        assertThat(stringArrayRef.size(), equalTo(0));

        stringArrayRef = stringValues.getValues(2);
        assertThat(stringArrayRef.size(), equalTo(1));
        assertThat(stringArrayRef.values[stringArrayRef.start], equalTo(three()));

        StringValues.Iter stringValuesIter = stringValues.getIter(0);
        assertThat(stringValuesIter.hasNext(), equalTo(true));
        assertThat(stringValuesIter.next(), equalTo(two()));
        assertThat(stringValuesIter.hasNext(), equalTo(false));

        stringValuesIter = stringValues.getIter(1);
        assertThat(stringValuesIter.hasNext(), equalTo(false));

        stringValuesIter = stringValues.getIter(2);
        assertThat(stringValuesIter.hasNext(), equalTo(true));
        assertThat(stringValuesIter.next(), equalTo(three()));
        assertThat(stringValuesIter.hasNext(), equalTo(false));

        stringValues.forEachValueInDoc(0, new StringValuesVerifierProc(0).addExpected(two()));
        stringValues.forEachValueInDoc(1, new StringValuesVerifierProc(1).addMissing());
        stringValues.forEachValueInDoc(2, new StringValuesVerifierProc(2).addExpected(three()));

        // TODO properly support missing....
    }

    protected void fillMultiValueAllSet() throws Exception {
        Document d = new Document();
        d.add(new StringField("_id", "1", Field.Store.NO));
        d.add(new StringField("value", "2", Field.Store.NO));
        d.add(new StringField("value", "4", Field.Store.NO));
        writer.addDocument(d);

        d = new Document();
        d.add(new StringField("_id", "2", Field.Store.NO));
        d.add(new StringField("value", "1", Field.Store.NO));
        writer.addDocument(d);
        writer.commit(); // TODO: Have tests with more docs for sorting

        d = new Document();
        d.add(new StringField("_id", "3", Field.Store.NO));
        d.add(new StringField("value", "3", Field.Store.NO));
        writer.addDocument(d);
    }

    @Test
    public void testMultiValueAllSet() throws Exception {
        fillMultiValueAllSet();
        IndexFieldData indexFieldData = getForField("value");
        AtomicFieldData fieldData = indexFieldData.load(refreshReader());

        assertThat(fieldData.getNumDocs(), equalTo(3));

        BytesValues bytesValues = fieldData.getBytesValues();

        assertThat(bytesValues.isMultiValued(), equalTo(true));

        assertThat(bytesValues.hasValue(0), equalTo(true));
        assertThat(bytesValues.hasValue(1), equalTo(true));
        assertThat(bytesValues.hasValue(2), equalTo(true));

        assertThat(bytesValues.getValue(0), equalTo(new BytesRef(two())));
        assertThat(bytesValues.getValue(1), equalTo(new BytesRef(one())));
        assertThat(bytesValues.getValue(2), equalTo(new BytesRef(three())));

        BytesRef bytesRef = new BytesRef();
        assertThat(bytesValues.getValueScratch(0, bytesRef), equalTo(new BytesRef(two())));
        assertThat(bytesRef, equalTo(new BytesRef(two())));
        assertThat(bytesValues.getValueScratch(1, bytesRef), equalTo(new BytesRef(one())));
        assertThat(bytesRef, equalTo(new BytesRef(one())));
        assertThat(bytesValues.getValueScratch(2, bytesRef), equalTo(new BytesRef(three())));
        assertThat(bytesRef, equalTo(new BytesRef(three())));

        BytesRefArrayRef bytesRefArrayRef = bytesValues.getValues(0);
        assertThat(bytesRefArrayRef.size(), equalTo(2));
        assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(two())));
        assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start + 1], equalTo(new BytesRef(four())));

        bytesRefArrayRef = bytesValues.getValues(1);
        assertThat(bytesRefArrayRef.size(), equalTo(1));
        assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(one())));

        bytesRefArrayRef = bytesValues.getValues(2);
        assertThat(bytesRefArrayRef.size(), equalTo(1));
        assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(three())));

        BytesValues.Iter bytesValuesIter = bytesValues.getIter(0);
        assertThat(bytesValuesIter.hasNext(), equalTo(true));
        assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two())));
        assertThat(bytesValuesIter.hasNext(), equalTo(true));
        assertThat(bytesValuesIter.next(), equalTo(new BytesRef(four())));
        assertThat(bytesValuesIter.hasNext(), equalTo(false));

        bytesValues.forEachValueInDoc(0, new BytesValuesVerifierProc(0).addExpected(two()).addExpected(four()));
        bytesValues.forEachValueInDoc(1, new BytesValuesVerifierProc(1).addExpected(one()));
        bytesValues.forEachValueInDoc(2, new BytesValuesVerifierProc(2).addExpected(three()));

        HashedBytesValues hashedBytesValues = fieldData.getHashedBytesValues();

        assertThat(hashedBytesValues.hasValue(0), equalTo(true));
        assertThat(hashedBytesValues.hasValue(1), equalTo(true));
        assertThat(hashedBytesValues.hasValue(2), equalTo(true));

        assertThat(hashedBytesValues.getValue(0), equalTo(new HashedBytesRef(two())));
        assertThat(hashedBytesValues.getValue(1), equalTo(new HashedBytesRef(one())));
        assertThat(hashedBytesValues.getValue(2), equalTo(new HashedBytesRef(three())));

        HashedBytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0);
        assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
        assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(two())));
        assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
        assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(four())));
        assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));

        hashedBytesValues.forEachValueInDoc(0,
                new HashedBytesValuesVerifierProc(0).addExpected(two()).addExpected(four()));
        hashedBytesValues.forEachValueInDoc(1, new HashedBytesValuesVerifierProc(1).addExpected(one()));
        hashedBytesValues.forEachValueInDoc(2, new HashedBytesValuesVerifierProc(2).addExpected(three()));

        StringValues stringValues = fieldData.getStringValues();

        assertThat(stringValues.hasValue(0), equalTo(true));
        assertThat(stringValues.hasValue(1), equalTo(true));
        assertThat(stringValues.hasValue(2), equalTo(true));

        assertThat(stringValues.getValue(0), equalTo(two()));
        assertThat(stringValues.getValue(1), equalTo(one()));
        assertThat(stringValues.getValue(2), equalTo(three()));

        StringArrayRef stringArrayRef;
        stringArrayRef = stringValues.getValues(0);
        assertThat(stringArrayRef.size(), equalTo(2));
        assertThat(stringArrayRef.values[stringArrayRef.start], equalTo(two()));
        assertThat(stringArrayRef.values[stringArrayRef.start + 1], equalTo(four()));

        stringArrayRef = stringValues.getValues(1);
        assertThat(stringArrayRef.size(), equalTo(1));
        assertThat(stringArrayRef.values[stringArrayRef.start], equalTo(one()));

        stringArrayRef = stringValues.getValues(2);
        assertThat(stringArrayRef.size(), equalTo(1));
        assertThat(stringArrayRef.values[stringArrayRef.start], equalTo(three()));

        StringValues.Iter stringValuesIter = stringValues.getIter(0);
        assertThat(stringValuesIter.hasNext(), equalTo(true));
        assertThat(stringValuesIter.next(), equalTo(two()));
        assertThat(stringValuesIter.hasNext(), equalTo(true));
        assertThat(stringValuesIter.next(), equalTo(four()));
        assertThat(stringValuesIter.hasNext(), equalTo(false));

        stringValuesIter = stringValues.getIter(1);
        assertThat(stringValuesIter.hasNext(), equalTo(true));
        assertThat(stringValuesIter.next(), equalTo(one()));
        assertThat(stringValuesIter.hasNext(), equalTo(false));

        stringValuesIter = stringValues.getIter(2);
        assertThat(stringValuesIter.hasNext(), equalTo(true));
        assertThat(stringValuesIter.next(), equalTo(three()));
        assertThat(stringValuesIter.hasNext(), equalTo(false));

        stringValues.forEachValueInDoc(0, new StringValuesVerifierProc(0).addExpected(two()).addExpected(four()));
        stringValues.forEachValueInDoc(1, new StringValuesVerifierProc(1).addExpected(one()));
        stringValues.forEachValueInDoc(2, new StringValuesVerifierProc(2).addExpected(three()));

        IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, true));
        TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10,
                new Sort(new SortField("value", indexFieldData.comparatorSource(null, SortMode.MIN))));
        assertThat(topDocs.totalHits, equalTo(3));
        assertThat(topDocs.scoreDocs.length, equalTo(3));
        assertThat(topDocs.scoreDocs[0].doc, equalTo(1));
        assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
        assertThat(topDocs.scoreDocs[2].doc, equalTo(2));

        topDocs = searcher.search(new MatchAllDocsQuery(), 10,
                new Sort(new SortField("value", indexFieldData.comparatorSource(null, SortMode.MAX), true)));
        assertThat(topDocs.totalHits, equalTo(3));
        assertThat(topDocs.scoreDocs.length, equalTo(3));
        assertThat(topDocs.scoreDocs[0].doc, equalTo(0));
        assertThat(topDocs.scoreDocs[1].doc, equalTo(2));
        assertThat(topDocs.scoreDocs[2].doc, equalTo(1));
    }

    protected void fillMultiValueWithMissing() throws Exception {
        Document d = new Document();
        d.add(new StringField("_id", "1", Field.Store.NO));
        d.add(new StringField("value", "2", Field.Store.NO));
        d.add(new StringField("value", "4", Field.Store.NO));
        writer.addDocument(d);

        d = new Document();
        d.add(new StringField("_id", "2", Field.Store.NO));
        //d.add(new StringField("value", one(), Field.Store.NO)); // MISSING
        writer.addDocument(d);

        d = new Document();
        d.add(new StringField("_id", "3", Field.Store.NO));
        d.add(new StringField("value", "3", Field.Store.NO));
        writer.addDocument(d);
    }

    @Test
    public void testMultiValueWithMissing() throws Exception {
        fillMultiValueWithMissing();
        IndexFieldData indexFieldData = getForField("value");
        AtomicFieldData fieldData = indexFieldData.load(refreshReader());

        assertThat(fieldData.getNumDocs(), equalTo(3));

        BytesValues bytesValues = fieldData.getBytesValues();

        assertThat(bytesValues.isMultiValued(), equalTo(true));

        assertThat(bytesValues.hasValue(0), equalTo(true));
        assertThat(bytesValues.hasValue(1), equalTo(false));
        assertThat(bytesValues.hasValue(2), equalTo(true));

        assertThat(bytesValues.getValue(0), equalTo(new BytesRef(two())));
        assertThat(bytesValues.getValue(1), nullValue());
        assertThat(bytesValues.getValue(2), equalTo(new BytesRef(three())));

        BytesRef bytesRef = new BytesRef();
        assertThat(bytesValues.getValueScratch(0, bytesRef), equalTo(new BytesRef(two())));
        assertThat(bytesRef, equalTo(new BytesRef(two())));
        assertThat(bytesValues.getValueScratch(1, bytesRef), equalTo(new BytesRef()));
        assertThat(bytesRef, equalTo(new BytesRef()));
        assertThat(bytesValues.getValueScratch(2, bytesRef), equalTo(new BytesRef(three())));
        assertThat(bytesRef, equalTo(new BytesRef(three())));

        BytesRefArrayRef bytesRefArrayRef = bytesValues.getValues(0);
        assertThat(bytesRefArrayRef.size(), equalTo(2));
        assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(two())));
        assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start + 1], equalTo(new BytesRef(four())));

        bytesRefArrayRef = bytesValues.getValues(1);
        assertThat(bytesRefArrayRef.size(), equalTo(0));

        bytesRefArrayRef = bytesValues.getValues(2);
        assertThat(bytesRefArrayRef.size(), equalTo(1));
        assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(three())));

        BytesValues.Iter bytesValuesIter = bytesValues.getIter(0);
        assertThat(bytesValuesIter.hasNext(), equalTo(true));
        assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two())));
        assertThat(bytesValuesIter.hasNext(), equalTo(true));
        assertThat(bytesValuesIter.next(), equalTo(new BytesRef(four())));
        assertThat(bytesValuesIter.hasNext(), equalTo(false));

        bytesValuesIter = bytesValues.getIter(1);
        assertThat(bytesValuesIter.hasNext(), equalTo(false));

        bytesValues.forEachValueInDoc(0, new BytesValuesVerifierProc(0).addExpected(two()).addExpected(four()));
        bytesValues.forEachValueInDoc(1, new BytesValuesVerifierProc(1).addMissing());
        bytesValues.forEachValueInDoc(2, new BytesValuesVerifierProc(2).addExpected(three()));

        HashedBytesValues hashedBytesValues = fieldData.getHashedBytesValues();

        assertThat(hashedBytesValues.hasValue(0), equalTo(true));
        assertThat(hashedBytesValues.hasValue(1), equalTo(false));
        assertThat(hashedBytesValues.hasValue(2), equalTo(true));

        assertThat(hashedBytesValues.getValue(0), equalTo(new HashedBytesRef(two())));
        assertThat(hashedBytesValues.getValue(1), nullValue());
        assertThat(hashedBytesValues.getValue(2), equalTo(new HashedBytesRef(three())));

        HashedBytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0);
        assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
        assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(two())));
        assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
        assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(four())));
        assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));

        hashedBytesValuesIter = hashedBytesValues.getIter(1);
        assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));

        hashedBytesValues.forEachValueInDoc(0,
                new HashedBytesValuesVerifierProc(0).addExpected(two()).addExpected(four()));
        hashedBytesValues.forEachValueInDoc(1, new HashedBytesValuesVerifierProc(1).addMissing());
        hashedBytesValues.forEachValueInDoc(2, new HashedBytesValuesVerifierProc(2).addExpected(three()));

        StringValues stringValues = fieldData.getStringValues();

        assertThat(stringValues.hasValue(0), equalTo(true));
        assertThat(stringValues.hasValue(1), equalTo(false));
        assertThat(stringValues.hasValue(2), equalTo(true));

        assertThat(stringValues.getValue(0), equalTo(two()));
        assertThat(stringValues.getValue(1), nullValue());
        assertThat(stringValues.getValue(2), equalTo(three()));

        StringArrayRef stringArrayRef;
        stringArrayRef = stringValues.getValues(0);
        assertThat(stringArrayRef.size(), equalTo(2));
        assertThat(stringArrayRef.values[stringArrayRef.start], equalTo(two()));
        assertThat(stringArrayRef.values[stringArrayRef.start + 1], equalTo(four()));

        stringArrayRef = stringValues.getValues(1);
        assertThat(stringArrayRef.size(), equalTo(0));

        stringArrayRef = stringValues.getValues(2);
        assertThat(stringArrayRef.size(), equalTo(1));
        assertThat(stringArrayRef.values[stringArrayRef.start], equalTo(three()));

        StringValues.Iter stringValuesIter = stringValues.getIter(0);
        assertThat(stringValuesIter.hasNext(), equalTo(true));
        assertThat(stringValuesIter.next(), equalTo(two()));
        assertThat(stringValuesIter.hasNext(), equalTo(true));
        assertThat(stringValuesIter.next(), equalTo(four()));
        assertThat(stringValuesIter.hasNext(), equalTo(false));

        stringValuesIter = stringValues.getIter(1);
        assertThat(stringValuesIter.hasNext(), equalTo(false));

        stringValuesIter = stringValues.getIter(2);
        assertThat(stringValuesIter.hasNext(), equalTo(true));
        assertThat(stringValuesIter.next(), equalTo(three()));
        assertThat(stringValuesIter.hasNext(), equalTo(false));

        stringValues.forEachValueInDoc(0, new StringValuesVerifierProc(0).addExpected(two()).addExpected(four()));
        stringValues.forEachValueInDoc(1, new StringValuesVerifierProc(1).addMissing());
        stringValues.forEachValueInDoc(2, new StringValuesVerifierProc(2).addExpected(three()));
    }

    public void testMissingValueForAll() throws Exception {
        fillAllMissing();
        IndexFieldData indexFieldData = getForField("value");
        AtomicFieldData fieldData = indexFieldData.load(refreshReader());

        assertThat(fieldData.getNumDocs(), equalTo(3));

        BytesValues bytesValues = fieldData.getBytesValues();

        assertThat(bytesValues.isMultiValued(), equalTo(false));

        assertThat(bytesValues.hasValue(0), equalTo(false));
        assertThat(bytesValues.hasValue(1), equalTo(false));
        assertThat(bytesValues.hasValue(2), equalTo(false));

        assertThat(bytesValues.getValue(0), nullValue());
        assertThat(bytesValues.getValue(1), nullValue());
        assertThat(bytesValues.getValue(2), nullValue());

        BytesRef bytesRef = new BytesRef();
        assertThat(bytesValues.getValueScratch(0, bytesRef), equalTo(new BytesRef()));
        assertThat(bytesRef, equalTo(new BytesRef()));
        assertThat(bytesValues.getValueScratch(1, bytesRef), equalTo(new BytesRef()));
        assertThat(bytesRef, equalTo(new BytesRef()));
        assertThat(bytesValues.getValueScratch(2, bytesRef), equalTo(new BytesRef()));
        assertThat(bytesRef, equalTo(new BytesRef()));

        BytesRefArrayRef bytesRefArrayRef = bytesValues.getValues(0);
        assertThat(bytesRefArrayRef.size(), equalTo(0));

        bytesRefArrayRef = bytesValues.getValues(1);
        assertThat(bytesRefArrayRef.size(), equalTo(0));

        bytesRefArrayRef = bytesValues.getValues(2);
        assertThat(bytesRefArrayRef.size(), equalTo(0));

        BytesValues.Iter bytesValuesIter = bytesValues.getIter(0);
        assertThat(bytesValuesIter.hasNext(), equalTo(false));

        bytesValuesIter = bytesValues.getIter(1);
        assertThat(bytesValuesIter.hasNext(), equalTo(false));

        bytesValuesIter = bytesValues.getIter(2);
        assertThat(bytesValuesIter.hasNext(), equalTo(false));

        bytesValues.forEachValueInDoc(0, new BytesValuesVerifierProc(0).addMissing());
        bytesValues.forEachValueInDoc(1, new BytesValuesVerifierProc(1).addMissing());
        bytesValues.forEachValueInDoc(2, new BytesValuesVerifierProc(2).addMissing());

        HashedBytesValues hashedBytesValues = fieldData.getHashedBytesValues();

        assertThat(hashedBytesValues.hasValue(0), equalTo(false));
        assertThat(hashedBytesValues.hasValue(1), equalTo(false));
        assertThat(hashedBytesValues.hasValue(2), equalTo(false));

        assertThat(hashedBytesValues.getValue(0), nullValue());
        assertThat(hashedBytesValues.getValue(1), nullValue());
        assertThat(hashedBytesValues.getValue(2), nullValue());

        HashedBytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0);
        assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));

        hashedBytesValuesIter = hashedBytesValues.getIter(1);
        assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));

        hashedBytesValuesIter = hashedBytesValues.getIter(2);
        assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));

        hashedBytesValues.forEachValueInDoc(0, new HashedBytesValuesVerifierProc(0).addMissing());
        hashedBytesValues.forEachValueInDoc(1, new HashedBytesValuesVerifierProc(1).addMissing());
        hashedBytesValues.forEachValueInDoc(2, new HashedBytesValuesVerifierProc(2).addMissing());

        StringValues stringValues = fieldData.getStringValues();

        assertThat(stringValues.hasValue(0), equalTo(false));
        assertThat(stringValues.hasValue(1), equalTo(false));
        assertThat(stringValues.hasValue(2), equalTo(false));

        assertThat(stringValues.getValue(0), nullValue());
        assertThat(stringValues.getValue(1), nullValue());
        assertThat(stringValues.getValue(2), nullValue());

        StringArrayRef stringArrayRef;
        stringArrayRef = stringValues.getValues(0);
        assertThat(stringArrayRef.size(), equalTo(0));

        stringArrayRef = stringValues.getValues(1);
        assertThat(stringArrayRef.size(), equalTo(0));

        stringArrayRef = stringValues.getValues(2);
        assertThat(stringArrayRef.size(), equalTo(0));

        StringValues.Iter stringValuesIter = stringValues.getIter(0);
        assertThat(stringValuesIter.hasNext(), equalTo(false));

        stringValuesIter = stringValues.getIter(1);
        assertThat(stringValuesIter.hasNext(), equalTo(false));

        stringValuesIter = stringValues.getIter(2);
        assertThat(stringValuesIter.hasNext(), equalTo(false));

        stringValues.forEachValueInDoc(0, new StringValuesVerifierProc(0).addMissing());
        stringValues.forEachValueInDoc(1, new StringValuesVerifierProc(1).addMissing());
        stringValues.forEachValueInDoc(2, new StringValuesVerifierProc(2).addMissing());
    }

    protected void fillAllMissing() throws Exception {
        Document d = new Document();
        d.add(new StringField("_id", "1", Field.Store.NO));
        writer.addDocument(d);

        d = new Document();
        d.add(new StringField("_id", "2", Field.Store.NO));
        writer.addDocument(d);

        d = new Document();
        d.add(new StringField("_id", "3", Field.Store.NO));
        writer.addDocument(d);
    }

    @Test
    public void testSortMultiValuesFields() throws Exception {
        fillExtendedMvSet();
        IndexFieldData indexFieldData = getForField("value");

        IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer, true));
        TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10,
                new Sort(new SortField("value", indexFieldData.comparatorSource(null, SortMode.MIN))));
        assertThat(topDocs.totalHits, equalTo(8));
        assertThat(topDocs.scoreDocs.length, equalTo(8));
        assertThat(topDocs.scoreDocs[0].doc, equalTo(1));
        assertThat(((FieldDoc) topDocs.scoreDocs[0]).fields[0], equalTo(null));
        assertThat(topDocs.scoreDocs[1].doc, equalTo(5));
        assertThat(((FieldDoc) topDocs.scoreDocs[1]).fields[0], equalTo(null));
        assertThat(topDocs.scoreDocs[2].doc, equalTo(7));
        assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[2]).fields[0]).utf8ToString(), equalTo("!08"));
        assertThat(topDocs.scoreDocs[3].doc, equalTo(0));
        assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[3]).fields[0]).utf8ToString(), equalTo("02"));
        assertThat(topDocs.scoreDocs[4].doc, equalTo(2));
        assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).utf8ToString(), equalTo("03"));
        assertThat(topDocs.scoreDocs[5].doc, equalTo(3));
        assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[5]).fields[0]).utf8ToString(), equalTo("04"));
        assertThat(topDocs.scoreDocs[6].doc, equalTo(4));
        assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[6]).fields[0]).utf8ToString(), equalTo("06"));
        assertThat(topDocs.scoreDocs[7].doc, equalTo(6));
        assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[7]).fields[0]).utf8ToString(), equalTo("08"));

        topDocs = searcher.search(new MatchAllDocsQuery(), 10,
                new Sort(new SortField("value", indexFieldData.comparatorSource(null, SortMode.MAX), true)));
        assertThat(topDocs.totalHits, equalTo(8));
        assertThat(topDocs.scoreDocs.length, equalTo(8));
        assertThat(topDocs.scoreDocs[0].doc, equalTo(6));
        assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]).utf8ToString(), equalTo("10"));
        assertThat(topDocs.scoreDocs[1].doc, equalTo(4));
        assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[1]).fields[0]).utf8ToString(), equalTo("08"));
        assertThat(topDocs.scoreDocs[2].doc, equalTo(3));
        assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[2]).fields[0]).utf8ToString(), equalTo("06"));
        assertThat(topDocs.scoreDocs[3].doc, equalTo(0));
        assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[3]).fields[0]).utf8ToString(), equalTo("04"));
        assertThat(topDocs.scoreDocs[4].doc, equalTo(2));
        assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).utf8ToString(), equalTo("03"));
        assertThat(topDocs.scoreDocs[5].doc, equalTo(7));
        assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[5]).fields[0]).utf8ToString(), equalTo("!10"));
        assertThat(topDocs.scoreDocs[6].doc, equalTo(1));
        assertThat(((FieldDoc) topDocs.scoreDocs[6]).fields[0], equalTo(null));
        assertThat(topDocs.scoreDocs[7].doc, equalTo(5));
        assertThat(((FieldDoc) topDocs.scoreDocs[7]).fields[0], equalTo(null));
    }

    protected void fillExtendedMvSet() throws Exception {
        Document d = new Document();
        d.add(new StringField("_id", "1", Field.Store.NO));
        d.add(new StringField("value", "02", Field.Store.NO));
        d.add(new StringField("value", "04", Field.Store.NO));
        writer.addDocument(d);

        d = new Document();
        d.add(new StringField("_id", "2", Field.Store.NO));
        writer.addDocument(d);

        d = new Document();
        d.add(new StringField("_id", "3", Field.Store.NO));
        d.add(new StringField("value", "03", Field.Store.NO));
        writer.addDocument(d);
        writer.commit();

        d = new Document();
        d.add(new StringField("_id", "4", Field.Store.NO));
        d.add(new StringField("value", "04", Field.Store.NO));
        d.add(new StringField("value", "05", Field.Store.NO));
        d.add(new StringField("value", "06", Field.Store.NO));
        writer.addDocument(d);

        d = new Document();
        d.add(new StringField("_id", "5", Field.Store.NO));
        d.add(new StringField("value", "06", Field.Store.NO));
        d.add(new StringField("value", "07", Field.Store.NO));
        d.add(new StringField("value", "08", Field.Store.NO));
        writer.addDocument(d);

        d = new Document();
        d.add(new StringField("_id", "6", Field.Store.NO));
        writer.addDocument(d);

        d = new Document();
        d.add(new StringField("_id", "7", Field.Store.NO));
        d.add(new StringField("value", "08", Field.Store.NO));
        d.add(new StringField("value", "09", Field.Store.NO));
        d.add(new StringField("value", "10", Field.Store.NO));
        writer.addDocument(d);
        writer.commit();

        d = new Document();
        d.add(new StringField("_id", "8", Field.Store.NO));
        d.add(new StringField("value", "!08", Field.Store.NO));
        d.add(new StringField("value", "!09", Field.Store.NO));
        d.add(new StringField("value", "!10", Field.Store.NO));
        writer.addDocument(d);
    }

}