org.apache.solr.cloud.TestCloudPivotFacet.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.solr.cloud.TestCloudPivotFacet.java

Source

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.solr.cloud;

import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.Arrays;
import java.util.Date;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.commons.lang.StringUtils;
import org.apache.lucene.util.TestUtil;
import org.apache.solr.SolrTestCaseJ4.SuppressSSL;
import org.apache.solr.SolrTestCaseJ4.SuppressPointFields;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.FieldStatsInfo;
import org.apache.solr.client.solrj.response.PivotField;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.FacetParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.params.StatsParams;
import org.apache.solr.common.util.NamedList;
import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import static org.apache.solr.common.params.FacetParams.FACET;
import static org.apache.solr.common.params.FacetParams.FACET_LIMIT;
import static org.apache.solr.common.params.FacetParams.FACET_MISSING;
import static org.apache.solr.common.params.FacetParams.FACET_OFFSET;
import static org.apache.solr.common.params.FacetParams.FACET_OVERREQUEST_COUNT;
import static org.apache.solr.common.params.FacetParams.FACET_OVERREQUEST_RATIO;
import static org.apache.solr.common.params.FacetParams.FACET_PIVOT;
import static org.apache.solr.common.params.FacetParams.FACET_PIVOT_MINCOUNT;
import static org.apache.solr.common.params.FacetParams.FACET_SORT;
import static org.apache.solr.common.params.FacetParams.FACET_DISTRIB_MCO;

/**
 * <p>
 * Randomized testing of Pivot Faceting using SolrCloud.
 * </p>
 * <p>
 * After indexing a bunch of random docs, picks some random fields to pivot facet on, 
 * and then confirms that the resulting counts match the results of filtering on those 
 * values.  This gives us strong assertions on the correctness of the total counts for 
 * each pivot value, but no assertions that the correct "top" counts were chosen.
 * </p>
 * <p>
 * NOTE: this test ignores the control collection and only deals with the 
 * CloudSolrServer - this is because the randomized field values make it very easy for 
 * the term stats to miss values even with the overrequest.
 * (because so many values will tie for "1").  What we care about here is 
 * that the counts we get back are correct and match what we get when filtering on those 
 * constraints.
 * </p>
 *
 *
 *
 */
@SuppressSSL // Too Slow
@SuppressPointFields
public class TestCloudPivotFacet extends AbstractFullDistribZkTestBase {

    private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());

    // param used by test purely for tracing & validation
    private static String TRACE_MIN = "_test_min";
    // param used by test purely for tracing & validation
    private static String TRACE_DISTRIB_MIN = "_test_distrib_min";
    // param used by test purely for tracing & validation
    private static String TRACE_MISS = "_test_miss";
    // param used by test purely for tracing & validation
    private static String TRACE_SORT = "_test_sort";

    /** 
     * Controls the odds of any given doc having a value in any given field -- as this gets lower, 
     * the counts for "facet.missing" pivots should increase.
     * @see #useField()
     */
    private static int useFieldRandomizedFactor = -1;

    @BeforeClass
    public static void initUseFieldRandomizedFactor() {
        useFieldRandomizedFactor = TestUtil.nextInt(random(), 2, 30);
        log.info("init'ing useFieldRandomizedFactor = {}", useFieldRandomizedFactor);
    }

    @Test
    public void test() throws Exception {

        sanityCheckAssertNumerics();

        waitForThingsToLevelOut(30000); // TODO: why would we have to wait?
        // 
        handle.clear();
        handle.put("QTime", SKIPVAL);
        handle.put("timestamp", SKIPVAL);

        final Set<String> fieldNameSet = new HashSet<>();

        // build up a randomized index
        final int numDocs = atLeast(500);
        log.info("numDocs: {}", numDocs);

        for (int i = 1; i <= numDocs; i++) {
            SolrInputDocument doc = buildRandomDocument(i);

            // not efficient, but it guarantees that even if people change buildRandomDocument
            // we'll always have the full list of fields w/o needing to keep code in sync
            fieldNameSet.addAll(doc.getFieldNames());

            cloudClient.add(doc);
        }
        cloudClient.commit();

        fieldNameSet.remove("id");
        assertTrue("WTF, bogus field exists?", fieldNameSet.add("bogus_not_in_any_doc_s"));

        final String[] fieldNames = fieldNameSet.toArray(new String[fieldNameSet.size()]);
        Arrays.sort(fieldNames); // need determinism when picking random fields

        for (int i = 0; i < 5; i++) {

            String q = "*:*";
            if (random().nextBoolean()) {
                q = "id:[* TO " + TestUtil.nextInt(random(), 300, numDocs) + "]";
            }
            ModifiableSolrParams baseP = params("rows", "0", "q", q);

            if (random().nextBoolean()) {
                baseP.add("fq", "id:[* TO " + TestUtil.nextInt(random(), 200, numDocs) + "]");
            }

            final boolean stats = random().nextBoolean();
            if (stats) {
                baseP.add(StatsParams.STATS, "true");

                // if we are doing stats, then always generated the same # of STATS_FIELD
                // params, using multiple tags from a fixed set, but with diff fieldName values.
                // later, each pivot will randomly pick a tag.
                baseP.add(StatsParams.STATS_FIELD, "{!key=sk1 tag=st1,st2}" + pickRandomStatsFields(fieldNames));
                baseP.add(StatsParams.STATS_FIELD, "{!key=sk2 tag=st2,st3}" + pickRandomStatsFields(fieldNames));
                baseP.add(StatsParams.STATS_FIELD, "{!key=sk3 tag=st3,st4}" + pickRandomStatsFields(fieldNames));
                // NOTE: there's a chance that some of those stats field names
                // will be the same, but if so, all the better to test that edge case
            }

            ModifiableSolrParams pivotP = params(FACET, "true");

            // put our FACET_PIVOT params in a set in case we just happen to pick the same one twice
            LinkedHashSet<String> pivotParamValues = new LinkedHashSet<String>();
            pivotParamValues.add(buildPivotParamValue(buildRandomPivot(fieldNames)));

            if (random().nextBoolean()) {
                pivotParamValues.add(buildPivotParamValue(buildRandomPivot(fieldNames)));
            }
            pivotP.set(FACET_PIVOT, pivotParamValues.toArray(new String[pivotParamValues.size()]));

            // keep limit low - lots of unique values, and lots of depth in pivots
            pivotP.add(FACET_LIMIT, "" + TestUtil.nextInt(random(), 1, 17));

            // sometimes use an offset
            if (random().nextBoolean()) {
                pivotP.add(FACET_OFFSET, "" + TestUtil.nextInt(random(), 0, 7));
            }

            if (random().nextBoolean()) {
                String min = "" + TestUtil.nextInt(random(), 0, numDocs + 10);
                pivotP.add(FACET_PIVOT_MINCOUNT, min);
                // trace param for validation
                baseP.add(TRACE_MIN, min);
            }

            if (random().nextBoolean()) {
                pivotP.add(FACET_DISTRIB_MCO, "true");
                // trace param for validation
                baseP.add(TRACE_DISTRIB_MIN, "true");
            }

            if (random().nextBoolean()) {
                String missing = "" + random().nextBoolean();
                pivotP.add(FACET_MISSING, missing);
                // trace param for validation
                baseP.add(TRACE_MISS, missing);
            }

            if (random().nextBoolean()) {
                String sort = random().nextBoolean() ? "index" : "count";
                pivotP.add(FACET_SORT, sort);
                // trace param for validation
                baseP.add(TRACE_SORT, sort);
            }

            // overrequest
            //
            // NOTE: since this test focuses on accuracy of refinement, and doesn't do 
            // control collection comparisons, there isn't a lot of need for excessive
            // overrequesting -- we focus here on trying to exercise the various edge cases
            // involved as different values are used with overrequest
            if (0 == TestUtil.nextInt(random(), 0, 4)) {
                // we want a decent chance of no overrequest at all
                pivotP.add(FACET_OVERREQUEST_COUNT, "0");
                pivotP.add(FACET_OVERREQUEST_RATIO, "0");
            } else {
                if (random().nextBoolean()) {
                    pivotP.add(FACET_OVERREQUEST_COUNT, "" + TestUtil.nextInt(random(), 0, 5));
                }
                if (random().nextBoolean()) {
                    // sometimes give a ratio less then 1, code should be smart enough to deal
                    float ratio = 0.5F + random().nextFloat();
                    // sometimes go negative
                    if (random().nextBoolean()) {
                        ratio *= -1;
                    }
                    pivotP.add(FACET_OVERREQUEST_RATIO, "" + ratio);
                }
            }

            assertPivotCountsAreCorrect(baseP, pivotP);
        }
    }

    /**
     * Given some query params, executes the request against the cloudClient and 
     * then walks the pivot facet values in the response, treating each one as a 
     * filter query to assert the pivot counts are correct.
     */
    private void assertPivotCountsAreCorrect(SolrParams baseParams, SolrParams pivotParams)
            throws SolrServerException {

        SolrParams initParams = SolrParams.wrapAppended(pivotParams, baseParams);

        log.info("Doing full run: {}", initParams);
        countNumFoundChecks = 0;

        NamedList<List<PivotField>> pivots = null;
        try {
            QueryResponse initResponse = cloudClient.query(initParams);
            pivots = initResponse.getFacetPivot();
            assertNotNull(initParams + " has null pivots?", pivots);
            assertEquals(initParams + " num pivots", initParams.getParams("facet.pivot").length, pivots.size());
        } catch (Exception e) {
            throw new RuntimeException("init query failed: " + initParams + ": " + e.getMessage(), e);
        }
        try {
            for (Map.Entry<String, List<PivotField>> pivot : pivots) {
                final String pivotKey = pivot.getKey();
                // :HACK: for counting the max possible pivot depth
                final int maxDepth = 1 + pivotKey.length() - pivotKey.replace(",", "").length();

                assertTraceOk(pivotKey, baseParams, pivot.getValue());

                // NOTE: we can't make any assumptions/assertions about the number of
                // constraints here because of the random data - which means if pivotting is
                // completely broken and there are no constrains this loop could be a No-Op
                // but in that case we just have to trust that DistributedFacetPivotTest
                // will catch it.
                for (PivotField constraint : pivot.getValue()) {
                    int depth = assertPivotCountsAreCorrect(pivotKey, baseParams, constraint);

                    // we can't assert that the depth reached is the same as the depth requested
                    // because the fq and/or mincount may have pruned the tree too much
                    assertTrue("went too deep: " + depth + ": " + pivotKey + " ==> " + pivot, depth <= maxDepth);

                }
            }
        } catch (AssertionError e) {
            throw new AssertionError(initParams + " ==> " + e.getMessage(), e);
        } finally {
            log.info("Ending full run (countNumFoundChecks={}): {}", countNumFoundChecks, initParams);
        }
    }

    /**
     * Recursive Helper method for asserting that pivot constraint counts match
     * results when filtering on those constraints. Returns the recursive depth reached 
     * (for sanity checking)
     */
    private int assertPivotCountsAreCorrect(String pivotName, SolrParams baseParams, PivotField constraint)
            throws SolrServerException {

        SolrParams p = SolrParams.wrapAppended(baseParams, params("fq", buildFilter(constraint)));
        List<PivotField> subPivots = null;
        try {
            assertPivotData(pivotName, constraint, p);
            subPivots = constraint.getPivot();
        } catch (Exception e) {
            throw new RuntimeException(pivotName + ": count query failed: " + p + ": " + e.getMessage(), e);
        }
        int depth = 0;
        if (null != subPivots) {
            assertTraceOk(pivotName, baseParams, subPivots);

            for (PivotField subPivot : subPivots) {
                depth = assertPivotCountsAreCorrect(pivotName, p, subPivot);
            }
        }
        return depth + 1;
    }

    /**
     * Executes a query and compares the results with the data available in the 
     * {@link PivotField} constraint -- this method is not recursive, and doesn't 
     * check anything about the sub-pivots (if any).
     *
     * @param pivotName pivot name
     * @param constraint filters on pivot
     * @param params base solr parameters
     */
    private void assertPivotData(String pivotName, PivotField constraint, SolrParams params)
            throws SolrServerException, IOException {

        SolrParams p = SolrParams.wrapDefaults(params("rows", "0"), params);
        QueryResponse res = cloudClient.query(p);
        String msg = pivotName + ": " + p;

        assertNumFound(msg, constraint.getCount(), res);

        if (p.getBool(StatsParams.STATS, false)) {
            // only check stats if stats expected
            assertPivotStats(msg, constraint, res);
        }
    }

    /**
     * Compare top level stats in response with stats from pivot constraint
     */
    private void assertPivotStats(String message, PivotField constraint, QueryResponse response) {

        if (null == constraint.getFieldStatsInfo()) {
            // no stats for this pivot, nothing to check

            // TODO: use a trace param to know if/how-many to expect ?
            log.info("No stats to check for => " + message);
            return;
        }

        Map<String, FieldStatsInfo> actualFieldStatsInfoMap = response.getFieldStatsInfo();

        for (FieldStatsInfo pivotStats : constraint.getFieldStatsInfo().values()) {
            String statsKey = pivotStats.getName();

            FieldStatsInfo actualStats = actualFieldStatsInfoMap.get(statsKey);

            if (actualStats == null) {
                // handle case for not found stats (using stats query)
                //
                // these has to be a special case check due to the legacy behavior of "top level" 
                // StatsComponent results being "null" (and not even included in the 
                // getFieldStatsInfo() Map due to specila SolrJ logic) 

                log.info("Requested stats missing in verification query, pivot stats: " + pivotStats);
                assertEquals("Special Count", 0L, pivotStats.getCount().longValue());
                assertEquals("Special Missing", constraint.getCount(), pivotStats.getMissing().longValue());

            } else {
                // regular stats, compare everything...

                assert actualStats != null;
                String msg = " of " + statsKey + " => " + message;

                // no wiggle room, these should always be exactly equals, regardless of field type
                assertEquals("Count" + msg, pivotStats.getCount(), actualStats.getCount());
                assertEquals("Missing" + msg, pivotStats.getMissing(), actualStats.getMissing());
                assertEquals("Min" + msg, pivotStats.getMin(), actualStats.getMin());
                assertEquals("Max" + msg, pivotStats.getMax(), actualStats.getMax());

                // precision loss can affect these in some field types depending on shards used
                // and the order that values are accumulated
                assertNumerics("Sum" + msg, pivotStats.getSum(), actualStats.getSum());
                assertNumerics("Mean" + msg, pivotStats.getMean(), actualStats.getMean());
                assertNumerics("Stddev" + msg, pivotStats.getStddev(), actualStats.getStddev());
                assertNumerics("SumOfSquares" + msg, pivotStats.getSumOfSquares(), actualStats.getSumOfSquares());
            }
        }

        if (constraint.getFieldStatsInfo().containsKey("sk2")) { // cheeseball hack
            // if "sk2" was one of hte stats we computed, then we must have also seen
            // sk1 or sk3 because of the way the tags are fixed
            assertEquals("had stats sk2, but not another stat?", 2, constraint.getFieldStatsInfo().size());
        } else {
            // if we did not see "sk2", then 1 of the others must be alone
            assertEquals("only expected 1 stat", 1, constraint.getFieldStatsInfo().size());
            assertTrue("not sk1 or sk3", constraint.getFieldStatsInfo().containsKey("sk1")
                    || constraint.getFieldStatsInfo().containsKey("sk3"));
        }

    }

    /**
     * Verify that the PivotFields we're lookin at doesn't violate any of the expected 
     * behaviors based on the <code>TRACE_*</code> params found in the base params
     */
    private void assertTraceOk(String pivotName, SolrParams baseParams, List<PivotField> constraints) {
        if (null == constraints || 0 == constraints.size()) {
            return;
        }
        final int maxIdx = constraints.size() - 1;

        final int min = baseParams.getInt(TRACE_MIN, -1);
        final boolean expectMissing = baseParams.getBool(TRACE_MISS, false);
        final boolean checkCount = "count".equals(baseParams.get(TRACE_SORT, "count"));

        int prevCount = Integer.MAX_VALUE;

        for (int i = 0; i <= maxIdx; i++) {
            final PivotField constraint = constraints.get(i);
            final int count = constraint.getCount();

            if (0 < min) {
                assertTrue(pivotName + ": val #" + i + " of " + maxIdx + ": count(" + count + ") < facet.mincount("
                        + min + "): " + constraint, min <= count);
            }
            // missing value must always come last, but only if facet.missing was used
            // and may not exist at all (mincount, none missing for this sub-facet, etc...)
            if ((i < maxIdx) || (!expectMissing)) {
                assertNotNull(pivotName + ": val #" + i + " of " + maxIdx + " has null value: " + constraint,
                        constraint.getValue());
            }
            // if we are expecting count based sort, then the count of each constraint 
            // must be lt-or-eq the count that came before -- or it must be the last value and 
            // be "missing"
            if (checkCount) {
                assertTrue(
                        pivotName + ": val #" + i + " of" + maxIdx + ": count(" + count + ") > prevCount("
                                + prevCount + "): " + constraint,
                        ((count <= prevCount) || (expectMissing && i == maxIdx && null == constraint.getValue())));
                prevCount = count;
            }
        }
    }

    /**
     * Given a PivotField constraint, generate a query for the field+value
     * for use in an <code>fq</code> to verify the constraint count
     */
    private static String buildFilter(PivotField constraint) {
        Object value = constraint.getValue();
        if (null == value) {
            // facet.missing, exclude any indexed term
            return "-" + constraint.getField() + ":[* TO *]";
        }
        // otherwise, build up a term filter...
        String prefix = "{!term f=" + constraint.getField() + "}";
        if (value instanceof Date) {
            return prefix + ((Date) value).toInstant();
        } else {
            return prefix + value;
        }
    }

    /**
     * Creates a random facet.pivot param string using some of the specified fieldNames
     */
    private static String buildRandomPivot(String[] fieldNames) {
        final int depth = TestUtil.nextInt(random(), 1, 3);
        String[] fields = new String[depth];
        for (int i = 0; i < depth; i++) {
            // yes this means we might use the same field twice
            // makes it a robust test (especially for multi-valued fields)
            fields[i] = fieldNames[TestUtil.nextInt(random(), 0, fieldNames.length - 1)];
        }
        return StringUtils.join(fields, ",");
    }

    /**
     * Picks a random field to use for Stats
     */
    private static String pickRandomStatsFields(String[] fieldNames) {
        // we need to skip boolean fields when computing stats
        String fieldName;
        do {
            fieldName = fieldNames[TestUtil.nextInt(random(), 0, fieldNames.length - 1)];
        } while (fieldName.endsWith("_b") || fieldName.endsWith("_b1"));

        return fieldName;
    }

    /**
     * Generates a random {@link FacetParams#FACET_PIVOT} value w/ local params 
     * using the specified pivotValue.
     */
    private static String buildPivotParamValue(String pivotValue) {
        // randomly decide which stat tag to use

        // if this is 0, or stats aren't enabled, we'll be asking for a tag that doesn't exist
        // ...which should be fine (just like excluding a tagged fq that doesn't exist)
        final int statTag = TestUtil.nextInt(random(), -1, 4);

        if (0 <= statTag) {
            // only use 1 tag name in the 'stats' localparam - see SOLR-6663
            return "{!stats=st" + statTag + "}" + pivotValue;
        } else {
            // statTag < 0 == sanity check the case of a pivot w/o any stats
            return pivotValue;
        }
    }

    /**
     * Creates a document with randomized field values, some of which be missing values, 
     * some of which will be multi-valued (per the schema) and some of which will be 
     * skewed so that small subsets of the ranges will be more common (resulting in an 
     * increased likelihood of duplicate values)
     * 
     * @see #buildRandomPivot
     */
    private static SolrInputDocument buildRandomDocument(int id) {
        SolrInputDocument doc = sdoc("id", id);
        // most fields are in most docs
        // if field is in a doc, then "skewed" chance val is from a dense range
        // (hopefully with lots of duplication)
        for (String prefix : new String[] { "pivot_i", "pivot_ti" }) {
            if (useField()) {
                doc.addField(prefix + "1", skewed(TestUtil.nextInt(random(), 20, 50), random().nextInt()));

            }
            if (useField()) {
                int numMulti = atLeast(1);
                while (0 < numMulti--) {
                    doc.addField(prefix, skewed(TestUtil.nextInt(random(), 20, 50), random().nextInt()));
                }
            }
        }
        for (String prefix : new String[] { "pivot_l", "pivot_tl" }) {
            if (useField()) {
                doc.addField(prefix + "1", skewed(TestUtil.nextInt(random(), 5000, 5100), random().nextLong()));
            }
            if (useField()) {
                int numMulti = atLeast(1);
                while (0 < numMulti--) {
                    doc.addField(prefix, skewed(TestUtil.nextInt(random(), 5000, 5100), random().nextLong()));
                }
            }
        }
        for (String prefix : new String[] { "pivot_f", "pivot_tf" }) {
            if (useField()) {
                doc.addField(prefix + "1",
                        skewed(1.0F / random().nextInt(13), random().nextFloat() * random().nextInt()));
            }
            if (useField()) {
                int numMulti = atLeast(1);
                while (0 < numMulti--) {
                    doc.addField(prefix,
                            skewed(1.0F / random().nextInt(13), random().nextFloat() * random().nextInt()));
                }
            }
        }
        for (String prefix : new String[] { "pivot_d", "pivot_td" }) {
            if (useField()) {
                doc.addField(prefix + "1",
                        skewed(1.0D / random().nextInt(19), random().nextDouble() * random().nextInt()));
            }
            if (useField()) {
                int numMulti = atLeast(1);
                while (0 < numMulti--) {
                    doc.addField(prefix,
                            skewed(1.0D / random().nextInt(19), random().nextDouble() * random().nextInt()));
                }
            }
        }
        for (String prefix : new String[] { "pivot_dt", "pivot_tdt" }) {
            if (useField()) {
                doc.addField(prefix + "1", skewed(randomSkewedDate(), randomDate()));

            }
            if (useField()) {
                int numMulti = atLeast(1);
                while (0 < numMulti--) {
                    doc.addField(prefix, skewed(randomSkewedDate(), randomDate()));

                }
            }
        }
        {
            String prefix = "pivot_b";
            if (useField()) {
                doc.addField(prefix + "1", random().nextBoolean() ? "t" : "f");
            }
            if (useField()) {
                int numMulti = atLeast(1);
                while (0 < numMulti--) {
                    doc.addField(prefix, random().nextBoolean() ? "t" : "f");
                }
            }
        }
        for (String prefix : new String[] { "pivot_x_s", "pivot_y_s", "pivot_z_s" }) {
            if (useField()) {
                doc.addField(prefix + "1",
                        skewed(TestUtil.randomSimpleString(random(), 1, 1), randomXmlUsableUnicodeString()));
            }
            if (useField()) {
                int numMulti = atLeast(1);
                while (0 < numMulti--) {
                    doc.addField(prefix,
                            skewed(TestUtil.randomSimpleString(random(), 1, 1), randomXmlUsableUnicodeString()));
                }
            }
        }

        //
        // for the remaining fields, make every doc have a value in a dense range
        //

        for (String prefix : new String[] { "dense_pivot_x_s", "dense_pivot_y_s" }) {
            if (useField()) {
                doc.addField(prefix + "1", TestUtil.randomSimpleString(random(), 1, 1));
            }
            if (useField()) {
                int numMulti = atLeast(1);
                while (0 < numMulti--) {
                    doc.addField(prefix, TestUtil.randomSimpleString(random(), 1, 1));
                }
            }
        }
        for (String prefix : new String[] { "dense_pivot_i", "dense_pivot_ti" }) {
            if (useField()) {
                doc.addField(prefix + "1", TestUtil.nextInt(random(), 20, 50));
            }
            if (useField()) {
                int numMulti = atLeast(1);
                while (0 < numMulti--) {
                    doc.addField(prefix, TestUtil.nextInt(random(), 20, 50));
                }
            }
        }

        return doc;
    }

    /** 
     * Similar to usually() but we want it to happen just as often regardless
     * of test multiplier and nightly status
     *
     * @see #useFieldRandomizedFactor
     */
    private static boolean useField() {
        assert 0 < useFieldRandomizedFactor;
        return 0 != TestUtil.nextInt(random(), 0, useFieldRandomizedFactor);
    }

    /**
     * Asserts the number of docs found in the response
     */
    private void assertNumFound(String msg, int expected, QueryResponse response) {

        countNumFoundChecks++;

        assertEquals(msg, expected, response.getResults().getNumFound());
    }

    /**
     * Given two objects returned as stat values asserts that they are they are either both <code>null</code> 
     * or all of the following are true:
     * <ul>
     *  <li>They have the exact same class</li>
     *  <li>They are both Numbers or they are both Dates -- in the later case, their millisecond's 
     *      since epoch are used for all subsequent comparisons
     *  </li>
     *  <li>Either:
     *   <ul>
     *    <li>They are Integer or Long objects with the exact same <code>longValue()</code></li>
     *    <li>They are Float or Double objects and their <code>doubleValue()</code>s
     *        are equally-ish with a "small" epsilon (relative to the scale of the expected value)
     *    </li>
     *   </ul>
     *  </li>
     * <ul>
     *
     * @see Date#getTime
     * @see Number#doubleValue
     * @see Number#longValue
     * @see #assertEquals(String,double,double,double)
     */
    private void assertNumerics(String msg, Object expected, Object actual) {
        if (null == expected || null == actual) {
            assertEquals(msg, expected, actual);
            return;
        }

        assertEquals(msg + " ... values do not have the same type: " + expected + " vs " + actual,
                expected.getClass(), actual.getClass());

        if (expected instanceof Date) {
            expected = ((Date) expected).getTime();
            actual = ((Date) actual).getTime();
            msg = msg + " (w/dates converted to ms)";
        }

        assertTrue(msg + " ... expected is not a Number: " + expected + "=>" + expected.getClass(),
                expected instanceof Number);

        if (expected instanceof Long || expected instanceof Integer) {
            assertEquals(msg, ((Number) expected).longValue(), ((Number) actual).longValue());

        } else if (expected instanceof Float || expected instanceof Double) {
            // compute an epsilon relative to the size of the expected value
            double expect = ((Number) expected).doubleValue();
            double epsilon = Math.abs(expect * 0.1E-7D);

            assertEquals(msg, expect, ((Number) actual).doubleValue(), epsilon);

        } else {
            fail(msg + " ... where did this come from: " + expected.getClass());
        }
    }

    /**
     * test the test
     */
    private void sanityCheckAssertNumerics() {

        assertNumerics("Null?", null, null);
        assertNumerics("large a", new Double(2.3005390038169265E9), new Double(2.300539003816927E9));
        assertNumerics("large b", new Double(1.2722582464444444E9), new Double(1.2722582464444442E9));
        assertNumerics("small", new Double(2.3005390038169265E-9), new Double(2.300539003816927E-9));

        assertNumerics("large a negative", new Double(-2.3005390038169265E9), new Double(-2.300539003816927E9));
        assertNumerics("large b negative", new Double(-1.2722582464444444E9), new Double(-1.2722582464444442E9));
        assertNumerics("small negative", new Double(-2.3005390038169265E-9), new Double(-2.300539003816927E-9));

        assertNumerics("high long", Long.MAX_VALUE, Long.MAX_VALUE);
        assertNumerics("high int", Integer.MAX_VALUE, Integer.MAX_VALUE);
        assertNumerics("low long", Long.MIN_VALUE, Long.MIN_VALUE);
        assertNumerics("low int", Integer.MIN_VALUE, Integer.MIN_VALUE);

        // NOTE: can't use 'fail' in these try blocks, because we are catching AssertionError
        // (ie: the code we are expecting to 'fail' is an actual test assertion generator)

        for (Object num : new Object[] { new Date(42), 42, 42L, 42.0F }) {
            try {
                assertNumerics("non-null", null, num);
                throw new RuntimeException("did not get assertion failure when expected was null");
            } catch (AssertionError e) {
            }

            try {
                assertNumerics("non-null", num, null);
                throw new RuntimeException("did not get assertion failure when actual was null");
            } catch (AssertionError e) {
            }
        }

        try {
            assertNumerics("non-number", "foo", 42);
            throw new RuntimeException("did not get assertion failure when expected was non-number");
        } catch (AssertionError e) {
        }

        try {
            assertNumerics("non-number", 42, "foo");
            throw new RuntimeException("did not get assertion failure when actual was non-number");
        } catch (AssertionError e) {
        }

        try {
            assertNumerics("diff", new Double(2.3005390038169265E9), new Double(2.267272520100462E9));
            throw new RuntimeException("did not get assertion failure when args are big & too diff");
        } catch (AssertionError e) {
        }
        try {
            assertNumerics("diff", new Double(2.3005390038169265E-9), new Double(2.267272520100462E-9));
            throw new RuntimeException("did not get assertion failure when args are small & too diff");
        } catch (AssertionError e) {
        }

        try {
            assertNumerics("diff long", Long.MAX_VALUE, Long.MAX_VALUE - 1);
            throw new RuntimeException("did not get assertion failure when args are diff longs");
        } catch (AssertionError e) {
        }
        try {
            assertNumerics("diff int", Integer.MAX_VALUE, Integer.MAX_VALUE - 1);
            throw new RuntimeException("did not get assertion failure when args are diff ints");
        } catch (AssertionError e) {
        }
        try {
            assertNumerics("diff date", new Date(42), new Date(43));
            throw new RuntimeException("did not get assertion failure when args are diff dates");
        } catch (AssertionError e) {
        }

    }

    /**
     * @see #assertNumFound
     * @see #assertPivotCountsAreCorrect(SolrParams,SolrParams)
     */
    private int countNumFoundChecks = 0;

}