org.apache.solr.cloud.TestRandomFlRTGCloud.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.solr.cloud.TestRandomFlRTGCloud.java

Source

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.solr.cloud;

import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.TreeSet;

import org.apache.commons.io.FilenameUtils;
import org.apache.lucene.util.TestUtil;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.response.transform.DocTransformer;
import org.apache.solr.response.transform.RawValueTransformerFactory;
import org.apache.solr.response.transform.TransformerFactory;
import org.apache.solr.util.RandomizeSSL;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/** @see TestCloudPseudoReturnFields */
@RandomizeSSL(clientAuth = 0.0, reason = "client auth uses too much RAM")
public class TestRandomFlRTGCloud extends SolrCloudTestCase {
    private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
    private static final String DEBUG_LABEL = MethodHandles.lookup().lookupClass().getName();
    private static final String COLLECTION_NAME = DEBUG_LABEL + "_collection";

    /** A basic client for operations at the cloud level, default collection will be set */
    private static CloudSolrClient CLOUD_CLIENT;
    /** One client per node */
    private static ArrayList<HttpSolrClient> CLIENTS = new ArrayList<>(5);

    /** Always included in fl so we can vet what doc we're looking at */
    private static final FlValidator ID_VALIDATOR = new SimpleFieldValueValidator("id");

    /** 
     * Types of things we will randomly ask for in fl param, and validate in response docs.
     *
     * @see #addRandomFlValidators
     */
    private static final List<FlValidator> FL_VALIDATORS = Collections.unmodifiableList(Arrays.<FlValidator>asList(
            new GlobValidator("*"), new GlobValidator("*_i"), new GlobValidator("*_s"), new GlobValidator("a*"),
            new DocIdValidator(), new DocIdValidator("my_docid_alias"), new ShardValidator(),
            new ShardValidator("my_shard_alias"), new ValueAugmenterValidator(42),
            new ValueAugmenterValidator(1976, "val_alias"),
            //
            new RenameFieldValueValidator("id", "my_id_alias"), new SimpleFieldValueValidator("aaa_i"),
            new RenameFieldValueValidator("bbb_i", "my_int_field_alias"), new SimpleFieldValueValidator("ccc_s"),
            new RenameFieldValueValidator("ddd_s", "my_str_field_alias"),
            //
            // SOLR-9376: RawValueTransformerFactory doesn't work in cloud mode 
            //
            // new RawFieldValueValidator("json", "eee_s", "my_json_field_alias"),
            // new RawFieldValueValidator("json", "fff_s"),
            // new RawFieldValueValidator("xml", "ggg_s", "my_xml_field_alias"),
            // new RawFieldValueValidator("xml", "hhh_s"),
            //
            new NotIncludedValidator("bogus_unused_field_ss"),
            new NotIncludedValidator("bogus_alias", "bogus_alias:other_bogus_field_i"),
            new NotIncludedValidator("bogus_raw_alias", "bogus_raw_alias:[xml f=bogus_raw_field_ss]"),
            //
            new FunctionValidator("aaa_i"), // fq field
            new FunctionValidator("aaa_i", "func_aaa_alias"), new GeoTransformerValidator("geo_1_srpt"),
            new GeoTransformerValidator("geo_2_srpt", "my_geo_alias"), new ExplainValidator(),
            new ExplainValidator("explain_alias"), new SubQueryValidator(), new NotIncludedValidator("score"),
            new NotIncludedValidator("score", "score_alias:score")));

    @BeforeClass
    public static void createMiniSolrCloudCluster() throws Exception {

        // 50% runs use single node/shard a FL_VALIDATORS with all validators known to work on single node
        // 50% runs use multi node/shard with FL_VALIDATORS only containing stuff that works in cloud
        final boolean singleCoreMode = random().nextBoolean();

        // (asuming multi core multi replicas shouldn't matter (assuming multi node) ...
        final int repFactor = singleCoreMode ? 1 : (usually() ? 1 : 2);
        // ... but we definitely want to ensure forwarded requests to other shards work ...
        final int numShards = singleCoreMode ? 1 : 2;
        // ... including some forwarded requests from nodes not hosting a shard
        final int numNodes = 1 + (singleCoreMode ? 0 : (numShards * repFactor));

        final String configName = DEBUG_LABEL + "_config-set";
        final Path configDir = Paths.get(TEST_HOME(), "collection1", "conf");

        configureCluster(numNodes).addConfig(configName, configDir).configure();

        CLOUD_CLIENT = cluster.getSolrClient();
        CLOUD_CLIENT.setDefaultCollection(COLLECTION_NAME);

        CollectionAdminRequest.createCollection(COLLECTION_NAME, configName, numShards, repFactor)
                .withProperty("config", "solrconfig-tlog.xml").withProperty("schema", "schema-psuedo-fields.xml")
                .process(CLOUD_CLIENT);

        waitForRecoveriesToFinish(CLOUD_CLIENT);

        for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
            CLIENTS.add(getHttpSolrClient(jetty.getBaseUrl() + "/" + COLLECTION_NAME + "/"));
        }
    }

    @AfterClass
    private static void afterClass() throws Exception {
        CLOUD_CLIENT.close();
        CLOUD_CLIENT = null;
        for (HttpSolrClient client : CLIENTS) {
            client.close();
        }
        CLIENTS = null;
    }

    /** 
     * Tests thta all TransformerFactories that are implicitly provided by Solr are tested in this class
     *
     * @see FlValidator#getDefaultTransformerFactoryName
     * @see #FL_VALIDATORS
     * @see TransformerFactory#defaultFactories
     */
    public void testCoverage() throws Exception {
        final Set<String> implicit = new LinkedHashSet<>();
        for (String t : TransformerFactory.defaultFactories.keySet()) {
            implicit.add(t);
        }

        final Set<String> covered = new LinkedHashSet<>();
        for (FlValidator v : FL_VALIDATORS) {
            String t = v.getDefaultTransformerFactoryName();
            if (null != t) {
                covered.add(t);
            }
        }

        // items should only be added to this list if it's known that they do not work with RTG
        // and a specific Jira for fixing this is listed as a comment
        final List<String> knownBugs = Arrays.asList("xml", "json", // SOLR-9376
                "child" // way to complicatd to vet with this test, see SOLR-9379 instead
        );

        for (String buggy : knownBugs) {
            assertFalse(
                    buggy + " is listed as a being a known bug, "
                            + "but it exists in the set of 'covered' TransformerFactories",
                    covered.contains(buggy));
            assertTrue(
                    buggy + " is listed as a known bug, "
                            + "but it does not even exist in the set of 'implicit' TransformerFactories",
                    implicit.remove(buggy));
        }

        implicit.removeAll(covered);
        assertEquals("Some implicit TransformerFactories are not yet tested by this class: " + implicit, 0,
                implicit.size());
    }

    public void testRandomizedUpdatesAndRTGs() throws Exception {

        final int maxNumDocs = atLeast(100);
        final int numSeedDocs = random().nextInt(maxNumDocs / 10); // at most ~10% of the max possible docs
        final int numIters = atLeast(maxNumDocs * 10);
        final SolrInputDocument[] knownDocs = new SolrInputDocument[maxNumDocs];

        log.info("Starting {} iters by seeding {} of {} max docs", numIters, numSeedDocs, maxNumDocs);

        int itersSinceLastCommit = 0;
        for (int i = 0; i < numIters; i++) {
            itersSinceLastCommit = maybeCommit(random(), itersSinceLastCommit, numIters);

            if (i < numSeedDocs) {
                // first N iters all we worry about is seeding
                knownDocs[i] = addRandomDocument(i);
            } else {
                assertOneIter(knownDocs);
            }
        }
    }

    /** 
     * Randomly chooses to do a commit, where the probability of doing so increases the longer it's been since 
     * a commit was done.
     *
     * @returns <code>0</code> if a commit was done, else <code>itersSinceLastCommit + 1</code>
     */
    private static int maybeCommit(final Random rand, final int itersSinceLastCommit, final int numIters)
            throws IOException, SolrServerException {
        final float threshold = itersSinceLastCommit / numIters;
        if (rand.nextFloat() < threshold) {
            log.info("COMMIT");
            assertEquals(0, getRandClient(rand).commit().getStatus());
            return 0;
        }
        return itersSinceLastCommit + 1;
    }

    private void assertOneIter(final SolrInputDocument[] knownDocs) throws IOException, SolrServerException {
        // we want to occasionally test more then one doc per RTG
        final int numDocsThisIter = TestUtil.nextInt(random(), 1, atLeast(2));
        int numDocsThisIterThatExist = 0;

        // pick some random docIds for this iteration and ...
        final int[] docIds = new int[numDocsThisIter];
        for (int i = 0; i < numDocsThisIter; i++) {
            docIds[i] = random().nextInt(knownDocs.length);
            if (null != knownDocs[docIds[i]]) {
                // ...check how many already exist
                numDocsThisIterThatExist++;
            }
        }

        // we want our RTG requests to occasionally include missing/deleted docs,
        // but that's not the primary focus of the test, so weight the odds accordingly
        if (random().nextInt(numDocsThisIter + 2) <= numDocsThisIterThatExist) {

            if (0 < TestUtil.nextInt(random(), 0, 13)) {
                log.info("RTG: numDocsThisIter={} numDocsThisIterThatExist={}, docIds={}", numDocsThisIter,
                        numDocsThisIterThatExist, docIds);
                assertRTG(knownDocs, docIds);
            } else {
                // sporadically delete some docs instead of doing an RTG
                log.info("DEL: numDocsThisIter={} numDocsThisIterThatExist={}, docIds={}", numDocsThisIter,
                        numDocsThisIterThatExist, docIds);
                assertDelete(knownDocs, docIds);
            }
        } else {
            log.info("UPD: numDocsThisIter={} numDocsThisIterThatExist={}, docIds={}", numDocsThisIter,
                    numDocsThisIterThatExist, docIds);
            assertUpdate(knownDocs, docIds);
        }
    }

    /**
     * Does some random indexing of the specified docIds and adds them to knownDocs
     */
    private void assertUpdate(final SolrInputDocument[] knownDocs, final int[] docIds)
            throws IOException, SolrServerException {

        for (final int docId : docIds) {
            // TODO: this method should also do some atomic update operations (ie: "inc" and "set")
            // (but make sure to eval the updates locally as well before modifying knownDocs)
            knownDocs[docId] = addRandomDocument(docId);
        }
    }

    /**
     * Deletes the docIds specified and asserts the results are valid, updateing knownDocs accordingly
     */
    private void assertDelete(final SolrInputDocument[] knownDocs, final int[] docIds)
            throws IOException, SolrServerException {
        List<String> ids = new ArrayList<>(docIds.length);
        for (final int docId : docIds) {
            ids.add("" + docId);
            knownDocs[docId] = null;
        }
        assertEquals("Failed delete: " + docIds, 0, getRandClient(random()).deleteById(ids).getStatus());
    }

    /**
     * Adds one randomly generated document with the specified docId, asserting success, and returns 
     * the document added
     */
    private SolrInputDocument addRandomDocument(final int docId) throws IOException, SolrServerException {
        final SolrClient client = getRandClient(random());

        final SolrInputDocument doc = sdoc("id", "" + docId, "aaa_i", random().nextInt(), "bbb_i",
                random().nextInt(),
                //
                "ccc_s", TestUtil.randomSimpleString(random()), "ddd_s", TestUtil.randomSimpleString(random()),
                "eee_s", TestUtil.randomSimpleString(random()), "fff_s", TestUtil.randomSimpleString(random()),
                "ggg_s", TestUtil.randomSimpleString(random()), "hhh_s", TestUtil.randomSimpleString(random()),
                //
                "geo_1_srpt", GeoTransformerValidator.getValueForIndexing(random()), "geo_2_srpt",
                GeoTransformerValidator.getValueForIndexing(random()),
                // for testing subqueries
                "next_2_ids_ss", String.valueOf(docId + 1), "next_2_ids_ss", String.valueOf(docId + 2),
                // for testing prefix globbing
                "axx_i", random().nextInt(), "ayy_i", random().nextInt(), "azz_s",
                TestUtil.randomSimpleString(random()));

        log.info("ADD: {} = {}", docId, doc);
        assertEquals(0, client.add(doc).getStatus());
        return doc;
    }

    /**
     * Does one or more RTG request for the specified docIds with a randomized fl &amp; fq params, asserting
     * that the returned document (if any) makes sense given the expected SolrInputDocuments
     */
    private void assertRTG(final SolrInputDocument[] knownDocs, final int[] docIds)
            throws IOException, SolrServerException {
        final SolrClient client = getRandClient(random());
        // NOTE: not using SolrClient.getById or getByIds because we want to force choice of "id" vs "ids" params
        final ModifiableSolrParams params = params("qt", "/get");

        // random fq -- nothing fancy, secondary concern for our test
        final Integer FQ_MAX = usually() ? null : random().nextInt();
        if (null != FQ_MAX) {
            params.add("fq", "aaa_i:[* TO " + FQ_MAX + "]");
        }

        final Set<FlValidator> validators = new LinkedHashSet<>();
        validators.add(ID_VALIDATOR); // always include id so we can be confident which doc we're looking at
        addRandomFlValidators(random(), validators);
        FlValidator.addParams(validators, params);

        final List<String> idsToRequest = new ArrayList<>(docIds.length);
        final List<SolrInputDocument> docsToExpect = new ArrayList<>(docIds.length);
        for (int docId : docIds) {
            // every docId will be included in the request
            idsToRequest.add("" + docId);

            // only docs that should actually exist and match our (optional) filter will be expected in response
            if (null != knownDocs[docId]) {
                Integer filterVal = (Integer) knownDocs[docId].getFieldValue("aaa_i");
                if (null == FQ_MAX || ((null != filterVal) && filterVal.intValue() <= FQ_MAX.intValue())) {
                    docsToExpect.add(knownDocs[docId]);
                }
            }
        }

        // even w/only 1 docId requested, the response format can vary depending on how we request it
        final boolean askForList = random().nextBoolean() || (1 != idsToRequest.size());
        if (askForList) {
            if (1 == idsToRequest.size()) {
                // have to be careful not to try to use "multi" 'id' params with only 1 docId
                // with a single docId, the only way to ask for a list is with the "ids" param
                params.add("ids", idsToRequest.get(0));
            } else {
                if (random().nextBoolean()) {
                    // each id in it's own param
                    for (String id : idsToRequest) {
                        params.add("id", id);
                    }
                } else {
                    // add one or more comma seperated ids params
                    params.add(buildCommaSepParams(random(), "ids", idsToRequest));
                }
            }
        } else {
            assert 1 == idsToRequest.size();
            params.add("id", idsToRequest.get(0));
        }

        final QueryResponse rsp = client.query(params);
        assertNotNull(params.toString(), rsp);

        final SolrDocumentList docs = getDocsFromRTGResponse(askForList, rsp);
        assertNotNull(params + " => " + rsp, docs);

        assertEquals("num docs mismatch: " + params + " => " + docsToExpect + " vs " + docs, docsToExpect.size(),
                docs.size());

        // NOTE: RTG makes no garuntees about the order docs will be returned in when multi requested
        for (SolrDocument actual : docs) {
            try {
                int actualId = assertParseInt("id", actual.getFirstValue("id"));
                final SolrInputDocument expected = knownDocs[actualId];
                assertNotNull("expected null doc but RTG returned: " + actual, expected);

                Set<String> expectedFieldNames = new TreeSet<>();
                for (FlValidator v : validators) {
                    expectedFieldNames.addAll(v.assertRTGResults(validators, expected, actual));
                }
                // ensure only expected field names are in the actual document
                Set<String> actualFieldNames = new TreeSet<>(actual.getFieldNames());
                assertEquals("Actual field names returned differs from expected", expectedFieldNames,
                        actualFieldNames);
            } catch (AssertionError ae) {
                throw new AssertionError(params + " => " + actual + ": " + ae.getMessage(), ae);
            }
        }
    }

    /** 
     * trivial helper method to deal with diff response structure between using a single 'id' param vs
     * 2 or more 'id' params (or 1 or more 'ids' params).
     *
     * @return List from response, or a synthetic one created from single response doc if 
     * <code>expectList</code> was false; May be empty; May be null if response included null list.
     */
    private static SolrDocumentList getDocsFromRTGResponse(final boolean expectList, final QueryResponse rsp) {
        if (expectList) {
            return rsp.getResults();
        }

        // else: expect single doc, make our own list...

        final SolrDocumentList result = new SolrDocumentList();
        NamedList<Object> raw = rsp.getResponse();
        Object doc = raw.get("doc");
        if (null != doc) {
            result.add((SolrDocument) doc);
            result.setNumFound(1);
        }
        return result;
    }

    /** 
     * returns a random SolrClient -- either a CloudSolrClient, or an HttpSolrClient pointed 
     * at a node in our cluster 
     */
    public static SolrClient getRandClient(Random rand) {
        int numClients = CLIENTS.size();
        int idx = TestUtil.nextInt(rand, 0, numClients);
        return (idx == numClients) ? CLOUD_CLIENT : CLIENTS.get(idx);
    }

    public static void waitForRecoveriesToFinish(CloudSolrClient client) throws Exception {
        assert null != client.getDefaultCollection();
        AbstractDistribZkTestBase.waitForRecoveriesToFinish(client.getDefaultCollection(),
                client.getZkStateReader(), true, true, 330);
    }

    /** 
     * Abstraction for diff types of things that can be added to an 'fl' param that can validate
     * the results are correct compared to an expected SolrInputDocument
     */
    private interface FlValidator {

        /** 
         * Given a list of FlValidators, adds one or more fl params that corrispond to the entire set, 
         * as well as any other special case top level params required by the validators.
         */
        public static void addParams(final Collection<FlValidator> validators, final ModifiableSolrParams params) {
            final List<String> fls = new ArrayList<>(validators.size());
            for (FlValidator v : validators) {
                params.add(v.getExtraRequestParams());
                fls.add(v.getFlParam());
            }
            params.add(buildCommaSepParams(random(), "fl", fls));
        }

        /**
         * Indicates if this validator is for a transformer that returns true from 
         * {@link DocTransformer#needsSolrIndexSearcher}.  Other validators for transformers that 
         * do <em>not</em> require a re-opened searcher (but may have slightly diff behavior depending 
         * on wether a doc comesfrom the index or from the update log) may use this information to 
         * decide wether they wish to enforce stricter assertions on the resulting document.
         *
         * The default implementation always returns <code>false</code>
         *
         * @see DocIdValidator
         */
        public default boolean requiresRealtimeSearcherReOpen() {
            return false;
        }

        /**
         * the name of a transformer listed in {@link TransformerFactory#defaultFactories} that this validator
         * corrisponds to, or null if not applicable.  Used for testing coverage of 
         * Solr's implicitly supported transformers.
         *
         * Default behavior is to return null
         * @see #testCoverage
         */
        public default String getDefaultTransformerFactoryName() {
            return null;
        }

        /**
         * Any special case params that must be added to the request for this validator
         */
        public default SolrParams getExtraRequestParams() {
            return params();
        }

        /** 
         * Must return a non null String that can be used in an fl param -- either by itself, 
         * or with other items separated by commas
         */
        public String getFlParam();

        /** 
         * Given the expected document and the actual document returned from an RTG, this method
         * should assert that relative to what {@link #getFlParam} returns, the actual document contained
         * what it should relative to the expected document.
         *
         * @param validators all validators in use for this request, including the current one
         * @param expected a document containing the expected fields &amp; values that should be in the index
         * @param actual A document that was returned by an RTG request
         * @return A set of "field names" in the actual document that this validator expected.
         */
        public Collection<String> assertRTGResults(final Collection<FlValidator> validators,
                final SolrInputDocument expected, final SolrDocument actual);
    }

    /** 
     * Some validators behave in a way that "suppresses" real fields even when they would otherwise match a glob
     * @see GlobValidator
     */
    private interface SuppressRealFields {
        public Set<String> getSuppressedFields();
    }

    private abstract static class FieldValueValidator implements FlValidator {
        protected final String expectedFieldName;
        protected final String actualFieldName;

        public FieldValueValidator(final String expectedFieldName, final String actualFieldName) {
            this.expectedFieldName = expectedFieldName;
            this.actualFieldName = actualFieldName;
        }

        public abstract String getFlParam();

        public Collection<String> assertRTGResults(final Collection<FlValidator> validators,
                final SolrInputDocument expected, final SolrDocument actual) {
            assertEquals(expectedFieldName + " vs " + actualFieldName, expected.getFieldValue(expectedFieldName),
                    actual.getFirstValue(actualFieldName));
            return Collections.<String>singleton(actualFieldName);
        }
    }

    private static class SimpleFieldValueValidator extends FieldValueValidator {
        public SimpleFieldValueValidator(final String fieldName) {
            super(fieldName, fieldName);
        }

        public String getFlParam() {
            return expectedFieldName;
        }
    }

    private static class RenameFieldValueValidator extends FieldValueValidator implements SuppressRealFields {
        public RenameFieldValueValidator(final String origFieldName, final String alias) {
            super(origFieldName, alias);
        }

        public String getFlParam() {
            return actualFieldName + ":" + expectedFieldName;
        }

        public Set<String> getSuppressedFields() {
            return Collections.singleton(expectedFieldName);
        }
    }

    /**
     * Validator for {@link RawValueTransformerFactory}
     *
     * This validator is fairly weak, because it doesn't do anything to verify the conditional logic
     * in RawValueTransformerFactory realted to the output format -- but that's out of the scope of 
     * this randomized testing.  
     * 
     * What we're primarily concerned with is that the transformer does it's job and puts the string 
     * in the response, regardless of cloud/RTG/uncommited state of the document.
     */
    private static class RawFieldValueValidator extends RenameFieldValueValidator {
        final String type;
        final String alias;

        public RawFieldValueValidator(final String type, final String fieldName, final String alias) {
            // transformer is weird, default result key doesn't care what params are used...
            super(fieldName, null == alias ? "[" + type + "]" : alias);
            this.type = type;
            this.alias = alias;
        }

        public RawFieldValueValidator(final String type, final String fieldName) {
            this(type, fieldName, null);
        }

        public String getFlParam() {
            return (null == alias ? "" : (alias + ":")) + "[" + type + " f=" + expectedFieldName + "]";
        }

        public String getDefaultTransformerFactoryName() {
            return type;
        }
    }

    /** 
     * enforces that a valid <code>[docid]</code> is present in the response, possibly using a 
     * resultKey alias.  By default the only validation of docId values is that they are an integer 
     * greater than or equal to <code>-1</code> -- but if any other validator in use returns true 
     * from {@link #requiresRealtimeSearcherReOpen} then the constraint is tightened and values must 
     * be greater than or equal to <code>0</code> 
     */
    private static class DocIdValidator implements FlValidator {
        private static final String NAME = "docid";
        private static final String USAGE = "[" + NAME + "]";
        private final String resultKey;

        public DocIdValidator(final String resultKey) {
            this.resultKey = resultKey;
        }

        public DocIdValidator() {
            this(USAGE);
        }

        public String getDefaultTransformerFactoryName() {
            return NAME;
        }

        public String getFlParam() {
            return USAGE.equals(resultKey) ? resultKey : resultKey + ":" + USAGE;
        }

        public Collection<String> assertRTGResults(final Collection<FlValidator> validators,
                final SolrInputDocument expected, final SolrDocument actual) {
            final Object value = actual.getFirstValue(resultKey);
            assertNotNull(getFlParam() + " => no value in actual doc", value);
            assertTrue(USAGE + " must be an Integer: " + value, value instanceof Integer);

            int minValidDocId = -1; // if it comes from update log
            for (FlValidator other : validators) {
                if (other.requiresRealtimeSearcherReOpen()) {
                    minValidDocId = 0;
                    break;
                }
            }
            assertTrue(USAGE + " must be >= " + minValidDocId + ": " + value,
                    minValidDocId <= ((Integer) value).intValue());
            return Collections.<String>singleton(resultKey);
        }
    }

    /** Trivial validator of ShardAugmenterFactory */
    private static class ShardValidator implements FlValidator {
        private static final String NAME = "shard";
        private static final String USAGE = "[" + NAME + "]";
        private final String resultKey;

        public ShardValidator(final String resultKey) {
            this.resultKey = resultKey;
        }

        public ShardValidator() {
            this(USAGE);
        }

        public String getDefaultTransformerFactoryName() {
            return NAME;
        }

        public String getFlParam() {
            return USAGE.equals(resultKey) ? resultKey : resultKey + ":" + USAGE;
        }

        public Collection<String> assertRTGResults(final Collection<FlValidator> validators,
                final SolrInputDocument expected, final SolrDocument actual) {
            final Object value = actual.getFirstValue(resultKey);
            assertNotNull(getFlParam() + " => no value in actual doc", value);
            assertTrue(USAGE + " must be an String: " + value, value instanceof String);

            // trivial sanity check
            assertFalse(USAGE + " => blank string", value.toString().trim().isEmpty());
            return Collections.<String>singleton(resultKey);
        }
    }

    /** Trivial validator of ValueAugmenter */
    private static class ValueAugmenterValidator implements FlValidator {
        private static final String NAME = "value";

        private static String trans(final int value) {
            return "[" + NAME + " v=" + value + " t=int]";
        }

        private final String resultKey;
        private final String fl;
        private final Integer expectedVal;

        private ValueAugmenterValidator(final String fl, final int expectedVal, final String resultKey) {
            this.resultKey = resultKey;
            this.expectedVal = expectedVal;
            this.fl = fl;
        }

        public ValueAugmenterValidator(final int expectedVal, final String resultKey) {
            this(resultKey + ":" + trans(expectedVal), expectedVal, resultKey);
        }

        public ValueAugmenterValidator(final int expectedVal) {
            // value transformer is weird, default result key doesn't care what params are used...
            this(trans(expectedVal), expectedVal, "[" + NAME + "]");
        }

        public String getDefaultTransformerFactoryName() {
            return NAME;
        }

        public String getFlParam() {
            return fl;
        }

        public Collection<String> assertRTGResults(final Collection<FlValidator> validators,
                final SolrInputDocument expected, final SolrDocument actual) {
            final Object actualVal = actual.getFirstValue(resultKey);
            assertNotNull(getFlParam() + " => no value in actual doc", actualVal);
            assertEquals(getFlParam(), expectedVal, actualVal);
            return Collections.<String>singleton(resultKey);
        }
    }

    /** Trivial validator of a ValueSourceAugmenter */
    private static class FunctionValidator implements FlValidator {
        private static String func(String fieldName) {
            return "add(1.3,sub(" + fieldName + "," + fieldName + "))";
        }

        protected final String fl;
        protected final String resultKey;
        protected final String fieldName;

        public FunctionValidator(final String fieldName) {
            this(func(fieldName), fieldName, func(fieldName));
        }

        public FunctionValidator(final String fieldName, final String resultKey) {
            this(resultKey + ":" + func(fieldName), fieldName, resultKey);
        }

        private FunctionValidator(final String fl, final String fieldName, final String resultKey) {
            this.fl = fl;
            this.resultKey = resultKey;
            this.fieldName = fieldName;
        }

        /** always returns true */
        public boolean requiresRealtimeSearcherReOpen() {
            return true;
        }

        public String getFlParam() {
            return fl;
        }

        public Collection<String> assertRTGResults(final Collection<FlValidator> validators,
                final SolrInputDocument expected, final SolrDocument actual) {
            final Object origVal = expected.getFieldValue(fieldName);
            assertTrue("this validator only works on numeric fields: " + origVal, origVal instanceof Number);

            assertEquals(fl, 1.3F, actual.getFirstValue(resultKey));
            return Collections.<String>singleton(resultKey);
        }
    }

    /** 
     * Trivial validator of a SubQueryAugmenter.  
     *
     * This validator ignores 90% of the features/complexity
     * of SubQueryAugmenter, and instead just focuses on the basics of:
     * <ul>
     *  <li>do a subquery for docs where SUBQ_FIELD contains the id of the top level doc</li>
     *  <li>verify that any subquery match is expected based on indexing pattern</li>
     * </ul>
     */
    private static class SubQueryValidator implements FlValidator {

        // HACK to work around SOLR-9396...
        // 
        // we're using "id" (and only "id") in the subquery.q as a workarround limitation in
        // "$rows.foo" parsing -- it only works reliably if "foo" is in fl, so we only use "$rows.id",
        // which we know is in every request (and is a valid integer)

        public final static String NAME = "subquery";
        public final static String SUBQ_KEY = "subq";
        public final static String SUBQ_FIELD = "next_2_ids_i";

        public String getFlParam() {
            return SUBQ_KEY + ":[" + NAME + "]";
        }

        public Collection<String> assertRTGResults(final Collection<FlValidator> validators,
                final SolrInputDocument expected, final SolrDocument actual) {
            final int compVal = assertParseInt("expected id", expected.getFieldValue("id"));

            final Object actualVal = actual.getFieldValue(SUBQ_KEY);
            assertTrue("Expected a doclist: " + actualVal, actualVal instanceof SolrDocumentList);
            assertTrue("should be at most 2 docs in doc list: " + actualVal,
                    ((SolrDocumentList) actualVal).getNumFound() <= 2);

            for (SolrDocument subDoc : (SolrDocumentList) actualVal) {
                final int subDocIdVal = assertParseInt("subquery id", subDoc.getFirstValue("id"));
                assertTrue(
                        "subDocId=" + subDocIdVal + " not in valid range for id=" + compVal + " (expected "
                                + (compVal - 1) + " or " + (compVal - 2) + ")",
                        ((subDocIdVal < compVal) && ((compVal - 2) <= subDocIdVal)));

            }

            return Collections.<String>singleton(SUBQ_KEY);
        }

        public String getDefaultTransformerFactoryName() {
            return NAME;
        }

        public SolrParams getExtraRequestParams() {
            return params(SubQueryValidator.SUBQ_KEY + ".q",
                    "{!field f=" + SubQueryValidator.SUBQ_FIELD + " v=$row.id}");
        }
    }

    /** Trivial validator of a GeoTransformer */
    private static class GeoTransformerValidator implements FlValidator, SuppressRealFields {
        private static final String NAME = "geo";
        /** 
         * we're not worried about testing the actual geo parsing/formatting of values,
         * just that the transformer gets called with the expected field value.
         * so have a small set of fixed input values we use when indexing docs,
         * and the expected output for each
         */
        private static final Map<String, String> VALUES = new HashMap<>();
        /** 
         * The set of legal field values this validator is willing to test as a list so we can
         * reliably index into it with random ints
         */
        private static final List<String> ALLOWED_FIELD_VALUES;
        static {
            for (int i = -42; i < 66; i += 13) {
                VALUES.put("POINT( 42 " + i + " )", "{\"type\":\"Point\",\"coordinates\":[42," + i + "]}");
            }
            ALLOWED_FIELD_VALUES = Collections.unmodifiableList(new ArrayList<>(VALUES.keySet()));
        }

        /** 
         * returns a random field value usable when indexing a document that this validator will
         * be able to handle.
         */
        public static String getValueForIndexing(final Random rand) {
            return ALLOWED_FIELD_VALUES.get(rand.nextInt(ALLOWED_FIELD_VALUES.size()));
        }

        private static String trans(String fieldName) {
            return "[" + NAME + " f=" + fieldName + "]";
        }

        protected final String fl;
        protected final String resultKey;
        protected final String fieldName;

        public GeoTransformerValidator(final String fieldName) {
            // geo transformer is weird, default result key doesn't care what params are used...
            this(trans(fieldName), fieldName, "[" + NAME + "]");
        }

        public GeoTransformerValidator(final String fieldName, final String resultKey) {
            this(resultKey + ":" + trans(fieldName), fieldName, resultKey);
        }

        private GeoTransformerValidator(final String fl, final String fieldName, final String resultKey) {
            this.fl = fl;
            this.resultKey = resultKey;
            this.fieldName = fieldName;
        }

        public String getDefaultTransformerFactoryName() {
            return NAME;
        }

        public String getFlParam() {
            return fl;
        }

        public Collection<String> assertRTGResults(final Collection<FlValidator> validators,
                final SolrInputDocument expected, final SolrDocument actual) {
            final Object origVal = expected.getFieldValue(fieldName);
            assertTrue(fl + ": orig field value is not supported: " + origVal, VALUES.containsKey(origVal));

            assertEquals(fl, VALUES.get(origVal), actual.getFirstValue(resultKey));
            return Collections.<String>singleton(resultKey);
        }

        public Set<String> getSuppressedFields() {
            return Collections.singleton(fieldName);
        }
    }

    /** 
     * Glob based validator.
     * This class checks that every field in the expected doc exists in the actual doc with the expected 
     * value -- with special exceptions for fields that are "suppressed" (usually via an alias)
     *
     * By design, fields that are aliased are "moved" unless the original field name was explicitly included 
     * in the fl, globs don't count.
     *
     * @see RenameFieldValueValidator
     */
    private static class GlobValidator implements FlValidator {
        private final String glob;

        public GlobValidator(final String glob) {
            this.glob = glob;
        }

        private final Set<String> matchingFieldsCache = new LinkedHashSet<>();

        public String getFlParam() {
            return glob;
        }

        private boolean matchesGlob(final String fieldName) {
            if (FilenameUtils.wildcardMatch(fieldName, glob)) {
                matchingFieldsCache.add(fieldName); // Don't calculate it again
                return true;
            }
            return false;
        }

        public Collection<String> assertRTGResults(final Collection<FlValidator> validators,
                final SolrInputDocument expected, final SolrDocument actual) {

            final Set<String> renamed = new LinkedHashSet<>(validators.size());
            for (FlValidator v : validators) {
                if (v instanceof SuppressRealFields) {
                    renamed.addAll(((SuppressRealFields) v).getSuppressedFields());
                }
            }

            // every real field name matching the glob that is not renamed should be in the results
            Set<String> result = new LinkedHashSet<>(expected.getFieldNames().size());
            for (String f : expected.getFieldNames()) {
                if (matchesGlob(f) && (!renamed.contains(f))) {
                    result.add(f);
                    assertEquals(glob + " => " + f, expected.getFieldValue(f), actual.getFirstValue(f));
                }
            }
            return result;
        }
    }

    /** 
     * for things like "score" and "[explain]" where we explicitly expect what we ask for in the fl
     * to <b>not</b> be returned when using RTG.
     */
    private static class NotIncludedValidator implements FlValidator {
        private final String fieldName;
        private final String fl;

        public NotIncludedValidator(final String fl) {
            this(fl, fl);
        }

        public NotIncludedValidator(final String fieldName, final String fl) {
            this.fieldName = fieldName;
            this.fl = fl;
        }

        public String getFlParam() {
            return fl;
        }

        public Collection<String> assertRTGResults(final Collection<FlValidator> validators,
                final SolrInputDocument expected, final SolrDocument actual) {
            assertEquals(fl, null, actual.getFirstValue(fieldName));
            return Collections.emptySet();
        }
    }

    /** explain should always be ignored when using RTG */
    private static class ExplainValidator extends NotIncludedValidator {
        private final static String NAME = "explain";
        private final static String USAGE = "[" + NAME + "]";

        public ExplainValidator() {
            super(USAGE);
        }

        public ExplainValidator(final String resultKey) {
            super(USAGE, resultKey + ":" + USAGE);
        }

        public String getDefaultTransformerFactoryName() {
            return NAME;
        }
    }

    /** helper method for adding a random number (may be 0) of items from {@link #FL_VALIDATORS} */
    private static void addRandomFlValidators(final Random r, final Set<FlValidator> validators) {
        List<FlValidator> copyToShuffle = new ArrayList<>(FL_VALIDATORS);
        Collections.shuffle(copyToShuffle, r);
        final int numToReturn = r.nextInt(copyToShuffle.size());
        validators.addAll(copyToShuffle.subList(0, numToReturn + 1));
    }

    /**
     * Given an ordered list of values to include in a (key) param, randomly groups them (ie: comma seperated) 
     * into actual param key=values which are returned as a new SolrParams instance
     */
    private static SolrParams buildCommaSepParams(final Random rand, final String key, Collection<String> values) {
        ModifiableSolrParams result = new ModifiableSolrParams();
        List<String> copy = new ArrayList<>(values);
        while (!copy.isEmpty()) {
            List<String> slice = copy.subList(0, random().nextInt(1 + copy.size()));
            result.add(key, String.join(",", slice));
            slice.clear();
        }
        return result;
    }

    /** helper method for asserting an object is a non-null String can be parsed as an int */
    public static int assertParseInt(String msg, Object orig) {
        assertNotNull(msg + ": is null", orig);
        assertTrue(msg + ": is not a string: " + orig, orig instanceof String);
        try {
            return Integer.parseInt(orig.toString());
        } catch (NumberFormatException nfe) {
            throw new AssertionError(msg + ": can't be parsed as a number: " + orig, nfe);
        }
    }
}