org.apache.solr.schema.TestSchemalessBufferedUpdates.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.solr.schema.TestSchemalessBufferedUpdates.java

Source

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.solr.schema;

import static org.apache.solr.update.processor.DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM;

import org.apache.commons.io.FileUtils;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.update.AddUpdateCommand;
import org.apache.solr.update.DirectUpdateHandler2;
import org.apache.solr.update.UpdateLog;
import org.apache.solr.update.UpdateHandler;
import org.apache.solr.update.processor.DistributedUpdateProcessorFactory;
import org.apache.solr.update.processor.UpdateRequestProcessor;
import org.apache.solr.update.processor.UpdateRequestProcessorChain;
import org.apache.solr.update.processor.UpdateRequestProcessorFactory;
import org.junit.BeforeClass;
import org.junit.Test;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.concurrent.Future;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;

import org.apache.solr.update.processor.DistributedUpdateProcessor.DistribPhase;

public class TestSchemalessBufferedUpdates extends SolrTestCaseJ4 {

    // means that we've seen the leader and have version info (i.e. we are a non-leader replica)
    private static final String FROM_LEADER = DistribPhase.FROMLEADER.toString();
    private static final String UPDATE_CHAIN = "add-unknown-fields-to-the-schema";
    private static final int TIMEOUT = 10;

    private static final String collection = "collection1";
    private static final String confDir = collection + "/conf";

    @BeforeClass
    public static void beforeClass() throws Exception {
        File tmpSolrHome = createTempDir().toFile();
        File tmpConfDir = new File(tmpSolrHome, confDir);
        File testHomeConfDir = new File(TEST_HOME(), confDir);
        FileUtils.copyFileToDirectory(new File(testHomeConfDir, "solrconfig-schemaless.xml"), tmpConfDir);
        FileUtils.copyFileToDirectory(new File(testHomeConfDir, "schema-add-schema-fields-update-processor.xml"),
                tmpConfDir);
        FileUtils.copyFileToDirectory(new File(testHomeConfDir, "solrconfig.snippet.randomindexconfig.xml"),
                tmpConfDir);
        initCore("solrconfig-schemaless.xml", "schema-add-schema-fields-update-processor.xml",
                tmpSolrHome.getPath());
    }

    @Test
    public void test() throws Exception {
        DirectUpdateHandler2.commitOnClose = false;
        final Semaphore logReplay = new Semaphore(0);
        final Semaphore logReplayFinish = new Semaphore(0);
        UpdateLog.testing_logReplayHook = () -> {
            try {
                assertTrue(logReplay.tryAcquire(TIMEOUT, TimeUnit.SECONDS));
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        };
        UpdateLog.testing_logReplayFinishHook = logReplayFinish::release;

        SolrQueryRequest req = req();
        UpdateHandler uhandler = req.getCore().getUpdateHandler();
        UpdateLog ulog = uhandler.getUpdateLog();

        try {
            assertEquals(UpdateLog.State.ACTIVE, ulog.getState());

            // Invalid date will be normalized by ParseDateField URP
            updateJ(jsonAdd(processAdd(sdoc("id", "1", "f_dt", "2017-01-04"))),
                    params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
            assertU(commit());
            assertJQ(req("q", "*:*"), "/response/numFound==1");

            ulog.bufferUpdates();
            assertEquals(UpdateLog.State.BUFFERING, ulog.getState());

            // If the ParseDateField URP isn't ahead of the DUP, then the date won't be normalized in the buffered tlog entry,
            // and the doc won't be indexed on the replaying replica - a warning is logged as follows:
            // WARN [...] o.a.s.u.UpdateLog REYPLAY_ERR: IOException reading log
            //            org.apache.solr.common.SolrException: Invalid Date String:'2017-01-05'
            //              at org.apache.solr.util.DateMathParser.parseMath(DateMathParser.java:234)
            //              at org.apache.solr.schema.TrieField.createField(TrieField.java:725) [...]
            updateJ(jsonAdd(processAdd(sdoc("id", "2", "f_dt", "2017-01-05"))),
                    params(DISTRIB_UPDATE_PARAM, FROM_LEADER));

            Future<UpdateLog.RecoveryInfo> rinfoFuture = ulog.applyBufferedUpdates();

            assertTrue(rinfoFuture != null);

            assertEquals(UpdateLog.State.APPLYING_BUFFERED, ulog.getState());

            logReplay.release(1000);

            UpdateLog.RecoveryInfo rinfo = rinfoFuture.get();
            assertEquals(UpdateLog.State.ACTIVE, ulog.getState());

            assertU(commit());
            assertJQ(req("q", "*:*"), "/response/numFound==2");
        } finally {
            DirectUpdateHandler2.commitOnClose = true;
            UpdateLog.testing_logReplayHook = null;
            UpdateLog.testing_logReplayFinishHook = null;
            req().close();
        }
    }

    private SolrInputDocument processAdd(final SolrInputDocument docIn) throws IOException {
        UpdateRequestProcessorChain processorChain = h.getCore().getUpdateProcessingChain(UPDATE_CHAIN);
        assertNotNull("Undefined URP chain '" + UPDATE_CHAIN + "'", processorChain);
        List<UpdateRequestProcessorFactory> factoriesUpToDUP = new ArrayList<>();
        for (UpdateRequestProcessorFactory urpFactory : processorChain.getProcessors()) {
            factoriesUpToDUP.add(urpFactory);
            if (urpFactory.getClass().equals(DistributedUpdateProcessorFactory.class))
                break;
        }
        UpdateRequestProcessorChain chainUpToDUP = new UpdateRequestProcessorChain(factoriesUpToDUP, h.getCore());
        assertNotNull("URP chain '" + UPDATE_CHAIN + "'", chainUpToDUP);
        SolrQueryResponse rsp = new SolrQueryResponse();
        SolrQueryRequest req = req();
        try {
            SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp));
            AddUpdateCommand cmd = new AddUpdateCommand(req);
            cmd.solrDoc = docIn;
            UpdateRequestProcessor processor = chainUpToDUP.createProcessor(req, rsp);
            processor.processAdd(cmd);
            if (cmd.solrDoc.get("f_dt").getValue() instanceof Date) {
                // Non-JSON types (Date in this case) aren't handled properly in noggit-0.6.  Although this is fixed in 
                // https://github.com/yonik/noggit/commit/ec3e732af7c9425e8f40297463cbe294154682b1 to call obj.toString(), 
                // Date::toString produces a Date representation that Solr doesn't like, so we convert using Instant::toString
                cmd.solrDoc.get("f_dt").setValue(((Date) cmd.solrDoc.get("f_dt").getValue()).toInstant().toString(),
                        1.0f);
            }
            return cmd.solrDoc;
        } finally {
            SolrRequestInfo.clearRequestInfo();
            req.close();
        }
    }
}