org.apache.solr.cloud.LeaderFailureAfterFreshStartTest.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.solr.cloud.LeaderFailureAfterFreshStartTest.java

Source

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.solr.cloud;

import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;

import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.cloud.ZkTestServer.LimitViolationAction;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.util.TimeOut;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import static java.util.Collections.singletonList;
import static java.util.concurrent.TimeUnit.SECONDS;

/**
 * 
 * Test for SOLR-9446
 *
 * This test is modeled after SyncSliceTest
 */
@Slow
public class LeaderFailureAfterFreshStartTest extends AbstractFullDistribZkTestBase {

    private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());

    private boolean success = false;
    int docId = 0;

    List<CloudJettyRunner> nodesDown = new ArrayList<>();

    @Override
    public void distribTearDown() throws Exception {
        if (!success) {
            printLayoutOnTearDown = true;
        }
        System.clearProperty("solr.directoryFactory");
        System.clearProperty("solr.ulog.numRecordsToKeep");
        System.clearProperty("tests.zk.violationReportAction");
        super.distribTearDown();
    }

    public LeaderFailureAfterFreshStartTest() {
        super();
        sliceCount = 1;
        fixShardCount(3);
    }

    protected String getCloudSolrConfig() {
        return "solrconfig-tlog.xml";
    }

    @Override
    public void distribSetUp() throws Exception {
        // tlog gets deleted after node restarts if we use CachingDirectoryFactory.
        // make sure that tlog stays intact after we restart a node
        System.setProperty("solr.directoryFactory", "solr.StandardDirectoryFactory");
        System.setProperty("solr.ulog.numRecordsToKeep", "1000");
        System.setProperty("tests.zk.violationReportAction", LimitViolationAction.IGNORE.toString());
        super.distribSetUp();
    }

    @Test
    public void test() throws Exception {
        handle.clear();
        handle.put("timestamp", SKIPVAL);

        try {
            CloudJettyRunner initialLeaderJetty = shardToLeaderJetty.get("shard1");
            List<CloudJettyRunner> otherJetties = getOtherAvailableJetties(initialLeaderJetty);

            log.info("Leader node_name: {},  url: {}", initialLeaderJetty.coreNodeName, initialLeaderJetty.url);
            for (CloudJettyRunner cloudJettyRunner : otherJetties) {
                log.info("Nonleader node_name: {},  url: {}", cloudJettyRunner.coreNodeName, cloudJettyRunner.url);
            }

            CloudJettyRunner secondNode = otherJetties.get(0);
            CloudJettyRunner freshNode = otherJetties.get(1);

            // shutdown a node to simulate fresh start
            otherJetties.remove(freshNode);
            forceNodeFailures(singletonList(freshNode));

            del("*:*");
            waitForThingsToLevelOut(30);

            checkShardConsistency(false, true);

            // index a few docs and commit
            for (int i = 0; i < 100; i++) {
                indexDoc(id, docId, i1, 50, tlong, 50, t1, "document number " + docId++);
            }
            commit();
            waitForThingsToLevelOut(30);

            checkShardConsistency(false, true);

            // bring down the other node and index a few docs; so the leader and other node segments diverge
            forceNodeFailures(singletonList(secondNode));
            for (int i = 0; i < 10; i++) {
                indexDoc(id, docId, i1, 50, tlong, 50, t1, "document number " + docId++);
                if (i % 2 == 0) {
                    commit();
                }
            }
            commit();
            restartNodes(singletonList(secondNode));

            // start the freshNode 
            restartNodes(singletonList(freshNode));

            String replicationProperties = (String) freshNode.jetty.getSolrHome() + "/cores/"
                    + DEFAULT_TEST_COLLECTION_NAME + "/data/replication.properties";
            String md5 = DigestUtils.md5Hex(Files.readAllBytes(Paths.get(replicationProperties)));

            // shutdown the original leader
            log.info("Now shutting down initial leader");
            forceNodeFailures(singletonList(initialLeaderJetty));
            waitForNewLeader(cloudClient, "shard1", (Replica) initialLeaderJetty.client.info,
                    new TimeOut(15, SECONDS));
            waitTillNodesActive();
            log.info("Updating mappings from zk");
            updateMappingsFromZk(jettys, clients, true);
            assertEquals("Node went into replication", md5,
                    DigestUtils.md5Hex(Files.readAllBytes(Paths.get(replicationProperties))));

            success = true;
        } finally {
            System.clearProperty("solr.disableFingerprint");
        }
    }

    private void restartNodes(List<CloudJettyRunner> nodesToRestart) throws Exception {
        for (CloudJettyRunner node : nodesToRestart) {
            chaosMonkey.start(node.jetty);
            nodesDown.remove(node);
        }
        waitTillNodesActive();
        checkShardConsistency(false, true);
    }

    private void forceNodeFailures(List<CloudJettyRunner> replicasToShutDown) throws Exception {
        for (CloudJettyRunner replicaToShutDown : replicasToShutDown) {
            chaosMonkey.killJetty(replicaToShutDown);
        }

        int totalDown = 0;

        Set<CloudJettyRunner> jetties = new HashSet<>();
        jetties.addAll(shardToJetty.get("shard1"));

        if (replicasToShutDown != null) {
            jetties.removeAll(replicasToShutDown);
            totalDown += replicasToShutDown.size();
        }

        jetties.removeAll(nodesDown);
        totalDown += nodesDown.size();

        assertEquals(getShardCount() - totalDown, jetties.size());

        nodesDown.addAll(replicasToShutDown);
    }

    private void waitTillNodesActive() throws Exception {
        for (int i = 0; i < 60; i++) {
            Thread.sleep(3000);
            ZkStateReader zkStateReader = cloudClient.getZkStateReader();
            ClusterState clusterState = zkStateReader.getClusterState();
            DocCollection collection1 = clusterState.getCollection("collection1");
            Slice slice = collection1.getSlice("shard1");
            Collection<Replica> replicas = slice.getReplicas();
            boolean allActive = true;

            Collection<String> nodesDownNames = nodesDown.stream().map(n -> n.coreNodeName)
                    .collect(Collectors.toList());

            Collection<Replica> replicasToCheck = null;
            replicasToCheck = replicas.stream().filter(r -> !nodesDownNames.contains(r.getName()))
                    .collect(Collectors.toList());

            for (Replica replica : replicasToCheck) {
                if (!clusterState.liveNodesContain(replica.getNodeName())
                        || replica.getState() != Replica.State.ACTIVE) {
                    allActive = false;
                    break;
                }
            }
            if (allActive) {
                return;
            }
        }
        printLayout();
        fail("timeout waiting to see all nodes active");
    }

    private List<CloudJettyRunner> getOtherAvailableJetties(CloudJettyRunner leader) {
        List<CloudJettyRunner> candidates = new ArrayList<>();
        candidates.addAll(shardToJetty.get("shard1"));

        if (leader != null) {
            candidates.remove(leader);
        }

        candidates.removeAll(nodesDown);

        return candidates;
    }

    protected void indexDoc(Object... fields) throws IOException, SolrServerException {
        SolrInputDocument doc = new SolrInputDocument();

        addFields(doc, fields);
        addFields(doc, "rnd_s", RandomStringUtils.random(random().nextInt(100) + 100));

        UpdateRequest ureq = new UpdateRequest();
        ureq.add(doc);
        ModifiableSolrParams params = new ModifiableSolrParams();
        ureq.setParams(params);
        ureq.process(cloudClient);
    }

    // skip the randoms - they can deadlock...
    @Override
    protected void indexr(Object... fields) throws Exception {
        SolrInputDocument doc = new SolrInputDocument();
        addFields(doc, fields);
        addFields(doc, "rnd_b", true);
        indexDoc(doc);
    }

}