Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.solr.cloud; import java.io.IOException; import java.lang.invoke.MethodHandles; import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.stream.Collectors; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang.RandomStringUtils; import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.cloud.ZkTestServer.LimitViolationAction; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.util.TimeOut; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import static java.util.Collections.singletonList; import static java.util.concurrent.TimeUnit.SECONDS; /** * * Test for SOLR-9446 * * This test is modeled after SyncSliceTest */ @Slow public class LeaderFailureAfterFreshStartTest extends AbstractFullDistribZkTestBase { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private boolean success = false; int docId = 0; List<CloudJettyRunner> nodesDown = new ArrayList<>(); @Override public void distribTearDown() throws Exception { if (!success) { printLayoutOnTearDown = true; } System.clearProperty("solr.directoryFactory"); System.clearProperty("solr.ulog.numRecordsToKeep"); System.clearProperty("tests.zk.violationReportAction"); super.distribTearDown(); } public LeaderFailureAfterFreshStartTest() { super(); sliceCount = 1; fixShardCount(3); } protected String getCloudSolrConfig() { return "solrconfig-tlog.xml"; } @Override public void distribSetUp() throws Exception { // tlog gets deleted after node restarts if we use CachingDirectoryFactory. // make sure that tlog stays intact after we restart a node System.setProperty("solr.directoryFactory", "solr.StandardDirectoryFactory"); System.setProperty("solr.ulog.numRecordsToKeep", "1000"); System.setProperty("tests.zk.violationReportAction", LimitViolationAction.IGNORE.toString()); super.distribSetUp(); } @Test public void test() throws Exception { handle.clear(); handle.put("timestamp", SKIPVAL); try { CloudJettyRunner initialLeaderJetty = shardToLeaderJetty.get("shard1"); List<CloudJettyRunner> otherJetties = getOtherAvailableJetties(initialLeaderJetty); log.info("Leader node_name: {}, url: {}", initialLeaderJetty.coreNodeName, initialLeaderJetty.url); for (CloudJettyRunner cloudJettyRunner : otherJetties) { log.info("Nonleader node_name: {}, url: {}", cloudJettyRunner.coreNodeName, cloudJettyRunner.url); } CloudJettyRunner secondNode = otherJetties.get(0); CloudJettyRunner freshNode = otherJetties.get(1); // shutdown a node to simulate fresh start otherJetties.remove(freshNode); forceNodeFailures(singletonList(freshNode)); del("*:*"); waitForThingsToLevelOut(30); checkShardConsistency(false, true); // index a few docs and commit for (int i = 0; i < 100; i++) { indexDoc(id, docId, i1, 50, tlong, 50, t1, "document number " + docId++); } commit(); waitForThingsToLevelOut(30); checkShardConsistency(false, true); // bring down the other node and index a few docs; so the leader and other node segments diverge forceNodeFailures(singletonList(secondNode)); for (int i = 0; i < 10; i++) { indexDoc(id, docId, i1, 50, tlong, 50, t1, "document number " + docId++); if (i % 2 == 0) { commit(); } } commit(); restartNodes(singletonList(secondNode)); // start the freshNode restartNodes(singletonList(freshNode)); String replicationProperties = (String) freshNode.jetty.getSolrHome() + "/cores/" + DEFAULT_TEST_COLLECTION_NAME + "/data/replication.properties"; String md5 = DigestUtils.md5Hex(Files.readAllBytes(Paths.get(replicationProperties))); // shutdown the original leader log.info("Now shutting down initial leader"); forceNodeFailures(singletonList(initialLeaderJetty)); waitForNewLeader(cloudClient, "shard1", (Replica) initialLeaderJetty.client.info, new TimeOut(15, SECONDS)); waitTillNodesActive(); log.info("Updating mappings from zk"); updateMappingsFromZk(jettys, clients, true); assertEquals("Node went into replication", md5, DigestUtils.md5Hex(Files.readAllBytes(Paths.get(replicationProperties)))); success = true; } finally { System.clearProperty("solr.disableFingerprint"); } } private void restartNodes(List<CloudJettyRunner> nodesToRestart) throws Exception { for (CloudJettyRunner node : nodesToRestart) { chaosMonkey.start(node.jetty); nodesDown.remove(node); } waitTillNodesActive(); checkShardConsistency(false, true); } private void forceNodeFailures(List<CloudJettyRunner> replicasToShutDown) throws Exception { for (CloudJettyRunner replicaToShutDown : replicasToShutDown) { chaosMonkey.killJetty(replicaToShutDown); } int totalDown = 0; Set<CloudJettyRunner> jetties = new HashSet<>(); jetties.addAll(shardToJetty.get("shard1")); if (replicasToShutDown != null) { jetties.removeAll(replicasToShutDown); totalDown += replicasToShutDown.size(); } jetties.removeAll(nodesDown); totalDown += nodesDown.size(); assertEquals(getShardCount() - totalDown, jetties.size()); nodesDown.addAll(replicasToShutDown); } private void waitTillNodesActive() throws Exception { for (int i = 0; i < 60; i++) { Thread.sleep(3000); ZkStateReader zkStateReader = cloudClient.getZkStateReader(); ClusterState clusterState = zkStateReader.getClusterState(); DocCollection collection1 = clusterState.getCollection("collection1"); Slice slice = collection1.getSlice("shard1"); Collection<Replica> replicas = slice.getReplicas(); boolean allActive = true; Collection<String> nodesDownNames = nodesDown.stream().map(n -> n.coreNodeName) .collect(Collectors.toList()); Collection<Replica> replicasToCheck = null; replicasToCheck = replicas.stream().filter(r -> !nodesDownNames.contains(r.getName())) .collect(Collectors.toList()); for (Replica replica : replicasToCheck) { if (!clusterState.liveNodesContain(replica.getNodeName()) || replica.getState() != Replica.State.ACTIVE) { allActive = false; break; } } if (allActive) { return; } } printLayout(); fail("timeout waiting to see all nodes active"); } private List<CloudJettyRunner> getOtherAvailableJetties(CloudJettyRunner leader) { List<CloudJettyRunner> candidates = new ArrayList<>(); candidates.addAll(shardToJetty.get("shard1")); if (leader != null) { candidates.remove(leader); } candidates.removeAll(nodesDown); return candidates; } protected void indexDoc(Object... fields) throws IOException, SolrServerException { SolrInputDocument doc = new SolrInputDocument(); addFields(doc, fields); addFields(doc, "rnd_s", RandomStringUtils.random(random().nextInt(100) + 100)); UpdateRequest ureq = new UpdateRequest(); ureq.add(doc); ModifiableSolrParams params = new ModifiableSolrParams(); ureq.setParams(params); ureq.process(cloudClient); } // skip the randoms - they can deadlock... @Override protected void indexr(Object... fields) throws Exception { SolrInputDocument doc = new SolrInputDocument(); addFields(doc, fields); addFields(doc, "rnd_b", true); indexDoc(doc); } }