Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.solr.cloud; import org.apache.commons.collections.CollectionUtils; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.zookeeper.KeeperException; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.List; import java.util.concurrent.TimeUnit; public class RollingRestartTest extends AbstractFullDistribZkTestBase { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private static final long MAX_WAIT_TIME = TimeUnit.NANOSECONDS.convert(300, TimeUnit.SECONDS); public RollingRestartTest() { sliceCount = 2; fixShardCount(TEST_NIGHTLY ? 16 : 2); } @Override public void distribSetUp() throws Exception { super.distribSetUp(); useFactory("solr.StandardDirectoryFactory"); } @Test public void test() throws Exception { waitForRecoveriesToFinish(false); restartWithRolesTest(); waitForRecoveriesToFinish(false); } public void restartWithRolesTest() throws Exception { String leader = OverseerCollectionConfigSetProcessor .getLeaderNode(cloudClient.getZkStateReader().getZkClient()); assertNotNull(leader); log.info("Current overseer leader = {}", leader); cloudClient.getZkStateReader().getZkClient().printLayoutToStdOut(); int numDesignateOverseers = TEST_NIGHTLY ? 16 : 2; numDesignateOverseers = Math.max(getShardCount(), numDesignateOverseers); List<String> designates = new ArrayList<>(); List<CloudJettyRunner> designateJettys = new ArrayList<>(); for (int i = 0; i < numDesignateOverseers; i++) { int n = random().nextInt(getShardCount()); String nodeName = cloudJettys.get(n).nodeName; log.info("Chose {} as overseer designate", nodeName); new CollectionAdminRequest.AddRole().setRole("overseer").setNode(nodeName).process(cloudClient); designates.add(nodeName); designateJettys.add(cloudJettys.get(n)); } waitUntilOverseerDesignateIsLeader(cloudClient.getZkStateReader().getZkClient(), designates, MAX_WAIT_TIME); cloudClient.getZkStateReader().getZkClient().printLayoutToStdOut(); boolean sawLiveDesignate = false; int numRestarts = 1 + random().nextInt(TEST_NIGHTLY ? 12 : 2); for (int i = 0; i < numRestarts; i++) { log.info("Rolling restart #{}", i + 1); for (CloudJettyRunner cloudJetty : designateJettys) { log.info("Restarting {}", cloudJetty); chaosMonkey.stopJetty(cloudJetty); cloudClient.getZkStateReader().updateLiveNodes(); boolean liveDesignates = CollectionUtils .intersection(cloudClient.getZkStateReader().getClusterState().getLiveNodes(), designates) .size() > 0; if (liveDesignates) { sawLiveDesignate = true; boolean success = waitUntilOverseerDesignateIsLeader( cloudClient.getZkStateReader().getZkClient(), designates, MAX_WAIT_TIME); if (!success) { leader = OverseerCollectionConfigSetProcessor .getLeaderNode(cloudClient.getZkStateReader().getZkClient()); if (leader == null) log.error("NOOVERSEER election queue is :" + OverseerCollectionConfigSetProcessor .getSortedElectionNodes(cloudClient.getZkStateReader().getZkClient(), "/overseer_elect/election")); fail("No overseer designate as leader found after restart #" + (i + 1) + ": " + leader); } } assertTrue("Unable to restart (#" + i + "): " + cloudJetty, ChaosMonkey.start(cloudJetty.jetty)); boolean success = waitUntilOverseerDesignateIsLeader(cloudClient.getZkStateReader().getZkClient(), designates, MAX_WAIT_TIME); if (!success) { leader = OverseerCollectionConfigSetProcessor .getLeaderNode(cloudClient.getZkStateReader().getZkClient()); if (leader == null) log.error("NOOVERSEER election queue is :" + OverseerCollectionConfigSetProcessor.getSortedElectionNodes( cloudClient.getZkStateReader().getZkClient(), "/overseer_elect/election")); fail("No overseer leader found after restart #" + (i + 1) + ": " + leader); } cloudClient.getZkStateReader().updateLiveNodes(); sawLiveDesignate = CollectionUtils .intersection(cloudClient.getZkStateReader().getClusterState().getLiveNodes(), designates) .size() > 0; } } assertTrue("Test may not be working if we never saw a live designate", sawLiveDesignate); leader = OverseerCollectionConfigSetProcessor.getLeaderNode(cloudClient.getZkStateReader().getZkClient()); assertNotNull(leader); log.info("Current overseer leader (after restart) = {}", leader); cloudClient.getZkStateReader().getZkClient().printLayoutToStdOut(); } static boolean waitUntilOverseerDesignateIsLeader(SolrZkClient testZkClient, List<String> overseerDesignates, long timeoutInNanos) throws KeeperException, InterruptedException { long now = System.nanoTime(); long maxTimeout = now + timeoutInNanos; // the maximum amount of time we're willing to wait to see the designate as leader long timeout = now + TimeUnit.NANOSECONDS.convert(60, TimeUnit.SECONDS); boolean firstTime = true; int stableCheckTimeout = 2000; String oldleader = null; while (System.nanoTime() < timeout && System.nanoTime() < maxTimeout) { String newLeader = OverseerCollectionConfigSetProcessor.getLeaderNode(testZkClient); if (newLeader != null && !newLeader.equals(oldleader)) { // the leaders have changed, let's move the timeout further timeout = System.nanoTime() + TimeUnit.NANOSECONDS.convert(60, TimeUnit.SECONDS); log.info("oldLeader={} newLeader={} - Advancing timeout to: {}", oldleader, newLeader, timeout); oldleader = newLeader; } if (!overseerDesignates.contains(newLeader)) { Thread.sleep(500); } else { if (firstTime) { firstTime = false; Thread.sleep(stableCheckTimeout); } else { return true; } } } if (System.nanoTime() < maxTimeout) { log.error("Max wait time exceeded"); } return false; } }