org.apache.solr.handler.TestRestoreCore.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.solr.handler.TestRestoreCore.java

Source

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.solr.handler;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.net.URLEncoder;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.io.IOUtils;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.util.TestUtil;
import org.apache.solr.SolrJettyTestBase;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.embedded.JettyConfig;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.util.FileUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

@SolrTestCaseJ4.SuppressSSL // Currently unknown why SSL does not work with this test
public class TestRestoreCore extends SolrJettyTestBase {

    JettySolrRunner masterJetty;
    TestReplicationHandler.SolrInstance master = null;
    SolrClient masterClient;

    private static final String CONF_DIR = "solr" + File.separator + DEFAULT_TEST_CORENAME + File.separator + "conf"
            + File.separator;

    private static String context = "/solr";
    private static long docsSeed; // see indexDocs()

    private static JettySolrRunner createJetty(TestReplicationHandler.SolrInstance instance) throws Exception {
        FileUtils.copyFile(new File(SolrTestCaseJ4.TEST_HOME(), "solr.xml"),
                new File(instance.getHomeDir(), "solr.xml"));
        Properties nodeProperties = new Properties();
        nodeProperties.setProperty("solr.data.dir", instance.getDataDir());
        JettyConfig jettyConfig = JettyConfig.builder().setContext("/solr").setPort(0).build();
        JettySolrRunner jetty = new JettySolrRunner(instance.getHomeDir(), nodeProperties, jettyConfig);
        jetty.start();
        return jetty;
    }

    private static SolrClient createNewSolrClient(int port) {
        try {
            // setup the client...
            final String baseUrl = buildUrl(port, context);
            HttpSolrClient client = getHttpSolrClient(baseUrl);
            client.setConnectionTimeout(15000);
            client.setSoTimeout(60000);
            return client;
        } catch (Exception ex) {
            throw new RuntimeException(ex);
        }
    }

    @Before
    public void setUp() throws Exception {
        super.setUp();
        String configFile = "solrconfig-master.xml";

        master = new TestReplicationHandler.SolrInstance(createTempDir("solr-instance").toFile(), "master", null);
        master.setUp();
        master.copyConfigFile(CONF_DIR + configFile, "solrconfig.xml");

        masterJetty = createJetty(master);
        masterClient = createNewSolrClient(masterJetty.getLocalPort());
        docsSeed = random().nextLong();
    }

    @Override
    @After
    public void tearDown() throws Exception {
        super.tearDown();
        masterClient.close();
        masterClient = null;
        masterJetty.stop();
        masterJetty = null;
        master = null;
    }

    @Test
    public void testSimpleRestore() throws Exception {

        int nDocs = usually() ? BackupRestoreUtils.indexDocs(masterClient, "collection1", docsSeed) : 0;

        String snapshotName;
        String location;
        String params = "";
        String baseUrl = masterJetty.getBaseUrl().toString();

        //Use the default backup location or an externally provided location.
        if (random().nextBoolean()) {
            location = createTempDir().toFile().getAbsolutePath();
            params += "&location=" + URLEncoder.encode(location, "UTF-8");
        }

        //named snapshot vs default snapshot name
        if (random().nextBoolean()) {
            snapshotName = TestUtil.randomSimpleString(random(), 1, 5);
            params += "&name=" + snapshotName;
        }

        TestReplicationHandlerBackup.runBackupCommand(masterJetty, ReplicationHandler.CMD_BACKUP, params);

        CheckBackupStatus checkBackupStatus = new CheckBackupStatus((HttpSolrClient) masterClient,
                DEFAULT_TEST_CORENAME, null);
        while (!checkBackupStatus.success) {
            checkBackupStatus.fetchStatus();
            Thread.sleep(1000);
        }

        int numRestoreTests = nDocs > 0 ? TestUtil.nextInt(random(), 1, 5) : 1;

        for (int attempts = 0; attempts < numRestoreTests; attempts++) {
            //Modify existing index before we call restore.

            if (nDocs > 0) {
                //Delete a few docs
                int numDeletes = TestUtil.nextInt(random(), 1, nDocs);
                for (int i = 0; i < numDeletes; i++) {
                    masterClient.deleteByQuery(DEFAULT_TEST_CORENAME, "id:" + i);
                }
                masterClient.commit(DEFAULT_TEST_CORENAME);

                //Add a few more
                int moreAdds = TestUtil.nextInt(random(), 1, 100);
                for (int i = 0; i < moreAdds; i++) {
                    SolrInputDocument doc = new SolrInputDocument();
                    doc.addField("id", i + nDocs);
                    doc.addField("name", "name = " + (i + nDocs));
                    masterClient.add(DEFAULT_TEST_CORENAME, doc);
                }
                //Purposely not calling commit once in a while. There can be some docs which are not committed
                if (usually()) {
                    masterClient.commit(DEFAULT_TEST_CORENAME);
                }
            }

            TestReplicationHandlerBackup.runBackupCommand(masterJetty, ReplicationHandler.CMD_RESTORE, params);

            while (!fetchRestoreStatus(baseUrl, DEFAULT_TEST_CORENAME)) {
                Thread.sleep(1000);
            }

            //See if restore was successful by checking if all the docs are present again
            BackupRestoreUtils.verifyDocs(nDocs, masterClient, DEFAULT_TEST_CORENAME);
        }

    }

    @Test
    public void testFailedRestore() throws Exception {
        int nDocs = BackupRestoreUtils.indexDocs(masterClient, "collection1", docsSeed);

        String location = createTempDir().toFile().getAbsolutePath();
        String snapshotName = TestUtil.randomSimpleString(random(), 1, 5);
        String params = "&name=" + snapshotName + "&location=" + URLEncoder.encode(location, "UTF-8");
        String baseUrl = masterJetty.getBaseUrl().toString();

        TestReplicationHandlerBackup.runBackupCommand(masterJetty, ReplicationHandler.CMD_BACKUP, params);

        CheckBackupStatus checkBackupStatus = new CheckBackupStatus((HttpSolrClient) masterClient,
                DEFAULT_TEST_CORENAME, null);
        while (!checkBackupStatus.success) {
            checkBackupStatus.fetchStatus();
            Thread.sleep(1000);
        }

        //Remove the segments_n file so that the backup index is corrupted.
        //Restore should fail and it should automatically rollback to the original index.
        Path restoreIndexPath = Paths.get(location).resolve("snapshot." + snapshotName);
        try (DirectoryStream<Path> stream = Files.newDirectoryStream(restoreIndexPath,
                IndexFileNames.SEGMENTS + "*")) {
            Path segmentFileName = stream.iterator().next();
            Files.delete(segmentFileName);
        }

        TestReplicationHandlerBackup.runBackupCommand(masterJetty, ReplicationHandler.CMD_RESTORE, params);

        try {
            while (!fetchRestoreStatus(baseUrl, DEFAULT_TEST_CORENAME)) {
                Thread.sleep(1000);
            }
            fail("Should have thrown an error because restore could not have been successful");
        } catch (AssertionError e) {
            //supposed to happen
        }

        BackupRestoreUtils.verifyDocs(nDocs, masterClient, DEFAULT_TEST_CORENAME);

        //make sure we can write to the index again
        nDocs = BackupRestoreUtils.indexDocs(masterClient, "collection1", docsSeed);
        BackupRestoreUtils.verifyDocs(nDocs, masterClient, DEFAULT_TEST_CORENAME);

    }

    public static boolean fetchRestoreStatus(String baseUrl, String coreName) throws IOException {
        String masterUrl = baseUrl + "/" + coreName + ReplicationHandler.PATH + "?command="
                + ReplicationHandler.CMD_RESTORE_STATUS;
        final Pattern pException = Pattern.compile("<str name=\"exception\">(.*?)</str>");

        InputStream stream = null;
        try {
            URL url = new URL(masterUrl);
            stream = url.openStream();
            String response = IOUtils.toString(stream, "UTF-8");
            Matcher matcher = pException.matcher(response);
            if (matcher.find()) {
                fail("Failed to complete restore action with exception " + matcher.group(1));
            }
            if (response.contains("<str name=\"status\">success</str>")) {
                return true;
            } else if (response.contains("<str name=\"status\">failed</str>")) {
                fail("Restore Failed");
            }
            stream.close();
        } finally {
            IOUtils.closeQuietly(stream);
        }
        return false;
    }
}