Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.solr.core.snapshots; import java.io.IOException; import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import org.apache.lucene.index.IndexCommit; import org.apache.lucene.index.IndexDeletionPolicy; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.store.Directory; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.util.Utils; import org.apache.solr.core.SolrCore; import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager.SnapshotMetaData; import org.apache.solr.update.SolrIndexWriter; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * This class provides functionality required to handle the data files corresponding to Solr snapshots. */ public class SolrSnapshotManager { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); public static final String INDEX_DIR_PATH = "indexDirPath"; public static final String GENERATION_NUM = "generation"; public static final String SNAPSHOT_STATUS = "status"; public static final String CREATION_DATE = "creationDate"; public static final String SNAPSHOT_REPLICAS = "replicas"; public static final String SNAPSHOTS_INFO = "snapshots"; public static final String LEADER = "leader"; public static final String SHARD_ID = "shard_id"; public static final String FILE_LIST = "files"; /** * This method returns if a named snapshot exists for the specified collection. * * @param zkClient Zookeeper client * @param collectionName The name of the collection * @param commitName The name of the snapshot * @return true if the named snapshot exists * false Otherwise * @throws KeeperException In case of Zookeeper error * @throws InterruptedException In case of thread interruption. */ public static boolean snapshotExists(SolrZkClient zkClient, String collectionName, String commitName) throws KeeperException, InterruptedException { String zkPath = getSnapshotMetaDataZkPath(collectionName, Optional.ofNullable(commitName)); return zkClient.exists(zkPath, true); } /** * This method creates an entry for the named snapshot for the specified collection in Zookeeper. * * @param zkClient Zookeeper client * @param collectionName The name of the collection * @param meta The {@linkplain CollectionSnapshotMetaData} corresponding to named snapshot * @throws KeeperException In case of Zookeeper error * @throws InterruptedException In case of thread interruption. */ public static void createCollectionLevelSnapshot(SolrZkClient zkClient, String collectionName, CollectionSnapshotMetaData meta) throws KeeperException, InterruptedException { String zkPath = getSnapshotMetaDataZkPath(collectionName, Optional.of(meta.getName())); zkClient.makePath(zkPath, Utils.toJSON(meta), CreateMode.PERSISTENT, true); } /** * This method updates an entry for the named snapshot for the specified collection in Zookeeper. * * @param zkClient Zookeeper client * @param collectionName The name of the collection * @param meta The {@linkplain CollectionSnapshotMetaData} corresponding to named snapshot * @throws KeeperException In case of Zookeeper error * @throws InterruptedException In case of thread interruption. */ public static void updateCollectionLevelSnapshot(SolrZkClient zkClient, String collectionName, CollectionSnapshotMetaData meta) throws KeeperException, InterruptedException { String zkPath = getSnapshotMetaDataZkPath(collectionName, Optional.of(meta.getName())); zkClient.setData(zkPath, Utils.toJSON(meta), -1, true); } /** * This method deletes an entry for the named snapshot for the specified collection in Zookeeper. * * @param zkClient Zookeeper client * @param collectionName The name of the collection * @param commitName The name of the snapshot * @throws InterruptedException In case of thread interruption. * @throws KeeperException In case of Zookeeper error */ public static void deleteCollectionLevelSnapshot(SolrZkClient zkClient, String collectionName, String commitName) throws InterruptedException, KeeperException { String zkPath = getSnapshotMetaDataZkPath(collectionName, Optional.of(commitName)); zkClient.delete(zkPath, -1, true); } /** * This method deletes all snapshots for the specified collection in Zookeeper. * * @param zkClient Zookeeper client * @param collectionName The name of the collection * @throws InterruptedException In case of thread interruption. * @throws KeeperException In case of Zookeeper error */ public static void cleanupCollectionLevelSnapshots(SolrZkClient zkClient, String collectionName) throws InterruptedException, KeeperException { String zkPath = getSnapshotMetaDataZkPath(collectionName, Optional.empty()); try { // Delete the meta-data for each snapshot. Collection<String> snapshots = zkClient.getChildren(zkPath, null, true); for (String snapshot : snapshots) { String path = getSnapshotMetaDataZkPath(collectionName, Optional.of(snapshot)); try { zkClient.delete(path, -1, true); } catch (KeeperException ex) { // Gracefully handle the case when the zk node doesn't exist if (ex.code() != KeeperException.Code.NONODE) { throw ex; } } } // Delete the parent node. zkClient.delete(zkPath, -1, true); } catch (KeeperException ex) { // Gracefully handle the case when the zk node doesn't exist (e.g. if no snapshots were created for this collection). if (ex.code() != KeeperException.Code.NONODE) { throw ex; } } } /** * This method returns the {@linkplain CollectionSnapshotMetaData} for the named snapshot for the specified collection in Zookeeper. * * @param zkClient Zookeeper client * @param collectionName The name of the collection * @param commitName The name of the snapshot * @return (Optional) the {@linkplain CollectionSnapshotMetaData} * @throws InterruptedException In case of thread interruption. * @throws KeeperException In case of Zookeeper error */ public static Optional<CollectionSnapshotMetaData> getCollectionLevelSnapshot(SolrZkClient zkClient, String collectionName, String commitName) throws InterruptedException, KeeperException { String zkPath = getSnapshotMetaDataZkPath(collectionName, Optional.of(commitName)); try { Map<String, Object> data = (Map<String, Object>) Utils .fromJSON(zkClient.getData(zkPath, null, null, true)); return Optional.of(new CollectionSnapshotMetaData(data)); } catch (KeeperException ex) { // Gracefully handle the case when the zk node for a specific // snapshot doesn't exist (e.g. due to a concurrent delete operation). if (ex.code() == KeeperException.Code.NONODE) { return Optional.empty(); } throw ex; } } /** * This method returns the {@linkplain CollectionSnapshotMetaData} for each named snapshot for the specified collection in Zookeeper. * * @param zkClient Zookeeper client * @param collectionName The name of the collection * @return the {@linkplain CollectionSnapshotMetaData} for each named snapshot * @throws InterruptedException In case of thread interruption. * @throws KeeperException In case of Zookeeper error */ public static Collection<CollectionSnapshotMetaData> listSnapshots(SolrZkClient zkClient, String collectionName) throws InterruptedException, KeeperException { Collection<CollectionSnapshotMetaData> result = new ArrayList<>(); String zkPath = getSnapshotMetaDataZkPath(collectionName, Optional.empty()); try { Collection<String> snapshots = zkClient.getChildren(zkPath, null, true); for (String snapshot : snapshots) { Optional<CollectionSnapshotMetaData> s = getCollectionLevelSnapshot(zkClient, collectionName, snapshot); if (s.isPresent()) { result.add(s.get()); } } } catch (KeeperException ex) { // Gracefully handle the case when the zk node doesn't exist (e.g. due to a concurrent delete collection operation). if (ex.code() != KeeperException.Code.NONODE) { throw ex; } } return result; } /** * This method deletes index files of the {@linkplain IndexCommit} for the specified generation number. * * @param core The Solr core * @param dir The index directory storing the snapshot. * @param gen The generation number of the {@linkplain IndexCommit} to be deleted. * @throws IOException in case of I/O errors. */ public static void deleteSnapshotIndexFiles(SolrCore core, Directory dir, final long gen) throws IOException { deleteSnapshotIndexFiles(core, dir, new IndexDeletionPolicy() { @Override public void onInit(List<? extends IndexCommit> commits) throws IOException { for (IndexCommit ic : commits) { if (gen == ic.getGeneration()) { log.info("Deleting non-snapshotted index commit with generation {}", ic.getGeneration()); ic.delete(); } } } @Override public void onCommit(List<? extends IndexCommit> commits) throws IOException { } }); } /** * This method deletes index files not associated with the specified <code>snapshots</code>. * * @param core The Solr core * @param dir The index directory storing the snapshot. * @param snapshots The snapshots to be preserved. * @throws IOException in case of I/O errors. */ public static void deleteNonSnapshotIndexFiles(SolrCore core, Directory dir, Collection<SnapshotMetaData> snapshots) throws IOException { final Set<Long> genNumbers = new HashSet<>(); for (SnapshotMetaData m : snapshots) { genNumbers.add(m.getGenerationNumber()); } deleteSnapshotIndexFiles(core, dir, new IndexDeletionPolicy() { @Override public void onInit(List<? extends IndexCommit> commits) throws IOException { for (IndexCommit ic : commits) { if (!genNumbers.contains(ic.getGeneration())) { log.info("Deleting non-snapshotted index commit with generation {}", ic.getGeneration()); ic.delete(); } } } @Override public void onCommit(List<? extends IndexCommit> commits) throws IOException { } }); } /** * This method deletes index files of the {@linkplain IndexCommit} for the specified generation number. * * @param core The Solr core * @param dir The index directory storing the snapshot. * @throws IOException in case of I/O errors. */ private static void deleteSnapshotIndexFiles(SolrCore core, Directory dir, IndexDeletionPolicy delPolicy) throws IOException { IndexWriterConfig conf = core.getSolrConfig().indexConfig.toIndexWriterConfig(core); conf.setOpenMode(OpenMode.APPEND); conf.setMergePolicy(NoMergePolicy.INSTANCE);//Don't want to merge any commits here! conf.setIndexDeletionPolicy(delPolicy); conf.setCodec(core.getCodec()); try (SolrIndexWriter iw = new SolrIndexWriter("SolrSnapshotCleaner", dir, conf)) { // Do nothing. The only purpose of opening index writer is to invoke the Lucene IndexDeletionPolicy#onInit // method so that we can cleanup the files associated with specified index commit. // Note the index writer creates a new commit during the close() operation (which is harmless). } } private static String getSnapshotMetaDataZkPath(String collectionName, Optional<String> commitName) { if (commitName.isPresent()) { return "/snapshots/" + collectionName + "/" + commitName.get(); } return "/snapshots/" + collectionName; } }