Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.hbase.snapshot; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InterruptedIOException; import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorCompletionService; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.io.HFileLink; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest; import org.apache.hadoop.hbase.regionserver.wal.HLogUtil; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.hbase.util.FSVisitor; /** * Utility methods for interacting with the snapshot referenced files. */ @InterfaceAudience.Private public final class SnapshotReferenceUtil { public static final Log LOG = LogFactory.getLog(SnapshotReferenceUtil.class); public interface StoreFileVisitor { void storeFile(final HRegionInfo regionInfo, final String familyName, final SnapshotRegionManifest.StoreFile storeFile) throws IOException; } public interface SnapshotVisitor extends StoreFileVisitor, FSVisitor.LogFileVisitor { } private SnapshotReferenceUtil() { // private constructor for utility class } /** * Get log directory for a server in a snapshot. * * @param snapshotDir directory where the specific snapshot is stored * @param serverName name of the parent regionserver for the log files * @return path to the log home directory for the archive files. */ public static Path getLogsDir(Path snapshotDir, String serverName) { return new Path(snapshotDir, HLogUtil.getHLogDirectoryName(serverName)); } /** * Iterate over the snapshot store files, restored.edits and logs * * @param conf The current {@link Configuration} instance. * @param fs {@link FileSystem} * @param snapshotDir {@link Path} to the Snapshot directory * @param visitor callback object to get the referenced files * @throws IOException if an error occurred while scanning the directory */ public static void visitReferencedFiles(final Configuration conf, final FileSystem fs, final Path snapshotDir, final SnapshotVisitor visitor) throws IOException { SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); visitReferencedFiles(conf, fs, snapshotDir, desc, visitor); } /** * Iterate over the snapshot store files, restored.edits and logs * * @param conf The current {@link Configuration} instance. * @param fs {@link FileSystem} * @param snapshotDir {@link Path} to the Snapshot directory * @param desc the {@link SnapshotDescription} of the snapshot to verify * @param visitor callback object to get the referenced files * @throws IOException if an error occurred while scanning the directory */ public static void visitReferencedFiles(final Configuration conf, final FileSystem fs, final Path snapshotDir, final SnapshotDescription desc, final SnapshotVisitor visitor) throws IOException { visitTableStoreFiles(conf, fs, snapshotDir, desc, visitor); visitLogFiles(fs, snapshotDir, visitor); } /** * Iterate over the snapshot store files * * @param conf The current {@link Configuration} instance. * @param fs {@link FileSystem} * @param snapshotDir {@link Path} to the Snapshot directory * @param desc the {@link SnapshotDescription} of the snapshot to verify * @param visitor callback object to get the store files * @throws IOException if an error occurred while scanning the directory */ static void visitTableStoreFiles(final Configuration conf, final FileSystem fs, final Path snapshotDir, final SnapshotDescription desc, final StoreFileVisitor visitor) throws IOException { SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, desc); List<SnapshotRegionManifest> regionManifests = manifest.getRegionManifests(); if (regionManifests == null || regionManifests.size() == 0) { LOG.debug("No manifest files present: " + snapshotDir); return; } for (SnapshotRegionManifest regionManifest : regionManifests) { visitRegionStoreFiles(regionManifest, visitor); } } /** * Iterate over the snapshot store files in the specified region * * @param manifest snapshot manifest to inspect * @param visitor callback object to get the store files * @throws IOException if an error occurred while scanning the directory */ static void visitRegionStoreFiles(final SnapshotRegionManifest manifest, final StoreFileVisitor visitor) throws IOException { HRegionInfo regionInfo = HRegionInfo.convert(manifest.getRegionInfo()); for (SnapshotRegionManifest.FamilyFiles familyFiles : manifest.getFamilyFilesList()) { String familyName = familyFiles.getFamilyName().toStringUtf8(); for (SnapshotRegionManifest.StoreFile storeFile : familyFiles.getStoreFilesList()) { visitor.storeFile(regionInfo, familyName, storeFile); } } } /** * Iterate over the snapshot log files * * @param fs {@link FileSystem} * @param snapshotDir {@link Path} to the Snapshot directory * @param visitor callback object to get the log files * @throws IOException if an error occurred while scanning the directory */ public static void visitLogFiles(final FileSystem fs, final Path snapshotDir, final FSVisitor.LogFileVisitor visitor) throws IOException { FSVisitor.visitLogFiles(fs, snapshotDir, visitor); } /** * Verify the validity of the snapshot * * @param conf The current {@link Configuration} instance. * @param fs {@link FileSystem} * @param snapshotDir {@link Path} to the Snapshot directory of the snapshot to verify * @param snapshotDesc the {@link SnapshotDescription} of the snapshot to verify * @throws CorruptedSnapshotException if the snapshot is corrupted * @throws IOException if an error occurred while scanning the directory */ public static void verifySnapshot(final Configuration conf, final FileSystem fs, final Path snapshotDir, final SnapshotDescription snapshotDesc) throws IOException { SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc); verifySnapshot(conf, fs, manifest); } /** * Verify the validity of the snapshot * * @param conf The current {@link Configuration} instance. * @param fs {@link FileSystem} * @param manifest snapshot manifest to inspect * @throws CorruptedSnapshotException if the snapshot is corrupted * @throws IOException if an error occurred while scanning the directory */ public static void verifySnapshot(final Configuration conf, final FileSystem fs, final SnapshotManifest manifest) throws IOException { final SnapshotDescription snapshotDesc = manifest.getSnapshotDescription(); final Path snapshotDir = manifest.getSnapshotDir(); concurrentVisitReferencedFiles(conf, fs, manifest, new StoreFileVisitor() { @Override public void storeFile(final HRegionInfo regionInfo, final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException { verifyStoreFile(conf, fs, snapshotDir, snapshotDesc, regionInfo, family, storeFile); } }); } public static void concurrentVisitReferencedFiles(final Configuration conf, final FileSystem fs, final SnapshotManifest manifest, final StoreFileVisitor visitor) throws IOException { final SnapshotDescription snapshotDesc = manifest.getSnapshotDescription(); final Path snapshotDir = manifest.getSnapshotDir(); List<SnapshotRegionManifest> regionManifests = manifest.getRegionManifests(); if (regionManifests == null || regionManifests.size() == 0) { LOG.debug("No manifest files present: " + snapshotDir); return; } ExecutorService exec = SnapshotManifest.createExecutor(conf, "VerifySnapshot"); final ExecutorCompletionService<Void> completionService = new ExecutorCompletionService<Void>(exec); try { for (final SnapshotRegionManifest regionManifest : regionManifests) { completionService.submit(new Callable<Void>() { @Override public Void call() throws IOException { visitRegionStoreFiles(regionManifest, visitor); return null; } }); } try { for (int i = 0; i < regionManifests.size(); ++i) { completionService.take().get(); } } catch (InterruptedException e) { throw new InterruptedIOException(e.getMessage()); } catch (ExecutionException e) { if (e.getCause() instanceof CorruptedSnapshotException) { throw new CorruptedSnapshotException(e.getCause().getMessage(), snapshotDesc); } else { IOException ex = new IOException(); ex.initCause(e.getCause()); throw ex; } } } finally { exec.shutdown(); } } /** * Verify the validity of the snapshot store file * * @param conf The current {@link Configuration} instance. * @param fs {@link FileSystem} * @param snapshotDir {@link Path} to the Snapshot directory of the snapshot to verify * @param snapshot the {@link SnapshotDescription} of the snapshot to verify * @param regionInfo {@link HRegionInfo} of the region that contains the store file * @param family family that contains the store file * @param storeFile the store file to verify * @throws CorruptedSnapshotException if the snapshot is corrupted * @throws IOException if an error occurred while scanning the directory */ private static void verifyStoreFile(final Configuration conf, final FileSystem fs, final Path snapshotDir, final SnapshotDescription snapshot, final HRegionInfo regionInfo, final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException { TableName table = TableName.valueOf(snapshot.getTable()); String fileName = storeFile.getName(); Path refPath = null; if (StoreFileInfo.isReference(fileName)) { // If is a reference file check if the parent file is present in the snapshot refPath = new Path(new Path(regionInfo.getEncodedName(), family), fileName); refPath = StoreFileInfo.getReferredToFile(refPath); String refRegion = refPath.getParent().getParent().getName(); refPath = HFileLink.createPath(table, refRegion, family, refPath.getName()); if (!new HFileLink(conf, refPath).exists(fs)) { throw new CorruptedSnapshotException("Missing parent hfile for: " + fileName + " path=" + refPath, snapshot); } if (storeFile.hasReference()) { // We don't really need to look for the file on-disk // we already have the Reference information embedded here. return; } } Path linkPath; if (refPath != null && HFileLink.isHFileLink(refPath)) { linkPath = new Path(family, refPath.getName()); } else if (HFileLink.isHFileLink(fileName)) { linkPath = new Path(family, fileName); } else { linkPath = new Path(family, HFileLink.createHFileLinkName(table, regionInfo.getEncodedName(), fileName)); } // check if the linked file exists (in the archive, or in the table dir) HFileLink link = new HFileLink(conf, linkPath); try { FileStatus fstat = link.getFileStatus(fs); if (storeFile.hasFileSize() && storeFile.getFileSize() != fstat.getLen()) { String msg = "hfile: " + fileName + " size does not match with the expected one. " + " found=" + fstat.getLen() + " expected=" + storeFile.getFileSize(); LOG.error(msg); throw new CorruptedSnapshotException(msg, snapshot); } } catch (FileNotFoundException e) { String msg = "Can't find hfile: " + fileName + " in the real (" + link.getOriginPath() + ") or archive (" + link.getArchivePath() + ") directory for the primary table."; LOG.error(msg); throw new CorruptedSnapshotException(msg, snapshot); } } /** * Returns the store file names in the snapshot. * * @param conf The current {@link Configuration} instance. * @param fs {@link FileSystem} * @param snapshotDir {@link Path} to the Snapshot directory * @throws IOException if an error occurred while scanning the directory * @return the names of hfiles in the specified snaphot */ public static Set<String> getHFileNames(final Configuration conf, final FileSystem fs, final Path snapshotDir) throws IOException { SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); return getHFileNames(conf, fs, snapshotDir, desc); } /** * Returns the store file names in the snapshot. * * @param conf The current {@link Configuration} instance. * @param fs {@link FileSystem} * @param snapshotDir {@link Path} to the Snapshot directory * @param snapshotDesc the {@link SnapshotDescription} of the snapshot to inspect * @throws IOException if an error occurred while scanning the directory * @return the names of hfiles in the specified snaphot */ private static Set<String> getHFileNames(final Configuration conf, final FileSystem fs, final Path snapshotDir, final SnapshotDescription snapshotDesc) throws IOException { final Set<String> names = new HashSet<String>(); visitTableStoreFiles(conf, fs, snapshotDir, snapshotDesc, new StoreFileVisitor() { @Override public void storeFile(final HRegionInfo regionInfo, final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException { String hfile = storeFile.getName(); if (HFileLink.isHFileLink(hfile)) { names.add(HFileLink.getReferencedHFileName(hfile)); } else { names.add(hfile); } } }); return names; } /** * Returns the log file names available in the snapshot. * * @param fs {@link FileSystem} * @param snapshotDir {@link Path} to the Snapshot directory * @throws IOException if an error occurred while scanning the directory * @return the names of hlogs in the specified snaphot */ public static Set<String> getHLogNames(final FileSystem fs, final Path snapshotDir) throws IOException { final Set<String> names = new HashSet<String>(); visitLogFiles(fs, snapshotDir, new FSVisitor.LogFileVisitor() { @Override public void logFile(final String server, final String logfile) throws IOException { names.add(logfile); } }); return names; } }