org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil.java

Source

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.hbase.snapshot;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorCompletionService;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.HFileLink;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
import org.apache.hadoop.hbase.util.FSVisitor;

/**
 * Utility methods for interacting with the snapshot referenced files.
 */
@InterfaceAudience.Private
public final class SnapshotReferenceUtil {
    public static final Log LOG = LogFactory.getLog(SnapshotReferenceUtil.class);

    public interface StoreFileVisitor {
        void storeFile(final HRegionInfo regionInfo, final String familyName,
                final SnapshotRegionManifest.StoreFile storeFile) throws IOException;
    }

    public interface SnapshotVisitor extends StoreFileVisitor, FSVisitor.LogFileVisitor {
    }

    private SnapshotReferenceUtil() {
        // private constructor for utility class
    }

    /**
     * Get log directory for a server in a snapshot.
     *
     * @param snapshotDir directory where the specific snapshot is stored
     * @param serverName name of the parent regionserver for the log files
     * @return path to the log home directory for the archive files.
     */
    public static Path getLogsDir(Path snapshotDir, String serverName) {
        return new Path(snapshotDir, HLogUtil.getHLogDirectoryName(serverName));
    }

    /**
     * Iterate over the snapshot store files, restored.edits and logs
     *
     * @param conf The current {@link Configuration} instance.
     * @param fs {@link FileSystem}
     * @param snapshotDir {@link Path} to the Snapshot directory
     * @param visitor callback object to get the referenced files
     * @throws IOException if an error occurred while scanning the directory
     */
    public static void visitReferencedFiles(final Configuration conf, final FileSystem fs, final Path snapshotDir,
            final SnapshotVisitor visitor) throws IOException {
        SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
        visitReferencedFiles(conf, fs, snapshotDir, desc, visitor);
    }

    /**
     * Iterate over the snapshot store files, restored.edits and logs
     *
     * @param conf The current {@link Configuration} instance.
     * @param fs {@link FileSystem}
     * @param snapshotDir {@link Path} to the Snapshot directory
     * @param desc the {@link SnapshotDescription} of the snapshot to verify
     * @param visitor callback object to get the referenced files
     * @throws IOException if an error occurred while scanning the directory
     */
    public static void visitReferencedFiles(final Configuration conf, final FileSystem fs, final Path snapshotDir,
            final SnapshotDescription desc, final SnapshotVisitor visitor) throws IOException {
        visitTableStoreFiles(conf, fs, snapshotDir, desc, visitor);
        visitLogFiles(fs, snapshotDir, visitor);
    }

    /**
     * Iterate over the snapshot store files
     *
     * @param conf The current {@link Configuration} instance.
     * @param fs {@link FileSystem}
     * @param snapshotDir {@link Path} to the Snapshot directory
     * @param desc the {@link SnapshotDescription} of the snapshot to verify
     * @param visitor callback object to get the store files
     * @throws IOException if an error occurred while scanning the directory
     */
    static void visitTableStoreFiles(final Configuration conf, final FileSystem fs, final Path snapshotDir,
            final SnapshotDescription desc, final StoreFileVisitor visitor) throws IOException {
        SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, desc);
        List<SnapshotRegionManifest> regionManifests = manifest.getRegionManifests();
        if (regionManifests == null || regionManifests.size() == 0) {
            LOG.debug("No manifest files present: " + snapshotDir);
            return;
        }

        for (SnapshotRegionManifest regionManifest : regionManifests) {
            visitRegionStoreFiles(regionManifest, visitor);
        }
    }

    /**
     * Iterate over the snapshot store files in the specified region
     *
     * @param manifest snapshot manifest to inspect
     * @param visitor callback object to get the store files
     * @throws IOException if an error occurred while scanning the directory
     */
    static void visitRegionStoreFiles(final SnapshotRegionManifest manifest, final StoreFileVisitor visitor)
            throws IOException {
        HRegionInfo regionInfo = HRegionInfo.convert(manifest.getRegionInfo());
        for (SnapshotRegionManifest.FamilyFiles familyFiles : manifest.getFamilyFilesList()) {
            String familyName = familyFiles.getFamilyName().toStringUtf8();
            for (SnapshotRegionManifest.StoreFile storeFile : familyFiles.getStoreFilesList()) {
                visitor.storeFile(regionInfo, familyName, storeFile);
            }
        }
    }

    /**
     * Iterate over the snapshot log files
     *
     * @param fs {@link FileSystem}
     * @param snapshotDir {@link Path} to the Snapshot directory
     * @param visitor callback object to get the log files
     * @throws IOException if an error occurred while scanning the directory
     */
    public static void visitLogFiles(final FileSystem fs, final Path snapshotDir,
            final FSVisitor.LogFileVisitor visitor) throws IOException {
        FSVisitor.visitLogFiles(fs, snapshotDir, visitor);
    }

    /**
     * Verify the validity of the snapshot
     *
     * @param conf The current {@link Configuration} instance.
     * @param fs {@link FileSystem}
     * @param snapshotDir {@link Path} to the Snapshot directory of the snapshot to verify
     * @param snapshotDesc the {@link SnapshotDescription} of the snapshot to verify
     * @throws CorruptedSnapshotException if the snapshot is corrupted
     * @throws IOException if an error occurred while scanning the directory
     */
    public static void verifySnapshot(final Configuration conf, final FileSystem fs, final Path snapshotDir,
            final SnapshotDescription snapshotDesc) throws IOException {
        SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
        verifySnapshot(conf, fs, manifest);
    }

    /**
     * Verify the validity of the snapshot
     *
     * @param conf The current {@link Configuration} instance.
     * @param fs {@link FileSystem}
     * @param manifest snapshot manifest to inspect
     * @throws CorruptedSnapshotException if the snapshot is corrupted
     * @throws IOException if an error occurred while scanning the directory
     */
    public static void verifySnapshot(final Configuration conf, final FileSystem fs,
            final SnapshotManifest manifest) throws IOException {
        final SnapshotDescription snapshotDesc = manifest.getSnapshotDescription();
        final Path snapshotDir = manifest.getSnapshotDir();
        concurrentVisitReferencedFiles(conf, fs, manifest, new StoreFileVisitor() {
            @Override
            public void storeFile(final HRegionInfo regionInfo, final String family,
                    final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
                verifyStoreFile(conf, fs, snapshotDir, snapshotDesc, regionInfo, family, storeFile);
            }
        });
    }

    public static void concurrentVisitReferencedFiles(final Configuration conf, final FileSystem fs,
            final SnapshotManifest manifest, final StoreFileVisitor visitor) throws IOException {
        final SnapshotDescription snapshotDesc = manifest.getSnapshotDescription();
        final Path snapshotDir = manifest.getSnapshotDir();

        List<SnapshotRegionManifest> regionManifests = manifest.getRegionManifests();
        if (regionManifests == null || regionManifests.size() == 0) {
            LOG.debug("No manifest files present: " + snapshotDir);
            return;
        }

        ExecutorService exec = SnapshotManifest.createExecutor(conf, "VerifySnapshot");
        final ExecutorCompletionService<Void> completionService = new ExecutorCompletionService<Void>(exec);
        try {
            for (final SnapshotRegionManifest regionManifest : regionManifests) {
                completionService.submit(new Callable<Void>() {
                    @Override
                    public Void call() throws IOException {
                        visitRegionStoreFiles(regionManifest, visitor);
                        return null;
                    }
                });
            }
            try {
                for (int i = 0; i < regionManifests.size(); ++i) {
                    completionService.take().get();
                }
            } catch (InterruptedException e) {
                throw new InterruptedIOException(e.getMessage());
            } catch (ExecutionException e) {
                if (e.getCause() instanceof CorruptedSnapshotException) {
                    throw new CorruptedSnapshotException(e.getCause().getMessage(), snapshotDesc);
                } else {
                    IOException ex = new IOException();
                    ex.initCause(e.getCause());
                    throw ex;
                }
            }
        } finally {
            exec.shutdown();
        }
    }

    /**
     * Verify the validity of the snapshot store file
     *
     * @param conf The current {@link Configuration} instance.
     * @param fs {@link FileSystem}
     * @param snapshotDir {@link Path} to the Snapshot directory of the snapshot to verify
     * @param snapshot the {@link SnapshotDescription} of the snapshot to verify
     * @param regionInfo {@link HRegionInfo} of the region that contains the store file
     * @param family family that contains the store file
     * @param storeFile the store file to verify
     * @throws CorruptedSnapshotException if the snapshot is corrupted
     * @throws IOException if an error occurred while scanning the directory
     */
    private static void verifyStoreFile(final Configuration conf, final FileSystem fs, final Path snapshotDir,
            final SnapshotDescription snapshot, final HRegionInfo regionInfo, final String family,
            final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
        TableName table = TableName.valueOf(snapshot.getTable());
        String fileName = storeFile.getName();

        Path refPath = null;
        if (StoreFileInfo.isReference(fileName)) {
            // If is a reference file check if the parent file is present in the snapshot
            refPath = new Path(new Path(regionInfo.getEncodedName(), family), fileName);
            refPath = StoreFileInfo.getReferredToFile(refPath);
            String refRegion = refPath.getParent().getParent().getName();
            refPath = HFileLink.createPath(table, refRegion, family, refPath.getName());
            if (!new HFileLink(conf, refPath).exists(fs)) {
                throw new CorruptedSnapshotException("Missing parent hfile for: " + fileName + " path=" + refPath,
                        snapshot);
            }

            if (storeFile.hasReference()) {
                // We don't really need to look for the file on-disk
                // we already have the Reference information embedded here.
                return;
            }
        }

        Path linkPath;
        if (refPath != null && HFileLink.isHFileLink(refPath)) {
            linkPath = new Path(family, refPath.getName());
        } else if (HFileLink.isHFileLink(fileName)) {
            linkPath = new Path(family, fileName);
        } else {
            linkPath = new Path(family,
                    HFileLink.createHFileLinkName(table, regionInfo.getEncodedName(), fileName));
        }

        // check if the linked file exists (in the archive, or in the table dir)
        HFileLink link = new HFileLink(conf, linkPath);
        try {
            FileStatus fstat = link.getFileStatus(fs);
            if (storeFile.hasFileSize() && storeFile.getFileSize() != fstat.getLen()) {
                String msg = "hfile: " + fileName + " size does not match with the expected one. " + " found="
                        + fstat.getLen() + " expected=" + storeFile.getFileSize();
                LOG.error(msg);
                throw new CorruptedSnapshotException(msg, snapshot);
            }
        } catch (FileNotFoundException e) {
            String msg = "Can't find hfile: " + fileName + " in the real (" + link.getOriginPath()
                    + ") or archive (" + link.getArchivePath() + ") directory for the primary table.";
            LOG.error(msg);
            throw new CorruptedSnapshotException(msg, snapshot);
        }
    }

    /**
     * Returns the store file names in the snapshot.
     *
     * @param conf The current {@link Configuration} instance.
     * @param fs {@link FileSystem}
     * @param snapshotDir {@link Path} to the Snapshot directory
     * @throws IOException if an error occurred while scanning the directory
     * @return the names of hfiles in the specified snaphot
     */
    public static Set<String> getHFileNames(final Configuration conf, final FileSystem fs, final Path snapshotDir)
            throws IOException {
        SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
        return getHFileNames(conf, fs, snapshotDir, desc);
    }

    /**
     * Returns the store file names in the snapshot.
     *
     * @param conf The current {@link Configuration} instance.
     * @param fs {@link FileSystem}
     * @param snapshotDir {@link Path} to the Snapshot directory
     * @param snapshotDesc the {@link SnapshotDescription} of the snapshot to inspect
     * @throws IOException if an error occurred while scanning the directory
     * @return the names of hfiles in the specified snaphot
     */
    private static Set<String> getHFileNames(final Configuration conf, final FileSystem fs, final Path snapshotDir,
            final SnapshotDescription snapshotDesc) throws IOException {
        final Set<String> names = new HashSet<String>();
        visitTableStoreFiles(conf, fs, snapshotDir, snapshotDesc, new StoreFileVisitor() {
            @Override
            public void storeFile(final HRegionInfo regionInfo, final String family,
                    final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
                String hfile = storeFile.getName();
                if (HFileLink.isHFileLink(hfile)) {
                    names.add(HFileLink.getReferencedHFileName(hfile));
                } else {
                    names.add(hfile);
                }
            }
        });
        return names;
    }

    /**
     * Returns the log file names available in the snapshot.
     *
     * @param fs {@link FileSystem}
     * @param snapshotDir {@link Path} to the Snapshot directory
     * @throws IOException if an error occurred while scanning the directory
     * @return the names of hlogs in the specified snaphot
     */
    public static Set<String> getHLogNames(final FileSystem fs, final Path snapshotDir) throws IOException {
        final Set<String> names = new HashSet<String>();
        visitLogFiles(fs, snapshotDir, new FSVisitor.LogFileVisitor() {
            @Override
            public void logFile(final String server, final String logfile) throws IOException {
                names.add(logfile);
            }
        });
        return names;
    }
}