com.netflix.priam.backup.AbstractRestore.java Source code

Java tutorial

Introduction

Here is the source code for com.netflix.priam.backup.AbstractRestore.java

Source

/**
 * Copyright 2013 Netflix, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.netflix.priam.backup;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.math.BigInteger;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.atomic.AtomicInteger;

import org.apache.commons.io.IOUtils;
import org.bouncycastle.util.io.Streams;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.netflix.priam.IConfiguration;
import com.netflix.priam.backup.AbstractBackupPath.BackupFileType;
import com.netflix.priam.compress.ICompression;
import com.netflix.priam.cryptography.IFileCryptography;
import com.netflix.priam.scheduler.NamedThreadPoolExecutor;
import com.netflix.priam.scheduler.Task;
import com.netflix.priam.utils.FifoQueue;
import com.netflix.priam.utils.RetryableCallable;
import com.netflix.priam.utils.Sleeper;

/*
 * A means to perform a restore.  This class contains the following characteristics:
 * - It is agnostic to the source type of the restore, this is determine by the injected IBackupFileSystem.
 * - This class can be scheduled, i.e. it is a "Task".
 * - When this class is executed, it uses its own thread pool to execute the restores.
 */
public abstract class AbstractRestore extends Task {
    private static final Logger logger = LoggerFactory.getLogger(AbstractRestore.class);
    private static final String SYSTEM_KEYSPACE = "system";
    // keeps track of the last few download which was executed.
    // TODO fix the magic number of 1000 => the idea of 80% of 1000 files limit per s3 query
    protected static final FifoQueue<AbstractBackupPath> tracker = new FifoQueue<AbstractBackupPath>(800);
    private AtomicInteger count = new AtomicInteger();
    protected final IBackupFileSystem fs;

    protected final IConfiguration config;
    protected final ThreadPoolExecutor executor;

    public static BigInteger restoreToken;

    protected final Sleeper sleeper;

    public AbstractRestore(IConfiguration config, IBackupFileSystem fs, String name, Sleeper sleeper) {
        super(config);
        this.config = config;
        this.fs = fs;
        this.sleeper = sleeper;
        executor = new NamedThreadPoolExecutor(config.getMaxBackupDownloadThreads(), name);
        executor.allowCoreThreadTimeOut(true);
    }

    protected void download(Iterator<AbstractBackupPath> fsIterator, BackupFileType filter) throws Exception {
        while (fsIterator.hasNext()) {
            AbstractBackupPath temp = fsIterator.next();
            if (temp.type == BackupFileType.SST && tracker.contains(temp))
                continue;

            if (temp.getType() == filter) {
                File localFileHandler = temp.newRestoreFile();
                logger.debug("Created local file name: " + localFileHandler.getAbsolutePath() + File.pathSeparator
                        + localFileHandler.getName());
                download(temp, localFileHandler);
            }
        }
        waitToComplete();
    }

    public class BoundedList<E> extends LinkedList<E> {

        private final int limit;

        public BoundedList(int limit) {
            this.limit = limit;
        }

        @Override
        public boolean add(E o) {
            super.add(o);
            while (size() > limit) {
                super.remove();
            }
            return true;
        }
    }

    protected void download(Iterator<AbstractBackupPath> fsIterator, BackupFileType filter, int lastN)
            throws Exception {
        if (fsIterator == null)
            return;

        BoundedList bl = new BoundedList(lastN);
        while (fsIterator.hasNext()) {
            AbstractBackupPath temp = fsIterator.next();
            if (temp.type == BackupFileType.SST && tracker.contains(temp))
                continue;

            if (temp.getType() == filter) {
                bl.add(temp);
            }
        }

        download(bl.iterator(), filter);
    }

    /**
     * Download to specific location
     */
    public void download(final AbstractBackupPath path, final File restoreLocation) throws Exception {
        if (config.getRestoreKeySpaces().size() != 0
                && (!config.getRestoreKeySpaces().contains(path.keyspace) || path.keyspace.equals(SYSTEM_KEYSPACE)))
            return;
        count.incrementAndGet();
        executor.submit(new RetryableCallable<Integer>() {
            @Override
            public Integer retriableCall() throws Exception {
                logger.info(
                        "Downloading file: " + path.getRemotePath() + " to: " + restoreLocation.getAbsolutePath());
                fs.download(path, new FileOutputStream(restoreLocation), restoreLocation.getAbsolutePath());
                tracker.adjustAndAdd(path);
                // TODO: fix me -> if there is exception the why hang?

                logger.info("Completed download of file: " + path.getRemotePath() + " to: "
                        + restoreLocation.getAbsolutePath());
                return count.decrementAndGet();
            }
        });
    }

    /*
     * An overloaded download where it will not only download the object but decrypt and uncompress the
     * 
     *  @param path - path of object to download from source.
     *  @param out - handle to the FINAL destination stream.
     *  Note: if this behavior is successfull, it will close the output stream.
     *  
     *  @param temp - file handlle to the downloaded file (i.e. file not decrypted yet).  To ensure widest compatability with various encryption/decryption
     *  
     *  Note: the temp file will be removed on successful processing.
     *  
     *  algorithm, we download the file completely to disk and then decrypt.  This is a temporary file and will be deleted once this behavior completes.
     *  @param fileCrypotography - the implemented cryptography algorithm use to decrypt.
     *  @param passPhrase - if necessary, the pass phrase use by the cryptography algorithm to decrypt.
     */
    public void download(final AbstractBackupPath path, final OutputStream finalDestination, final File tempFile,
            final IFileCryptography fileCryptography, final char[] passPhrase, final ICompression compress) {

        if (config.getRestoreKeySpaces().size() != 0
                && (!config.getRestoreKeySpaces().contains(path.keyspace) || path.keyspace.equals(SYSTEM_KEYSPACE)))
            return;

        count.incrementAndGet();
        executor.submit(new RetryableCallable<Integer>() {

            @Override
            public Integer retriableCall() throws Exception {

                //== download object from source bucket
                try {

                    logger.info("Downloading file from: " + path.getRemotePath() + " to: "
                            + tempFile.getAbsolutePath());
                    fs.download(path, new FileOutputStream(tempFile), tempFile.getAbsolutePath());
                    tracker.adjustAndAdd(path);
                    logger.info("Completed downloading file from: " + path.getRemotePath() + " to: "
                            + tempFile.getAbsolutePath());

                } catch (Exception ex) {
                    //This behavior is retryable; therefore, lets get to a clean state before each retry.
                    if (tempFile.exists()) {
                        tempFile.createNewFile();
                    }

                    throw new Exception("Exception downloading file from: " + path.getRemotePath() + " to: "
                            + tempFile.getAbsolutePath(), ex);
                }

                //== object downloaded successfully from source, decrypt it.
                OutputStream fOut = null; //destination file after decryption
                File decryptedFile = new File(tempFile.getAbsolutePath() + ".decrypted");
                try {

                    InputStream in = new BufferedInputStream(new FileInputStream(tempFile.getAbsolutePath()));
                    InputStream encryptedDataInputStream = fileCryptography.decryptStream(in, passPhrase,
                            tempFile.getAbsolutePath());
                    fOut = new BufferedOutputStream(new FileOutputStream(decryptedFile));
                    Streams.pipeAll(encryptedDataInputStream, fOut);
                    logger.info("completed decrypting file: " + tempFile.getAbsolutePath() + "to final file dest: "
                            + decryptedFile.getAbsolutePath());

                } catch (Exception ex) {
                    //This behavior is retryable; therefore, lets get to a clean state before each retry.
                    if (tempFile.exists()) {
                        tempFile.createNewFile();
                    }

                    if (decryptedFile.exists()) {
                        decryptedFile.createNewFile();
                    }

                    throw new Exception("Exception during decryption file:  " + decryptedFile.getAbsolutePath(),
                            ex);

                } finally {
                    if (fOut != null) {
                        fOut.close();
                    }
                }

                //== object downloaded and decrypted successfully, now uncompress it
                logger.info("Start uncompressing file: " + decryptedFile.getAbsolutePath()
                        + " to the FINAL destination stream");
                FileInputStream fileIs = null;
                InputStream is = null;

                try {

                    fileIs = new FileInputStream(decryptedFile);
                    is = new BufferedInputStream(fileIs);

                    compress.decompressAndClose(is, finalDestination);

                } catch (Exception ex) {
                    IOUtils.closeQuietly(is);
                    throw new Exception("Exception uncompressing file: " + decryptedFile.getAbsolutePath()
                            + " to the FINAL destination stream");
                }

                logger.info("Completed uncompressing file: " + decryptedFile.getAbsolutePath()
                        + " to the FINAL destination stream " + " current worker: "
                        + Thread.currentThread().getName());
                //if here, everything was successful for this object, lets remove unneeded file(s)
                if (tempFile.exists())
                    tempFile.delete();

                if (decryptedFile.exists()) {
                    decryptedFile.delete();
                }

                //Note: removal of the tempFile is responsbility of the caller as this behavior did not create it.

                return count.decrementAndGet();

            }

        });

    }

    /*
     * An overloaded download where it will not only download the object but also decrypt and uncompress.
     * 
     *  @param path - path of object to download from source.
     *  @param restoreLocation - file handle to the FINAL file on disk
     *  @param temp - file handlle to the downloaded file (i.e. file not decrypted yet).  To ensure widest compatability with various encryption/decryption
     *  algorithm, we download the file completely to disk and then decrypt.  This is a temporary file and will be deleted once this behavior completes.
     *  @param fileCrypotography - the implemented cryptography algorithm use to decrypt.
     *  @param passPhrase - if necessary, the pass phrase use by the cryptography algorithm to decrypt.
     */
    public void download(final AbstractBackupPath path, final File restoreLocation, final File tempFile,
            final IFileCryptography fileCryptography, final char[] passPhrase, final ICompression compress)
            throws Exception {

        FileOutputStream fileOs = null;
        BufferedOutputStream os = null;
        try {
            fileOs = new FileOutputStream(restoreLocation);
            os = new BufferedOutputStream(fileOs);
            download(path, os, tempFile, fileCryptography, passPhrase, compress);
        } catch (Exception e) {
            fileOs.close();
            throw new Exception(
                    "Exception in download of:  " + path.getFileName() + ", msg: " + e.getLocalizedMessage(), e);

        } finally {

            //Note: no need to close buffered outpust stream as it is done within the called download() behavior
        }

    }

    /*
     * A means to wait until until all threads have completed.  It blocks calling thread
     * until all tasks (ala counter "count" is 0) are completed.
     */
    protected void waitToComplete() {
        while (count.get() != 0) {
            try {
                sleeper.sleep(1000);
            } catch (InterruptedException e) {
                logger.error("Interrupted: ", e);
                Thread.currentThread().interrupt();
            }
        }
    }

    protected AtomicInteger getFileCount() {
        return count;
    }

    protected void setFileCount(int cnt) {
        count.set(cnt);
    }
}