tachyon.master.EditLog.java Source code

Java tutorial

Introduction

Here is the source code for tachyon.master.EditLog.java

Source

/*
 * Licensed to the University of California, Berkeley under one or more contributor license
 * agreements. See the NOTICE file distributed with this work for additional information regarding
 * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance with the License. You may obtain a
 * copy of the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */

package tachyon.master;

import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.util.List;

import org.apache.hadoop.fs.FSDataOutputStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectWriter;
import com.google.common.base.Throwables;

import tachyon.Constants;
import tachyon.Pair;
import tachyon.TachyonURI;
import tachyon.UnderFileSystem;
import tachyon.conf.TachyonConf;
import tachyon.io.Utils;
import tachyon.thrift.BlockInfoException;
import tachyon.thrift.FileAlreadyExistException;
import tachyon.thrift.FileDoesNotExistException;
import tachyon.thrift.InvalidPathException;
import tachyon.thrift.SuspectedFileSizeException;
import tachyon.thrift.TableDoesNotExistException;
import tachyon.thrift.TachyonException;
import tachyon.util.CommonUtils;

/**
 * Master operation journal.
 */
public final class EditLog {
    private static final Logger LOG = LoggerFactory.getLogger(Constants.LOGGER_TYPE);

    private static int sBackUpLogStartNum = -1;
    private static long sCurrentTId = 0;

    /**
     * Load edit log.
     * 
     * @param info The Master Info.
     * @param path The path of the edit logs.
     * @param currentLogFileNum The smallest completed log number that this master has not loaded
     * @return The last transaction id.
     * @throws IOException
     */
    public static long load(MasterInfo info, String path, int currentLogFileNum) throws IOException {
        UnderFileSystem ufs = UnderFileSystem.get(path, info.getTachyonConf());
        if (!ufs.exists(path)) {
            LOG.info("Edit Log " + path + " does not exist.");
            return 0;
        }
        LOG.info("currentLogNum passed in was " + currentLogFileNum);
        int completedLogs = currentLogFileNum;
        sBackUpLogStartNum = currentLogFileNum;
        String completedPath = path.substring(0, path.lastIndexOf(TachyonURI.SEPARATOR) + 1) + "completed";
        if (!ufs.exists(completedPath)) {
            LOG.info("No completed edit logs to be parsed");
        } else {
            String curEditLogFile = CommonUtils.concat(completedPath, completedLogs + ".editLog");
            while (ufs.exists(curEditLogFile)) {
                LOG.info("Loading Edit Log " + curEditLogFile);
                loadSingleLog(info, curEditLogFile);
                completedLogs++;
                curEditLogFile = CommonUtils.concat(completedPath, completedLogs + ".editLog");
            }
        }
        LOG.info("Loading Edit Log " + path);
        loadSingleLog(info, path);

        ufs.close();
        return sCurrentTId;
    }

    /**
     * Load one edit log.
     * 
     * @param info The Master Info
     * @param path The path of the edit log
     * @throws IOException
     */
    public static void loadSingleLog(MasterInfo info, String path) throws IOException {
        UnderFileSystem ufs = UnderFileSystem.get(path, info.getTachyonConf());

        DataInputStream is = new DataInputStream(ufs.open(path));
        JsonParser parser = JsonObject.createObjectMapper().getFactory().createParser(is);

        while (true) {
            EditLogOperation op;
            try {
                op = parser.readValueAs(EditLogOperation.class);
                LOG.debug("Read operation: {}", op);
            } catch (IOException e) {
                // Unfortunately brittle, but Jackson rethrows EOF with this message.
                if (e.getMessage().contains("end-of-input")) {
                    break;
                } else {
                    throw e;
                }
            }

            sCurrentTId = op.mTransId;
            try {
                switch (op.mType) {
                case ADD_BLOCK: {
                    info.opAddBlock(op.getInt("fileId"), op.getInt("blockIndex"), op.getLong("blockLength"),
                            op.getLong("opTimeMs"));
                    break;
                }
                case ADD_CHECKPOINT: {
                    info._addCheckpoint(-1, op.getInt("fileId"), op.getLong("length"),
                            new TachyonURI(op.getString("path")), op.getLong("opTimeMs"));
                    break;
                }
                case CREATE_FILE: {
                    info._createFile(op.getBoolean("recursive"), new TachyonURI(op.getString("path")),
                            op.getBoolean("directory"), op.getLong("blockSizeByte"), op.getLong("creationTimeMs"));
                    break;
                }
                case COMPLETE_FILE: {
                    info._completeFile(op.get("fileId", Integer.class), op.getLong("opTimeMs"));
                    break;
                }
                case SET_PINNED: {
                    info._setPinned(op.getInt("fileId"), op.getBoolean("pinned"), op.getLong("opTimeMs"));
                    break;
                }
                case RENAME: {
                    info._rename(op.getInt("fileId"), new TachyonURI(op.getString("dstPath")),
                            op.getLong("opTimeMs"));
                    break;
                }
                case DELETE: {
                    info._delete(op.getInt("fileId"), op.getBoolean("recursive"), op.getLong("opTimeMs"));
                    break;
                }
                case CREATE_RAW_TABLE: {
                    info._createRawTable(op.getInt("tableId"), op.getInt("columns"), op.getByteBuffer("metadata"));
                    break;
                }
                case UPDATE_RAW_TABLE_METADATA: {
                    info.updateRawTableMetadata(op.getInt("tableId"), op.getByteBuffer("metadata"));
                    break;
                }
                case CREATE_DEPENDENCY: {
                    info._createDependency(op.get("parents", new TypeReference<List<Integer>>() {
                    }), op.get("children", new TypeReference<List<Integer>>() {
                    }), op.getString("commandPrefix"), op.getByteBufferList("data"), op.getString("comment"),
                            op.getString("framework"), op.getString("frameworkVersion"),
                            op.get("dependencyType", DependencyType.class), op.getInt("dependencyId"),
                            op.getLong("creationTimeMs"));
                    break;
                }
                default:
                    throw new IOException("Invalid op type " + op);
                }
            } catch (SuspectedFileSizeException e) {
                throw new IOException(e);
            } catch (BlockInfoException e) {
                throw new IOException(e);
            } catch (FileDoesNotExistException e) {
                throw new IOException(e);
            } catch (FileAlreadyExistException e) {
                throw new IOException(e);
            } catch (InvalidPathException e) {
                throw new IOException(e);
            } catch (TachyonException e) {
                throw new IOException(e);
            } catch (TableDoesNotExistException e) {
                throw new IOException(e);
            }
        }

        is.close();
        ufs.close();
    }

    /**
     * Make the edit log up-to-date, It will delete all editlogs since sBackUpLogStartNum.
     * 
     * @param path The path of the edit logs
     * @param info The Master Info
     */
    public static void markUpToDate(String path, MasterInfo info) {
        UnderFileSystem ufs = UnderFileSystem.get(path, info.getTachyonConf());
        String folder = path.substring(0, path.lastIndexOf(TachyonURI.SEPARATOR) + 1) + "completed";
        try {
            // delete all loaded editlogs since mBackupLogStartNum.
            String toDelete = CommonUtils.concat(folder, sBackUpLogStartNum + ".editLog");
            while (ufs.exists(toDelete)) {
                LOG.info("Deleting editlog " + toDelete);
                ufs.delete(toDelete, true);
                sBackUpLogStartNum++;
                toDelete = CommonUtils.concat(folder, sBackUpLogStartNum + ".editLog");
            }
        } catch (IOException e) {
            throw Throwables.propagate(e);
        }
        sBackUpLogStartNum = -1;
    }

    /** When a master is replaying an edit log, mark the current edit log as an mInactive one. */
    private final boolean mInactive;

    private final String mPath;

    /** Writer used to serialize Operations into the edit log. */
    private final ObjectWriter mWriter;

    private UnderFileSystem mUfs;

    /** Raw output stream to the UnderFS */
    private OutputStream mOs;

    /** Wraps the raw output stream. */
    private DataOutputStream mDos;

    // Starting from 1.
    private long mFlushedTransactionId = 0;

    private long mTransactionId = 0;

    private int mCurrentLogFileNum = 0;

    private int mMaxLogSize = 5 * Constants.MB;

    private final TachyonConf mTachyonConf;

    /**
     * Create a new EditLog
     * 
     * @param path The path of the edit logs.
     * @param inactive If a master is replaying an edit log, the current edit log is inactive.
     * @param transactionId The beginning transactionId of the edit log
     * @throws IOException
     */
    public EditLog(String path, boolean inactive, long transactionId, TachyonConf tachyonConf) throws IOException {
        mInactive = inactive;
        mTachyonConf = tachyonConf;
        if (!mInactive) {
            LOG.info("Creating edit log file " + path);
            mPath = path;
            mUfs = UnderFileSystem.get(path, mTachyonConf);
            if (sBackUpLogStartNum != -1) {
                LOG.info("Deleting completed editlogs that are part of the image.");
                deleteCompletedLogs(path, sBackUpLogStartNum, tachyonConf);
                LOG.info("Backing up logs from " + sBackUpLogStartNum + " since image is not updated.");
                String folder = path.substring(0, path.lastIndexOf(TachyonURI.SEPARATOR) + 1) + "/completed";
                mUfs.mkdirs(folder, true);
                String toRename = CommonUtils.concat(folder, sBackUpLogStartNum + ".editLog");
                int currentLogFileNum = 0;
                String dstPath = CommonUtils.concat(folder, currentLogFileNum + ".editLog");
                while (mUfs.exists(toRename)) {
                    mUfs.rename(toRename, dstPath);
                    LOG.info("Rename " + toRename + " to " + dstPath);
                    currentLogFileNum++;
                    sBackUpLogStartNum++;
                    toRename = CommonUtils.concat(folder, sBackUpLogStartNum + ".editLog");
                    dstPath = CommonUtils.concat(folder, currentLogFileNum + ".editLog");
                }
                if (mUfs.exists(path)) {
                    dstPath = CommonUtils.concat(folder, currentLogFileNum + ".editLog");
                    mUfs.rename(path, dstPath);
                    LOG.info("Rename " + path + " to " + dstPath);
                    currentLogFileNum++;
                }
                sBackUpLogStartNum = -1;
            }

            // In case this file is created by different dfs-clients, which has been
            // fixed in HDFS-3755 since 3.0.0, 2.0.2-alpha
            if (mUfs.exists(path)) {
                mUfs.delete(path, true);
            }
            mOs = mUfs.create(path);
            mDos = new DataOutputStream(mOs);
            LOG.info("Created file " + path);
            mFlushedTransactionId = transactionId;
            mTransactionId = transactionId;
            mWriter = JsonObject.createObjectMapper().writer();
        } else {
            mPath = null;
            mUfs = null;
            mOs = null;
            mDos = null;
            mWriter = null;
        }
    }

    /**
     * Only close the currently opened output streams.
     */
    private synchronized void _closeActiveStream() {
        try {
            if (mDos != null) {
                mDos.close();
            }
            if (mOs != null) {
                mOs.close();
            }
        } catch (IOException e) {
            throw Throwables.propagate(e);
        }
    }

    /**
     * Log an addBlock operation. Do nothing if the edit log is inactive.
     * 
     * @param fileId The id of the file
     * @param blockIndex The index of the block to be added
     * @param blockLength The length of the block to be added
     * @param opTimeMs The time of the addBlock operation, in milliseconds
     */
    public synchronized void addBlock(int fileId, int blockIndex, long blockLength, long opTimeMs) {
        if (mInactive) {
            return;
        }

        EditLogOperation operation = new EditLogOperation(EditLogOperationType.ADD_BLOCK, ++mTransactionId)
                .withParameter("fileId", fileId).withParameter("blockIndex", blockIndex)
                .withParameter("blockLength", blockLength).withParameter("opTimeMs", opTimeMs);
        writeOperation(operation);
    }

    /**
     * Log an addCheckpoint operation. Do nothing if the edit log is inactive.
     * 
     * @param fileId The file to add the checkpoint
     * @param length The length of the checkpoint
     * @param checkpointPath The path of the checkpoint
     * @param opTimeMs The time of the addCheckpoint operation, in milliseconds
     */
    public synchronized void addCheckpoint(int fileId, long length, TachyonURI checkpointPath, long opTimeMs) {
        if (mInactive) {
            return;
        }

        EditLogOperation operation = new EditLogOperation(EditLogOperationType.ADD_CHECKPOINT, ++mTransactionId)
                .withParameter("fileId", fileId).withParameter("length", length)
                .withParameter("path", checkpointPath.toString()).withParameter("opTimeMs", opTimeMs);
        writeOperation(operation);
    }

    /**
     * Close the log.
     */
    public synchronized void close() {
        if (mInactive) {
            return;
        }

        try {
            _closeActiveStream();
            mUfs.close();
        } catch (IOException e) {
            throw Throwables.propagate(e);
        }
    }

    /**
     * Log a completeFile operation. Do nothing if the edit log is inactive.
     * 
     * @param fileId The id of the file
     * @param opTimeMs The time of the completeFile operation, in milliseconds
     */
    public synchronized void completeFile(int fileId, long opTimeMs) {
        if (mInactive) {
            return;
        }

        EditLogOperation operation = new EditLogOperation(EditLogOperationType.COMPLETE_FILE, ++mTransactionId)
                .withParameter("fileId", fileId).withParameter("opTimeMs", opTimeMs);
        writeOperation(operation);
    }

    /**
     * Log a createDependency operation. The parameters are like creating a new Dependency. Do nothing
     * if the edit log is inactive.
     * 
     * @param parents The input files' id of the dependency
     * @param children The output files' id of the dependency
     * @param commandPrefix The prefix of the command used for recomputation
     * @param data The list of the data used for recomputation
     * @param comment The comment of the dependency
     * @param framework The framework of the dependency, used for recomputation
     * @param frameworkVersion The version of the framework
     * @param dependencyType The type of the dependency, DependencyType.Wide or DependencyType.Narrow
     * @param depId The id of the dependency
     * @param creationTimeMs The create time of the dependency, in milliseconds
     */
    public synchronized void createDependency(List<Integer> parents, List<Integer> children, String commandPrefix,
            List<ByteBuffer> data, String comment, String framework, String frameworkVersion,
            DependencyType dependencyType, int depId, long creationTimeMs) {
        if (mInactive) {
            return;
        }

        EditLogOperation operation = new EditLogOperation(EditLogOperationType.CREATE_DEPENDENCY, ++mTransactionId)
                .withParameter("parents", parents).withParameter("children", children)
                .withParameter("commandPrefix", commandPrefix)
                .withParameter("data", Utils.byteBufferListToBase64(data)).withParameter("comment", comment)
                .withParameter("framework", framework).withParameter("frameworkVersion", frameworkVersion)
                .withParameter("dependencyType", dependencyType).withParameter("dependencyId", depId)
                .withParameter("creationTimeMs", creationTimeMs);
        writeOperation(operation);
    }

    /**
     * Log a createFile operation. Do nothing if the edit log is inactive.
     * 
     * @param recursive If recursive is true and the filesystem tree is not filled in all the way to
     *        path yet, it fills in the missing components.
     * @param path The path to create
     * @param directory If true, creates an InodeFolder instead of an Inode
     * @param blockSizeByte If it's a file, the block size for the Inode
     * @param creationTimeMs The time the file was created
     */
    public synchronized void createFile(boolean recursive, TachyonURI path, boolean directory, long blockSizeByte,
            long creationTimeMs) {
        if (mInactive) {
            return;
        }

        EditLogOperation operation = new EditLogOperation(EditLogOperationType.CREATE_FILE, ++mTransactionId)
                .withParameter("recursive", recursive).withParameter("path", path.toString())
                .withParameter("directory", directory).withParameter("blockSizeByte", blockSizeByte)
                .withParameter("creationTimeMs", creationTimeMs);
        writeOperation(operation);
    }

    /**
     * Log a createRawTable operation. Do nothing if the edit log is inactive.
     * 
     * @param tableId The id of the raw table
     * @param columns The number of columns in the table
     * @param metadata Additional metadata about the table
     */
    public synchronized void createRawTable(int tableId, int columns, ByteBuffer metadata) {
        if (mInactive) {
            return;
        }

        EditLogOperation operation = new EditLogOperation(EditLogOperationType.CREATE_RAW_TABLE, ++mTransactionId)
                .withParameter("tableId", tableId).withParameter("columns", columns)
                .withParameter("metadata", Utils.byteBufferToBase64(metadata));
        writeOperation(operation);
    }

    /**
     * Log a delete operation. Do nothing if the edit log is inactive.
     * 
     * @param fileId the file to be deleted.
     * @param recursive whether delete the file recursively or not.
     * @param opTimeMs The time of the delete operation, in milliseconds
     */
    public synchronized void delete(int fileId, boolean recursive, long opTimeMs) {
        if (mInactive) {
            return;
        }

        EditLogOperation operation = new EditLogOperation(EditLogOperationType.DELETE, ++mTransactionId)
                .withParameter("fileId", fileId).withParameter("recursive", recursive)
                .withParameter("opTimeMs", opTimeMs);
        writeOperation(operation);
    }

    /**
     * Delete the completed logs.
     * 
     * @param path The path of the logs
     * @param upTo The logs in the path from 0 to upTo-1 are completed and to be deleted
     * @param conf The {@link tachyon.conf.TachyonConf} instance
     */
    public void deleteCompletedLogs(String path, int upTo, TachyonConf conf) {
        UnderFileSystem ufs = UnderFileSystem.get(path, conf);
        String folder = path.substring(0, path.lastIndexOf(TachyonURI.SEPARATOR) + 1) + "completed";
        try {
            for (int i = 0; i < upTo; i++) {
                String toDelete = CommonUtils.concat(folder, i + ".editLog");
                LOG.info("Deleting editlog " + toDelete);
                ufs.delete(toDelete, true);
            }
        } catch (IOException e) {
            throw Throwables.propagate(e);
        }
    }

    /**
     * Flush the log onto the storage.
     */
    public synchronized void flush() {
        if (mInactive) {
            return;
        }

        try {
            mDos.flush();
            if (mOs instanceof FSDataOutputStream) {
                ((FSDataOutputStream) mOs).sync();
            }
            if (mDos.size() > mMaxLogSize) {
                rotateEditLog(mPath);
            }
        } catch (IOException e) {
            throw Throwables.propagate(e);
        }

        mFlushedTransactionId = mTransactionId;
    }

    /**
     * Get the current TransactionId and FlushedTransactionId
     * 
     * @return (TransactionId, FlushedTransactionId)
     */
    public synchronized Pair<Long, Long> getTransactionIds() {
        return new Pair<Long, Long>(mTransactionId, mFlushedTransactionId);
    }

    /**
     * Log a rename operation. Do nothing if the edit log is inactive.
     * 
     * @param fileId The id of the file to rename
     * @param dstPath The new path of the file
     * @param opTimeMs The time of the rename operation, in milliseconds
     */
    public synchronized void rename(int fileId, TachyonURI dstPath, long opTimeMs) {
        if (mInactive) {
            return;
        }

        EditLogOperation operation = new EditLogOperation(EditLogOperationType.RENAME, ++mTransactionId)
                .withParameter("fileId", fileId).withParameter("dstPath", dstPath.toString())
                .withParameter("opTimeMs", opTimeMs);
        writeOperation(operation);
    }

    /**
     * The edit log reaches the max log size and needs rotate. Do nothing if the edit log is inactive.
     * 
     * @param path The path of the edit log
     */
    public void rotateEditLog(String path) {
        if (mInactive) {
            return;
        }

        _closeActiveStream();
        LOG.info("Edit log max size of " + mMaxLogSize + " bytes reached, rotating edit log");
        String pathPrefix = path.substring(0, path.lastIndexOf(TachyonURI.SEPARATOR) + 1) + "completed";
        LOG.info("path: " + path + " prefix: " + pathPrefix);
        try {
            if (!mUfs.exists(pathPrefix)) {
                mUfs.mkdirs(pathPrefix, true);
            }
            String newPath = CommonUtils.concat(pathPrefix, (mCurrentLogFileNum++) + ".editLog");
            mUfs.rename(path, newPath);
            LOG.info("Renamed " + path + " to " + newPath);
            mOs = mUfs.create(path);
            mDos = new DataOutputStream(mOs);
            LOG.info("Created new log file " + path);
        } catch (IOException e) {
            throw Throwables.propagate(e);
        }
    }

    /**
     * Changes the max log size for testing purposes.
     * 
     * @param size
     */
    void setMaxLogSize(int size) {
        mMaxLogSize = size;
    }

    /**
     * Changes backup log start number for testing purposes.
     *
     * Note that we must set it back to -1 when test case ended.
     *
     * @param num
     */
    static void setBackUpLogStartNum(int num) {
        sBackUpLogStartNum = num;
    }

    /**
     * Log a setPinned operation. Do nothing if the edit log is inactive.
     * 
     * @param fileId The id of the file
     * @param pinned If true, the file is never evicted from memory
     * @param opTimeMs The time of the setPinned operation, in milliseconds
     */
    public synchronized void setPinned(int fileId, boolean pinned, long opTimeMs) {
        if (mInactive) {
            return;
        }

        EditLogOperation operation = new EditLogOperation(EditLogOperationType.SET_PINNED, ++mTransactionId)
                .withParameter("fileId", fileId).withParameter("pinned", pinned)
                .withParameter("opTimeMs", opTimeMs);
        writeOperation(operation);
    }

    /**
     * Log an updateRawTableMetadata operation. Do nothing if the edit log is inactive.
     * 
     * @param tableId The id of the raw table
     * @param metadata The new metadata of the raw table
     */
    public synchronized void updateRawTableMetadata(int tableId, ByteBuffer metadata) {
        if (mInactive) {
            return;
        }

        EditLogOperation operation = new EditLogOperation(EditLogOperationType.UPDATE_RAW_TABLE_METADATA,
                ++mTransactionId).withParameter("tableId", tableId).withParameter("metadata",
                        Utils.byteBufferToBase64(metadata));
        writeOperation(operation);
    }

    private void writeOperation(EditLogOperation operation) {
        try {
            mWriter.writeValue(mDos, operation);
            mDos.writeByte('\n');
        } catch (IOException e) {
            throw Throwables.propagate(e);
        }
    }
}