com.alibaba.jstorm.hdfs.spout.DirLock.java Source code

Introduction

Here is the source code for com.alibaba.jstorm.hdfs.spout.DirLock.java

Source

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 * <p/>
 * http://www.apache.org/licenses/LICENSE-2.0
 * <p/>
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.alibaba.jstorm.hdfs.spout;

import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import com.alibaba.jstorm.hdfs.common.HdfsUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;

/**
 * Facility to synchronize access to HDFS directory. The lock itself is represented
 * as a file in the same directory. Relies on atomic file creation.
 */
public class DirLock {
    private FileSystem fs;
    private final Path lockFile;
    public static final String DIR_LOCK_FILE = "DIRLOCK";
    private static final Logger LOG = LoggerFactory.getLogger(DirLock.class);

    private DirLock(FileSystem fs, Path lockFile) throws IOException {
        if (fs.isDirectory(lockFile)) {
            throw new IllegalArgumentException(lockFile.toString() + " is not a directory");
        }
        this.fs = fs;
        this.lockFile = lockFile;
    }

    /** Get a lock on file if not already locked
     *
     * @param fs
     * @param dir  the dir on which to get a lock
     * @return The lock object if it the lock was acquired. Returns null if the dir is already locked.
     * @throws IOException if there were errors
     */
    public static DirLock tryLock(FileSystem fs, Path dir) throws IOException {
        Path lockFile = getDirLockFile(dir);

        try {
            FSDataOutputStream ostream = HdfsUtils.tryCreateFile(fs, lockFile);
            if (ostream != null) {
                LOG.debug("Thread ({}) Acquired lock on dir {}", threadInfo(), dir);
                ostream.close();
                return new DirLock(fs, lockFile);
            } else {
                LOG.debug("Thread ({}) cannot lock dir {} as its already locked.", threadInfo(), dir);
                return null;
            }
        } catch (IOException e) {
            LOG.error("Error when acquiring lock on dir " + dir, e);
            throw e;
        }
    }

    private static Path getDirLockFile(Path dir) {
        return new Path(dir.toString() + Path.SEPARATOR_CHAR + DIR_LOCK_FILE);
    }

    private static String threadInfo() {
        return "ThdId=" + Thread.currentThread().getId() + ", ThdName=" + Thread.currentThread().getName();
    }

    /** Release lock on dir by deleting the lock file */
    public void release() throws IOException {
        if (!fs.delete(lockFile, false)) {
            LOG.error("Thread {} could not delete dir lock {} ", threadInfo(), lockFile);
        } else {
            LOG.debug("Thread {} Released dir lock {} ", threadInfo(), lockFile);
        }
    }

    /** if the lock on the directory is stale, take ownership */
    public static DirLock takeOwnershipIfStale(FileSystem fs, Path dirToLock, int lockTimeoutSec) {
        Path dirLockFile = getDirLockFile(dirToLock);

        long now = System.currentTimeMillis();
        long expiryTime = now - (lockTimeoutSec * 1000);

        try {
            long modTime = fs.getFileStatus(dirLockFile).getModificationTime();
            if (modTime <= expiryTime) {
                return takeOwnership(fs, dirLockFile);
            }
            return null;
        } catch (IOException e) {
            return null;
        }
    }

    private static DirLock takeOwnership(FileSystem fs, Path dirLockFile) throws IOException {
        if (fs instanceof DistributedFileSystem) {
            if (!((DistributedFileSystem) fs).recoverLease(dirLockFile)) {
                LOG.warn("Unable to recover lease on dir lock file " + dirLockFile
                        + " right now. Cannot transfer ownership. Will need to try later.");
                return null;
            }
        }

        // delete and recreate lock file
        if (fs.delete(dirLockFile, false)) { // returns false if somebody else already deleted it (to take ownership)
            FSDataOutputStream ostream = HdfsUtils.tryCreateFile(fs, dirLockFile);
            if (ostream != null) {
                ostream.close();
            }
            return new DirLock(fs, dirLockFile);
        }
        return null;
    }

    public Path getLockFile() {
        return lockFile;
    }
}