org.apache.ranger.audit.provider.hdfs.HdfsLogDestination.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.ranger.audit.provider.hdfs.HdfsLogDestination.java

Source

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.ranger.audit.provider.hdfs;

import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.UnsupportedEncodingException;
import java.net.URI;
import java.util.Map;

import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.ranger.audit.model.AuditEventBase;
import org.apache.ranger.audit.provider.DebugTracer;
import org.apache.ranger.audit.provider.LogDestination;
import org.apache.ranger.audit.provider.MiscUtil;

public class HdfsLogDestination<T> implements LogDestination<T> {
    public final static String EXCP_MSG_FILESYSTEM_CLOSED = "Filesystem closed";

    private String name = getClass().getName();

    private String mDirectory = null;
    private String mFile = null;
    private int mFlushIntervalSeconds = 1 * 60;
    private String mEncoding = null;
    private boolean mIsAppend = false;
    private int mRolloverIntervalSeconds = 24 * 60 * 60;
    private int mOpenRetryIntervalSeconds = 60;
    private DebugTracer mLogger = null;

    private FSDataOutputStream mFsDataOutStream = null;
    private OutputStreamWriter mWriter = null;
    private String mHdfsFilename = null;
    private long mNextRolloverTime = 0;
    private long mNextFlushTime = 0;
    private long mLastOpenFailedTime = 0;
    private boolean mIsStopInProgress = false;
    private Map<String, String> configProps = null;

    public HdfsLogDestination(DebugTracer tracer) {
        mLogger = tracer;
    }

    public void setName(String name) {
        this.name = name;
    }

    /* (non-Javadoc)
     * @see org.apache.ranger.audit.provider.LogDestination#getName()
     */
    @Override
    public String getName() {
        return name;
    }

    public String getDirectory() {
        return mDirectory;
    }

    public void setDirectory(String directory) {
        this.mDirectory = directory;
    }

    public String getFile() {
        return mFile;
    }

    public void setFile(String file) {
        this.mFile = file;
    }

    public int getFlushIntervalSeconds() {
        return mFlushIntervalSeconds;
    }

    public void setFlushIntervalSeconds(int flushIntervalSeconds) {
        mFlushIntervalSeconds = flushIntervalSeconds;
    }

    public String getEncoding() {
        return mEncoding;
    }

    public void setEncoding(String encoding) {
        mEncoding = encoding;
    }

    public int getRolloverIntervalSeconds() {
        return mRolloverIntervalSeconds;
    }

    public void setRolloverIntervalSeconds(int rolloverIntervalSeconds) {
        this.mRolloverIntervalSeconds = rolloverIntervalSeconds;
    }

    public int getOpenRetryIntervalSeconds() {
        return mOpenRetryIntervalSeconds;
    }

    public void setOpenRetryIntervalSeconds(int minIntervalOpenRetrySeconds) {
        this.mOpenRetryIntervalSeconds = minIntervalOpenRetrySeconds;
    }

    @Override
    public void start() {
        mLogger.debug("==> HdfsLogDestination.start()");

        openFile();

        mLogger.debug("<== HdfsLogDestination.start()");
    }

    @Override
    public void stop() {
        mLogger.debug("==> HdfsLogDestination.stop()");

        mIsStopInProgress = true;

        closeFile();

        mIsStopInProgress = false;

        mLogger.debug("<== HdfsLogDestination.stop()");
    }

    @Override
    public boolean isAvailable() {
        return mWriter != null;
    }

    @Override
    public boolean send(AuditEventBase log) {
        boolean ret = true;

        if (log != null) {
            String msg = MiscUtil.stringify(log);

            ret = sendStringified(msg);
        }

        return ret;
    }

    @Override
    public boolean send(AuditEventBase[] logs) {
        for (int i = 0; i < logs.length; i++) {
            boolean ret = send(logs[i]);
            if (!ret) {
                return ret;
            }
        }
        return true;
    }

    @Override
    public boolean sendStringified(String log) {
        boolean ret = false;

        checkFileStatus();

        OutputStreamWriter writer = mWriter;

        if (writer != null) {
            try {
                writer.write(log + MiscUtil.LINE_SEPARATOR);

                ret = true;
            } catch (IOException excp) {
                mLogger.warn("HdfsLogDestination.sendStringified(): write failed", excp);

                closeFile();
            }
        }

        return ret;
    }

    @Override
    public boolean sendStringified(String[] logs) {
        for (int i = 0; i < logs.length; i++) {
            boolean ret = sendStringified(logs[i]);
            if (!ret) {
                return ret;
            }
        }
        return true;
    }

    @Override
    public boolean flush() {
        mLogger.debug("==> HdfsLogDestination.flush()");

        boolean ret = false;

        OutputStreamWriter writer = mWriter;

        if (writer != null) {
            try {
                writer.flush();

                ret = true;
            } catch (IOException excp) {
                logException("HdfsLogDestination: flush() failed", excp);
            }
        }

        FSDataOutputStream ostream = mFsDataOutStream;

        if (ostream != null) {
            try {
                ostream.hflush();

                ret = true;
            } catch (IOException excp) {
                logException("HdfsLogDestination: hflush() failed", excp);
            }
        }

        if (ret) {
            mNextFlushTime = System.currentTimeMillis() + (mFlushIntervalSeconds * 1000L);
        }

        mLogger.debug("<== HdfsLogDestination.flush()");

        return ret;
    }

    private void openFile() {
        mLogger.debug("==> HdfsLogDestination.openFile()");

        closeFile();

        mNextRolloverTime = MiscUtil.getNextRolloverTime(mNextRolloverTime, (mRolloverIntervalSeconds * 1000L));

        long startTime = MiscUtil.getRolloverStartTime(mNextRolloverTime, (mRolloverIntervalSeconds * 1000L));

        mHdfsFilename = MiscUtil.replaceTokens(mDirectory + Path.SEPARATOR + mFile, startTime);

        FSDataOutputStream ostream = null;
        FileSystem fileSystem = null;
        Path pathLogfile = null;
        Configuration conf = null;
        boolean bOverwrite = false;

        try {
            mLogger.debug("HdfsLogDestination.openFile(): opening file " + mHdfsFilename);

            URI uri = URI.create(mHdfsFilename);

            // TODO: mechanism to XA-HDFS plugin to disable auditing of access checks to the current HDFS file

            conf = createConfiguration();
            pathLogfile = new Path(mHdfsFilename);
            fileSystem = FileSystem.get(uri, conf);

            try {
                if (fileSystem.exists(pathLogfile)) { // file already exists. either append to the file or write to a new file
                    if (mIsAppend) {
                        mLogger.info("HdfsLogDestination.openFile(): opening file for append " + mHdfsFilename);

                        ostream = fileSystem.append(pathLogfile);
                    } else {
                        mHdfsFilename = getNewFilename(mHdfsFilename, fileSystem);
                        pathLogfile = new Path(mHdfsFilename);
                    }
                }

                // if file does not exist or if mIsAppend==false, create the file
                if (ostream == null) {
                    mLogger.info("HdfsLogDestination.openFile(): opening file for write " + mHdfsFilename);

                    createParents(pathLogfile, fileSystem);
                    ostream = fileSystem.create(pathLogfile, bOverwrite);
                }
            } catch (IOException excp) {
                // append may not be supported by the filesystem; or the file might already be open by another application. Try a different filename
                String failedFilename = mHdfsFilename;

                mHdfsFilename = getNewFilename(mHdfsFilename, fileSystem);
                pathLogfile = new Path(mHdfsFilename);

                mLogger.info("HdfsLogDestination.openFile(): failed in opening file " + failedFilename
                        + ". Will try opening " + mHdfsFilename);
            }

            if (ostream == null) {
                mLogger.info("HdfsLogDestination.openFile(): opening file for write " + mHdfsFilename);

                createParents(pathLogfile, fileSystem);
                ostream = fileSystem.create(pathLogfile, bOverwrite);
            }
        } catch (Throwable ex) {
            mLogger.warn("HdfsLogDestination.openFile() failed", ex);
            //      } finally {
            // TODO: unset the property set above to exclude auditing of logfile opening
            //        System.setProperty(hdfsCurrentFilenameProperty, null);
        }

        mWriter = createWriter(ostream);

        if (mWriter != null) {
            mLogger.debug("HdfsLogDestination.openFile(): opened file " + mHdfsFilename);

            mFsDataOutStream = ostream;
            mNextFlushTime = System.currentTimeMillis() + (mFlushIntervalSeconds * 1000L);
            mLastOpenFailedTime = 0;
        } else {
            mLogger.warn("HdfsLogDestination.openFile(): failed to open file for write " + mHdfsFilename);

            mHdfsFilename = null;
            mLastOpenFailedTime = System.currentTimeMillis();
        }

        mLogger.debug("<== HdfsLogDestination.openFile(" + mHdfsFilename + ")");
    }

    private void closeFile() {
        mLogger.debug("==> HdfsLogDestination.closeFile()");

        flush();

        OutputStreamWriter writer = mWriter;

        mWriter = null;
        mFsDataOutStream = null;

        if (writer != null) {
            try {
                mLogger.info("HdfsLogDestination.closeFile(): closing file " + mHdfsFilename);

                writer.close();
            } catch (IOException excp) {
                logException("HdfsLogDestination: failed to close file " + mHdfsFilename, excp);
            }
        }

        mLogger.debug("<== HdfsLogDestination.closeFile()");
    }

    private void rollover() {
        mLogger.debug("==> HdfsLogDestination.rollover()");

        closeFile();

        openFile();

        mLogger.debug("<== HdfsLogDestination.rollover()");
    }

    private void checkFileStatus() {
        long now = System.currentTimeMillis();

        if (mWriter == null) {
            if (now > (mLastOpenFailedTime + (mOpenRetryIntervalSeconds * 1000L))) {
                openFile();
            }
        } else if (now > mNextRolloverTime) {
            rollover();
        } else if (now > mNextFlushTime) {
            flush();
        }
    }

    private OutputStreamWriter createWriter(OutputStream os) {
        OutputStreamWriter writer = null;

        if (os != null) {
            if (mEncoding != null) {
                try {
                    writer = new OutputStreamWriter(os, mEncoding);
                } catch (UnsupportedEncodingException excp) {
                    mLogger.warn("HdfsLogDestination.createWriter(): failed to create output writer.", excp);
                }
            }

            if (writer == null) {
                writer = new OutputStreamWriter(os);
            }
        }

        return writer;
    }

    private void createParents(Path pathLogfile, FileSystem fileSystem) {
        try {
            Path parentPath = pathLogfile != null ? pathLogfile.getParent() : null;

            if (parentPath != null && fileSystem != null && !fileSystem.exists(parentPath)) {
                fileSystem.mkdirs(parentPath);
            }
        } catch (IOException e) {
            logException("HdfsLogDestination.createParents() failed", e);
        } catch (Throwable e) {
            mLogger.warn("HdfsLogDestination.createParents() failed", e);
        }
    }

    private String getNewFilename(String fileName, FileSystem fileSystem) {
        if (fileName == null) {
            return "";
        }

        for (int i = 1;; i++) {
            String ret = fileName;

            String strToAppend = "-" + Integer.toString(i);

            int extnPos = ret.lastIndexOf(".");

            if (extnPos < 0) {
                ret += strToAppend;
            } else {
                String extn = ret.substring(extnPos);

                ret = ret.substring(0, extnPos) + strToAppend + extn;
            }

            if (fileSystem != null && fileExists(ret, fileSystem)) {
                continue;
            } else {
                return ret;
            }
        }
    }

    private boolean fileExists(String fileName, FileSystem fileSystem) {
        boolean ret = false;

        if (fileName != null && fileSystem != null) {
            Path path = new Path(fileName);

            try {
                ret = fileSystem.exists(path);
            } catch (IOException excp) {
                // ignore
            }
        }

        return ret;
    }

    private void logException(String msg, IOException excp) {
        // during shutdown, the underlying FileSystem might already be closed; so don't print error details

        if (mIsStopInProgress) {
            return;
        }

        String excpMsgToExclude = EXCP_MSG_FILESYSTEM_CLOSED;
        String excpMsg = excp != null ? excp.getMessage() : null;
        boolean excpExcludeLogging = (excpMsg != null && excpMsg.contains(excpMsgToExclude));

        if (!excpExcludeLogging) {
            mLogger.warn(msg, excp);
        }
    }

    @Override
    public String toString() {
        StringBuilder sb = new StringBuilder();

        sb.append("HdfsLogDestination {");
        sb.append("Directory=").append(mDirectory).append("; ");
        sb.append("File=").append(mFile).append("; ");
        sb.append("RolloverIntervalSeconds=").append(mRolloverIntervalSeconds);
        sb.append("}");

        return sb.toString();
    }

    public void setConfigProps(Map<String, String> configProps) {
        this.configProps = configProps;
    }

    Configuration createConfiguration() {
        Configuration conf = new Configuration();
        if (configProps != null) {
            for (Map.Entry<String, String> entry : configProps.entrySet()) {
                String key = entry.getKey();
                String value = entry.getValue();
                // for ease of install config file may contain properties with empty value, skip those
                if (StringUtils.isNotEmpty(value)) {
                    conf.set(key, value);
                }
                mLogger.info("Adding property to HDFS config: " + key + " => " + value);
            }
        }

        mLogger.info("Returning HDFS Filesystem Config: " + conf.toString());
        return conf;
    }
}