Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.ranger.audit.provider.hdfs; import java.io.IOException; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.UnsupportedEncodingException; import java.net.URI; import java.util.Map; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.ranger.audit.model.AuditEventBase; import org.apache.ranger.audit.provider.DebugTracer; import org.apache.ranger.audit.provider.LogDestination; import org.apache.ranger.audit.provider.MiscUtil; public class HdfsLogDestination<T> implements LogDestination<T> { public final static String EXCP_MSG_FILESYSTEM_CLOSED = "Filesystem closed"; private String name = getClass().getName(); private String mDirectory = null; private String mFile = null; private int mFlushIntervalSeconds = 1 * 60; private String mEncoding = null; private boolean mIsAppend = false; private int mRolloverIntervalSeconds = 24 * 60 * 60; private int mOpenRetryIntervalSeconds = 60; private DebugTracer mLogger = null; private FSDataOutputStream mFsDataOutStream = null; private OutputStreamWriter mWriter = null; private String mHdfsFilename = null; private long mNextRolloverTime = 0; private long mNextFlushTime = 0; private long mLastOpenFailedTime = 0; private boolean mIsStopInProgress = false; private Map<String, String> configProps = null; public HdfsLogDestination(DebugTracer tracer) { mLogger = tracer; } public void setName(String name) { this.name = name; } /* (non-Javadoc) * @see org.apache.ranger.audit.provider.LogDestination#getName() */ @Override public String getName() { return name; } public String getDirectory() { return mDirectory; } public void setDirectory(String directory) { this.mDirectory = directory; } public String getFile() { return mFile; } public void setFile(String file) { this.mFile = file; } public int getFlushIntervalSeconds() { return mFlushIntervalSeconds; } public void setFlushIntervalSeconds(int flushIntervalSeconds) { mFlushIntervalSeconds = flushIntervalSeconds; } public String getEncoding() { return mEncoding; } public void setEncoding(String encoding) { mEncoding = encoding; } public int getRolloverIntervalSeconds() { return mRolloverIntervalSeconds; } public void setRolloverIntervalSeconds(int rolloverIntervalSeconds) { this.mRolloverIntervalSeconds = rolloverIntervalSeconds; } public int getOpenRetryIntervalSeconds() { return mOpenRetryIntervalSeconds; } public void setOpenRetryIntervalSeconds(int minIntervalOpenRetrySeconds) { this.mOpenRetryIntervalSeconds = minIntervalOpenRetrySeconds; } @Override public void start() { mLogger.debug("==> HdfsLogDestination.start()"); openFile(); mLogger.debug("<== HdfsLogDestination.start()"); } @Override public void stop() { mLogger.debug("==> HdfsLogDestination.stop()"); mIsStopInProgress = true; closeFile(); mIsStopInProgress = false; mLogger.debug("<== HdfsLogDestination.stop()"); } @Override public boolean isAvailable() { return mWriter != null; } @Override public boolean send(AuditEventBase log) { boolean ret = true; if (log != null) { String msg = MiscUtil.stringify(log); ret = sendStringified(msg); } return ret; } @Override public boolean send(AuditEventBase[] logs) { for (int i = 0; i < logs.length; i++) { boolean ret = send(logs[i]); if (!ret) { return ret; } } return true; } @Override public boolean sendStringified(String log) { boolean ret = false; checkFileStatus(); OutputStreamWriter writer = mWriter; if (writer != null) { try { writer.write(log + MiscUtil.LINE_SEPARATOR); ret = true; } catch (IOException excp) { mLogger.warn("HdfsLogDestination.sendStringified(): write failed", excp); closeFile(); } } return ret; } @Override public boolean sendStringified(String[] logs) { for (int i = 0; i < logs.length; i++) { boolean ret = sendStringified(logs[i]); if (!ret) { return ret; } } return true; } @Override public boolean flush() { mLogger.debug("==> HdfsLogDestination.flush()"); boolean ret = false; OutputStreamWriter writer = mWriter; if (writer != null) { try { writer.flush(); ret = true; } catch (IOException excp) { logException("HdfsLogDestination: flush() failed", excp); } } FSDataOutputStream ostream = mFsDataOutStream; if (ostream != null) { try { ostream.hflush(); ret = true; } catch (IOException excp) { logException("HdfsLogDestination: hflush() failed", excp); } } if (ret) { mNextFlushTime = System.currentTimeMillis() + (mFlushIntervalSeconds * 1000L); } mLogger.debug("<== HdfsLogDestination.flush()"); return ret; } private void openFile() { mLogger.debug("==> HdfsLogDestination.openFile()"); closeFile(); mNextRolloverTime = MiscUtil.getNextRolloverTime(mNextRolloverTime, (mRolloverIntervalSeconds * 1000L)); long startTime = MiscUtil.getRolloverStartTime(mNextRolloverTime, (mRolloverIntervalSeconds * 1000L)); mHdfsFilename = MiscUtil.replaceTokens(mDirectory + Path.SEPARATOR + mFile, startTime); FSDataOutputStream ostream = null; FileSystem fileSystem = null; Path pathLogfile = null; Configuration conf = null; boolean bOverwrite = false; try { mLogger.debug("HdfsLogDestination.openFile(): opening file " + mHdfsFilename); URI uri = URI.create(mHdfsFilename); // TODO: mechanism to XA-HDFS plugin to disable auditing of access checks to the current HDFS file conf = createConfiguration(); pathLogfile = new Path(mHdfsFilename); fileSystem = FileSystem.get(uri, conf); try { if (fileSystem.exists(pathLogfile)) { // file already exists. either append to the file or write to a new file if (mIsAppend) { mLogger.info("HdfsLogDestination.openFile(): opening file for append " + mHdfsFilename); ostream = fileSystem.append(pathLogfile); } else { mHdfsFilename = getNewFilename(mHdfsFilename, fileSystem); pathLogfile = new Path(mHdfsFilename); } } // if file does not exist or if mIsAppend==false, create the file if (ostream == null) { mLogger.info("HdfsLogDestination.openFile(): opening file for write " + mHdfsFilename); createParents(pathLogfile, fileSystem); ostream = fileSystem.create(pathLogfile, bOverwrite); } } catch (IOException excp) { // append may not be supported by the filesystem; or the file might already be open by another application. Try a different filename String failedFilename = mHdfsFilename; mHdfsFilename = getNewFilename(mHdfsFilename, fileSystem); pathLogfile = new Path(mHdfsFilename); mLogger.info("HdfsLogDestination.openFile(): failed in opening file " + failedFilename + ". Will try opening " + mHdfsFilename); } if (ostream == null) { mLogger.info("HdfsLogDestination.openFile(): opening file for write " + mHdfsFilename); createParents(pathLogfile, fileSystem); ostream = fileSystem.create(pathLogfile, bOverwrite); } } catch (Throwable ex) { mLogger.warn("HdfsLogDestination.openFile() failed", ex); // } finally { // TODO: unset the property set above to exclude auditing of logfile opening // System.setProperty(hdfsCurrentFilenameProperty, null); } mWriter = createWriter(ostream); if (mWriter != null) { mLogger.debug("HdfsLogDestination.openFile(): opened file " + mHdfsFilename); mFsDataOutStream = ostream; mNextFlushTime = System.currentTimeMillis() + (mFlushIntervalSeconds * 1000L); mLastOpenFailedTime = 0; } else { mLogger.warn("HdfsLogDestination.openFile(): failed to open file for write " + mHdfsFilename); mHdfsFilename = null; mLastOpenFailedTime = System.currentTimeMillis(); } mLogger.debug("<== HdfsLogDestination.openFile(" + mHdfsFilename + ")"); } private void closeFile() { mLogger.debug("==> HdfsLogDestination.closeFile()"); flush(); OutputStreamWriter writer = mWriter; mWriter = null; mFsDataOutStream = null; if (writer != null) { try { mLogger.info("HdfsLogDestination.closeFile(): closing file " + mHdfsFilename); writer.close(); } catch (IOException excp) { logException("HdfsLogDestination: failed to close file " + mHdfsFilename, excp); } } mLogger.debug("<== HdfsLogDestination.closeFile()"); } private void rollover() { mLogger.debug("==> HdfsLogDestination.rollover()"); closeFile(); openFile(); mLogger.debug("<== HdfsLogDestination.rollover()"); } private void checkFileStatus() { long now = System.currentTimeMillis(); if (mWriter == null) { if (now > (mLastOpenFailedTime + (mOpenRetryIntervalSeconds * 1000L))) { openFile(); } } else if (now > mNextRolloverTime) { rollover(); } else if (now > mNextFlushTime) { flush(); } } private OutputStreamWriter createWriter(OutputStream os) { OutputStreamWriter writer = null; if (os != null) { if (mEncoding != null) { try { writer = new OutputStreamWriter(os, mEncoding); } catch (UnsupportedEncodingException excp) { mLogger.warn("HdfsLogDestination.createWriter(): failed to create output writer.", excp); } } if (writer == null) { writer = new OutputStreamWriter(os); } } return writer; } private void createParents(Path pathLogfile, FileSystem fileSystem) { try { Path parentPath = pathLogfile != null ? pathLogfile.getParent() : null; if (parentPath != null && fileSystem != null && !fileSystem.exists(parentPath)) { fileSystem.mkdirs(parentPath); } } catch (IOException e) { logException("HdfsLogDestination.createParents() failed", e); } catch (Throwable e) { mLogger.warn("HdfsLogDestination.createParents() failed", e); } } private String getNewFilename(String fileName, FileSystem fileSystem) { if (fileName == null) { return ""; } for (int i = 1;; i++) { String ret = fileName; String strToAppend = "-" + Integer.toString(i); int extnPos = ret.lastIndexOf("."); if (extnPos < 0) { ret += strToAppend; } else { String extn = ret.substring(extnPos); ret = ret.substring(0, extnPos) + strToAppend + extn; } if (fileSystem != null && fileExists(ret, fileSystem)) { continue; } else { return ret; } } } private boolean fileExists(String fileName, FileSystem fileSystem) { boolean ret = false; if (fileName != null && fileSystem != null) { Path path = new Path(fileName); try { ret = fileSystem.exists(path); } catch (IOException excp) { // ignore } } return ret; } private void logException(String msg, IOException excp) { // during shutdown, the underlying FileSystem might already be closed; so don't print error details if (mIsStopInProgress) { return; } String excpMsgToExclude = EXCP_MSG_FILESYSTEM_CLOSED; ; String excpMsg = excp != null ? excp.getMessage() : null; boolean excpExcludeLogging = (excpMsg != null && excpMsg.contains(excpMsgToExclude)); if (!excpExcludeLogging) { mLogger.warn(msg, excp); } } @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("HdfsLogDestination {"); sb.append("Directory=").append(mDirectory).append("; "); sb.append("File=").append(mFile).append("; "); sb.append("RolloverIntervalSeconds=").append(mRolloverIntervalSeconds); sb.append("}"); return sb.toString(); } public void setConfigProps(Map<String, String> configProps) { this.configProps = configProps; } Configuration createConfiguration() { Configuration conf = new Configuration(); if (configProps != null) { for (Map.Entry<String, String> entry : configProps.entrySet()) { String key = entry.getKey(); String value = entry.getValue(); // for ease of install config file may contain properties with empty value, skip those if (StringUtils.isNotEmpty(value)) { conf.set(key, value); } mLogger.info("Adding property to HDFS config: " + key + " => " + value); } } mLogger.info("Returning HDFS Filesystem Config: " + conf.toString()); return conf; } }