Java tutorial
/** * Copyright 2012 Alcatel-Lucent. * * See the NOTICE file distributed with this work for additional * information regarding copyright ownership. * Licensed to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package com.alu.e3.logger; import java.io.BufferedReader; import java.io.File; import java.io.FileFilter; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.nio.channels.FileChannel; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.Arrays; import java.util.Comparator; import java.util.Date; import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import org.apache.commons.lang.StringEscapeUtils; import com.alu.e3.Utilities; import com.alu.e3.common.E3Constant; import com.alu.e3.common.logging.Category; import com.alu.e3.common.logging.CategoryLogger; import com.alu.e3.common.logging.CategoryLoggerFactory; import com.alu.e3.common.logging.LoggingUtil; import com.alu.e3.common.logging.LoggingUtil.LogFileSource; import com.alu.e3.common.logging.NonJavaLogger; import com.alu.e3.common.tools.CommonTools; import com.alu.e3.data.model.Instance; import com.alu.e3.installer.NonExistingManagerException; import com.alu.e3.installer.command.SSHCommand; import com.alu.e3.installer.command.ShellCommandResult; import com.alu.e3.osgi.api.ITopology; import com.jcraft.jsch.JSchException; /** * The LogCollector class provides an object to traverse the topology * and visit each instance, collecting log files to a repository on * the system manager. * * There are still some unresolved issues in the design and/or * implementation of the Log Collector. These include: * <ul> * <li> Throughout the code, instances are referred to by IP-Address. * This is probably not right, and will have to be changed to some * sort of (non-volatile) instance identifier. * <li> If logging config files are moved or significantly edited * (or not set up properly on install) the Collector may not be * able to find remote log files. * </ul> */ public class LogCollector implements Callable<Long> { private static final String COLLECTION_PATH = System.getProperty("user.home") + "/apache-servicemix/data/logCollection"; private static final CategoryLogger logger = CategoryLoggerFactory.getLogger(LogCollector.class, Category.LOG); // Should collected files be deleted on the source after collection? private static final boolean deleteAfterCollect = true; // If a file already exists with the copied log-file name, append a ".1" etc to uniquify? // Don't uniquify if we don't delete source logs, since this will lead to local duplicates private static final boolean uniquifyCopiedFilenames = deleteAfterCollect; private static final int uniquifyLimit = 100; // unique filename extension limit, after this overwrite // Should the collector only collect (and delete) files owned by the ssh user? // This feature hasn't yet been tested ... private static final boolean collectOnlyUserOwnedLogs = false; // Set a short ssh connect timeout since we assume all instances are close, // and on timeout we can try again on next collection private static final int sshSessionTimeout = 10000; // 10s /* * Synchronization policy: allow only one writer to operate at a time. * Allow concurrent readers and writer, except that readers must ignore * any files the writer is currently writing, indicated by a specific suffix */ // For now: use a lock to allow only one writer (collector) to run at a time private static final Lock writerLock = new ReentrantLock(); // Reader should ignore these files - collector is currently writing private static final String workingFileSuffix = "~"; // Each logCollector instance gets its own serial number // A class-static atomic variable keeps the serial numbers unique private static final AtomicLong serialNumber = new AtomicLong(); private static long generateSerialNumber() { return serialNumber.getAndIncrement(); } private static final AtomicLong lastCompletedCollector = new AtomicLong(); private final long collectorSerialNumber; // Each collector can have its own topology (we could split large // topologies for collection by multiple collectors) private final ITopology topology; /** * Data structure to hold information about a (remote) file * */ private static class FileInfo { public String filePath; public String fileSize; public String fileModtime; FileInfo(String path, String size, String modtime) { filePath = path; fileSize = size; fileModtime = modtime; } } /** * File-name filter to check for log files (by extension) * and to ignore "working files" (files being written by collector) */ private static class LogFileFilter implements FileFilter { private String basename; public LogFileFilter(String basename) { this.basename = basename; } public boolean accept(File pathname) { String filename = pathname.getName(); if (basename != null) { return ((filename.matches(basename + LoggingUtil.rollingFileExtPatternString) || filename.matches(basename + LoggingUtil.dailyRollingFileExtPatternString) && !filename.endsWith(LogCollector.workingFileSuffix))); } else { return ((filename.matches(LoggingUtil.rollingFilePatternString) || filename.matches(LoggingUtil.dailyRollingFilePatternString) && !filename.endsWith(LogCollector.workingFileSuffix))); } } } /** * LogCollector constructor without parameters to satisfy spring-bean requirements. */ public LogCollector(ITopology topology) { this.topology = topology; this.collectorSerialNumber = LogCollector.generateSerialNumber(); } public long getCollectorID() { return collectorSerialNumber; } public static long getLastCollectorID() { return lastCompletedCollector.longValue(); } /** * Traverse the topology and collect logs from each instance found. * * @return The number of bytes collected in all logs * @throws InterruptedException * @throws TimeoutException */ @Override public Long call() throws InterruptedException, TimeoutException, NonExistingManagerException { if (logger.isDebugEnabled()) { logger.debug("Call to LogCollector.call()"); } long bytesCollected = collectAllLogs(0, TimeUnit.SECONDS); return Long.valueOf(bytesCollected); } /** * Traverse the topology and collect logs from each instance found. * * @param waitTimeout The time to wait if another collector is running * @param unit The time unit for waitTimeout * @return The number of bytes collected in all logs * @throws InterruptedException * @throws TimeoutException * @throws NonExistingManagerException */ public long collectAllLogs(long waitTimeout, TimeUnit unit) throws InterruptedException, TimeoutException, NonExistingManagerException { DateFormat dateFormat = new SimpleDateFormat("yyyy.MM.dd 'at' HH:mm:ss z"); String launchDate = dateFormat.format(new Date()); logger.debug("Launching system-manager log-file collection ({}) ...", launchDate); if (!LogCollector.writerLock.tryLock(waitTimeout, unit)) { logger.warn( "Attempt to run log collector but cannot acquire lock (another collection must be running)"); throw new TimeoutException("Timeout waiting to acquire log collector write lock"); } long collectionCount = 0L; logger.debug("LogCollector ID: {}", getCollectorID()); try { Set<String> visitedIPs = new HashSet<String>(); // Create the top-level log collection directory, if this is the first time run File collectionDir = new File(COLLECTION_PATH); if (!collectionDir.exists() && !collectionDir.mkdirs()) { logger.error("Unable to create log-collection directory: {}", COLLECTION_PATH); return 0L; } // Iterate through all instances in the current topology Instance logCollectorInstance = Utilities.getManagerByIP(CommonTools.getLocalHostname(), CommonTools.getLocalAddress(), topology.getInstancesByType(E3Constant.E3MANAGER), logger); List<Instance> instances = getInstanceList(this.topology); if (logger.isDebugEnabled()) { logger.debug("There are {} instances in the current topology", instances.size()); } for (Instance logSource : instances) { LogCollector.logInstance(logSource); // for debugging // Avoid visiting the same address twice String ipAddress = logSource.getInternalIP(); if (ipAddress == null) { logger.warn("Encountered instance node with null ipAddress during log collection!"); continue; } if (CommonTools.isLocal(ipAddress)) { ipAddress = E3Constant.localhost; // stay consistent } if (visitedIPs.contains(ipAddress)) { if (logger.isDebugEnabled()) { logger.debug("Skipping already-visited address: {}", ipAddress); } continue; } visitedIPs.add(ipAddress); // Create or verify the existence of a log-collection target directory String sanitizedHost = ipToCollectionDirectory(ipAddress); File instanceCollectionDir = new File(COLLECTION_PATH, sanitizedHost); if (instanceCollectionDir.exists()) { if (!instanceCollectionDir.isDirectory()) { logger.error("Log-collection target exists but is not a directory: {}", instanceCollectionDir.getAbsolutePath()); continue; } } else { if (!instanceCollectionDir.mkdirs()) { logger.error("Unable to create log-collection directory: {}", instanceCollectionDir.getAbsolutePath()); continue; } } // Finally, perform log collection // There may be a chance for parallelism here by farming the collection work for each instance // out to a separate worker thread. At a minimum the local collection could occur in parallel with // collection on a remote host. if (ipAddress.equalsIgnoreCase(E3Constant.localhost)) { try { collectionCount += collectAllLocalLogs(instanceCollectionDir); } catch (IOException ex) { logger.warn("Error trying to copy local log files to {}", instanceCollectionDir.getAbsolutePath()); } } else { try { collectionCount += collectAllRemoteLogs(logSource, logCollectorInstance, instanceCollectionDir); } catch (JSchException ex) { if (logger.isDebugEnabled()) { logger.debug("Could not connect to host: {}", logSource.getInternalIP()); logger.debug(ex.getLocalizedMessage()); } } catch (IOException ex) { if (logger.isDebugEnabled()) { logger.debug("Got IOException while connecting to or transferring files from host: {}", logSource.getInternalIP()); logger.debug(ex.getLocalizedMessage()); } } } // At this point the collection has "completed", even if IOExceptions could have // occurred and been caught above LogCollector.lastCompletedCollector.set(getCollectorID()); logger.debug("Completed log collection with ID: {} ({})", getCollectorID(), dateFormat.format(new Date())); } } finally { LogCollector.writerLock.unlock(); } return collectionCount; } /** * Traverse the topology and collect a certain number of lines from the * active logs on each instance found. * * @param numLines The number of lines to retrieve from each log file on each instance * @return An XML-structured string with a single <code><logCollection></code> node and * <code><log></code> node for each log-file found (including multiple instances). * @throws InterruptedException * @throws TimeoutException */ public String collectAllActiveLogs(int numLines) { StringBuilder sb = new StringBuilder(); Set<String> visitedIPs = new HashSet<String>(); // Iterate through all instances in the current topology // (Note: use TRACE-level log statements so that our logging doesn't appear in returned log tails) List<Instance> instances = getInstanceList(this.topology); logger.trace("There are {} instances in the current topology", instances.size()); Instance collectorInstance; try { collectorInstance = Utilities.getManagerByIP(CommonTools.getLocalHostname(), CommonTools.getLocalAddress(), topology.getInstancesByType(E3Constant.E3MANAGER), logger); } catch (NonExistingManagerException e) { String errMsg = "Cannot find manager at " + CommonTools.getLocalAddress() + " while collecting active logs."; logger.error(errMsg); return errMsg; } for (Instance instance : instances) { // Avoid visiting the same address twice String ipAddress = instance.getInternalIP(); if (ipAddress == null) { logger.warn("Encountered instance node with null ipAddress during log collection!"); continue; } if (CommonTools.isLocal(ipAddress)) { ipAddress = E3Constant.localhost; // stay consistent } if (visitedIPs.contains(ipAddress)) { logger.trace("Skipping already-visited address: {}", ipAddress); continue; } visitedIPs.add(ipAddress); // Finally, perform log collection (of active-log tails) if (ipAddress.equalsIgnoreCase(E3Constant.localhost)) { try { String logs = getTailOfLocalActiveLogs(numLines); if (logs != null) { sb.append(logs); } } catch (IOException ex) { logger.warn("Error trying to get tail of active log files from {}", ipAddress); } } else { try { String logs = getTailOfRemoteActiveLogs(instance, collectorInstance, numLines); if (logs != null) { sb.append(logs); } } catch (JSchException ex) { logger.warn("Could not connect to host: {}", instance.getInternalIP()); logger.warn(ex.getLocalizedMessage()); } catch (IOException ex) { logger.warn("Got IOException while connecting to or transferring files from host: {}", instance.getInternalIP()); logger.warn(ex.getLocalizedMessage()); } } } return LoggingResponseBuilder.logCollectionToXml(sb.toString()); } /** * A top-level method to return a certain number of most-recent log * lines from previously-connected log files from all instances. * * @param numLines The requested number of log lines to retrieve. * Fewer lines may be returned if sufficient log entries are not available. * @return An XML-structured string with a single <code><logCollection></code> node and * <code><log></code> node for each log-file found (including multiple instances). */ public String getCollectedLogLines(int numLines) { File collectionDir = new File(COLLECTION_PATH); if (!collectionDir.exists() || !collectionDir.isDirectory()) { return null; } if (logger.isDebugEnabled()) { logger.debug("Attempting to get all collected logs ..."); } StringBuilder logLines = new StringBuilder(); File[] files = collectionDir.listFiles(); // get a list of instance directories for (File item : files) { if (logger.isDebugEnabled()) { logger.debug("Collection directory entry: {}", item.getAbsolutePath()); } if (item.isDirectory()) { String ipAddress = collectionDirectoryToIP(item.getName()); if (logger.isDebugEnabled()) { logger.debug("Getting logs from: {}", ipAddress); } // First get the java logs String contents = getCollectedLogLinesFromInstance(ipAddress, LogFileSource.JAVA, numLines); if (contents != null) { logLines.append(contents); } // Next get the servicemix logs contents = getCollectedLogLinesFromInstance(ipAddress, LogFileSource.SMX, numLines); if (contents != null) { logLines.append(contents); } // Get the E3-facility syslog files contents = getCollectedLogLinesFromInstance(ipAddress, LogFileSource.SYSLOG, numLines); if (contents != null) { logLines.append(contents); } } } return LoggingResponseBuilder.logCollectionToXml(logLines.toString()); } /** * Returns the specified number of most-recent log lines from previously- * collected log files for a particular instance and log source. * * @param ipAddress The ip-address of the instance * @param logSource The source for the logs (JAVA, SMX, SYSLOG) * @param numLines The number of lines to retrieve. Fewer lines may be returned if the * requested number is not available. * @return Log lines in XML structure, with a top-level <code><log></code> node. */ public String getCollectedLogLinesFromInstance(String ipAddress, LogFileSource logSource, int numLines) { StringBuilder logLines = null; String logFilePath = null; int lineCount = 0; File collectionDir = new File(COLLECTION_PATH, LogCollector.ipToCollectionDirectory(ipAddress)); if (!collectionDir.exists() || !collectionDir.isDirectory()) { logger.warn("No log collection directory for ipAddress: {}", ipAddress); return null; } File logSourceSubdir = new File(collectionDir, logSource.toString()); if (!logSourceSubdir.exists() || !logSourceSubdir.isDirectory()) { logger.warn("No log-type '{}' collection subdirectory for ipAddress: {}", logSource.toString(), ipAddress); return null; } // Get the list of collected log files in date order File[] files = logSourceSubdir.listFiles(new LogFileFilter(null)); // get all log files, regardless of basename and ext Arrays.sort(files, new Comparator<Object>() { public int compare(Object o1, Object o2) { // Sort by decreasing date first, and then decreasing alphabetical File f1 = (File) o1; File f2 = (File) o2; int result = (Long.valueOf(f2.lastModified())).compareTo(Long.valueOf(f1.lastModified())); if (result == 0) { result = f2.getName().compareTo(f1.getName()); } return result; } }); logger.trace("Sorted log-files:"); for (File logFile : files) { logger.trace("{}", logFile.getName()); } logLines = new StringBuilder(); try { for (File file : files) { String fileName = file.getName(); if (logger.isDebugEnabled()) { logger.debug("Consider file: {}", fileName); } logFilePath = file.getAbsolutePath(); if (logger.isDebugEnabled()) { logger.debug("Retrieving {} log lines from file {}", String.valueOf(numLines - lineCount), logFilePath); } String logContent = LogCollector.getTailOfFile(file, numLines - lineCount); logLines.insert(0, logContent); int retrievedCount = lineCount(logContent); if (logger.isDebugEnabled()) { logger.debug("Actually got {} lines", String.valueOf(retrievedCount)); } lineCount += retrievedCount; if (lineCount >= numLines) { break; } } } catch (IOException ex) { // Swallow exception from any one file read and hope to get lines from the next log logger.warn("Couldn't read from log file {}", logFilePath == null ? "(null)" : logFilePath); } if (logger.isDebugEnabled()) { logger.debug("Got {} of {} requested lines", String.valueOf(lineCount), String.valueOf(numLines)); } return LoggingResponseBuilder.logLinesToXml(logSource, ipAddress, StringEscapeUtils.escapeXml(logLines.toString())); } /** * Collects logs from the localhost for all log sources. * * @param instanceCollectionDir Target directory to place collected logs (subdirectories for * each log source will be created) * @return The number of bytes in all files collected * </ul> * @throws IOException */ private long collectAllLocalLogs(File instanceCollectionDir) throws IOException { // First, get the E3Appender (java) logs // Parse the log-config file to find path to logs long bytesCollected = 0L; String logFilePath = LoggingUtil.getLocalLogFilePath(LogFileSource.JAVA); if ((logFilePath == null) || (logFilePath.length() == 0)) { // If we can't determine the log-file path from the config file, // look anyway in the usual servicemix log directory for any log files with the default name logger.warn( "Localhost is not using E3Appender, using default log-file path (check log-config file: {})", LoggingUtil.defaultConfigPath); logFilePath = LoggingUtil.defaultLogPath; } if (logger.isDebugEnabled()) { logger.debug("java log file path {}", logFilePath); } File logFile = new File(logFilePath); bytesCollected += collectLocalLogs(logFile.getParentFile(), instanceCollectionDir, logFile.getName(), LogFileSource.JAVA); // Next, get the serviceMix logs logFilePath = LoggingUtil.getLocalLogFilePath(LogFileSource.SMX); if ((logFilePath == null) || (logFilePath.length() == 0)) { // Same situation as above logger.warn("Localhost log-config file ({}) does not specify servicemix log location, using default", LoggingUtil.defaultConfigPath); logFilePath = LoggingUtil.defaultSMXLogPath; } if (logger.isDebugEnabled()) { logger.debug("smx log file path {}", logFilePath); } File smxLogFile = new File(logFilePath); bytesCollected += collectLocalLogs(smxLogFile.getParentFile(), instanceCollectionDir, smxLogFile.getName(), LogFileSource.SMX); // Collect the E3-specific syslog files logFilePath = NonJavaLogger.getLogFilePath(); if ((logFilePath == null) || (logFilePath.length() == 0)) { // Same situation as above logFilePath = NonJavaLogger.defaultLogFilePath; logger.warn("Localhost syslog-config file ({}) does not specify log location, using default", logFilePath); } if (logger.isDebugEnabled()) { logger.debug("syslog file path: {}", logFilePath); } File syslogFile = new File(logFilePath); bytesCollected += collectLocalLogs(syslogFile.getParentFile(), instanceCollectionDir, syslogFile.getName(), LogFileSource.SYSLOG); return bytesCollected; } /** * Collects logs from the specified instance for all log sources. * * @param logSource The topology instance to collect logs from * @param instanceCollectionDir Target directory to place collected logs (subdirectories for * each log source will be created) * @return The number of bytes in all files collected * @throws JSchException, IOException */ private long collectAllRemoteLogs(Instance logSource, Instance logDestination, File instanceCollectionDir) throws JSchException, IOException { if (logSource == null) { throw new NullPointerException("The instance than contains remote logs cannot be null"); } if (logDestination == null) { throw new NullPointerException("The instance where to copy logs cannot be null"); } // First, try to open a new SSH session to instance long bytesCollected = 0L; String ipAddress = logSource.getInternalIP(); if (logger.isDebugEnabled()) { logger.debug("trying to connect to {} via ssh ...", ipAddress); } SSHCommand sshCommand = new SSHCommand(); sshCommand.connect(logDestination.getSSHKey(), ipAddress, 22, logSource.getUser(), logSource.getPassword(), sshSessionTimeout); // Start with E3Appender Java logs first // Get a local copy of the logging config file to determine log-file path String remoteLogPath = null; File localConfigFile = new File(instanceCollectionDir, "java-logging.cfg"); String localConfigFilePath = localConfigFile.getAbsolutePath(); if (copyRemoteConfigFile(sshCommand, localConfigFilePath, LogFileSource.JAVA)) { remoteLogPath = LoggingUtil.getLogFilePathFromConfigFile(localConfigFilePath, LogFileSource.JAVA, false); } else { logger.warn("Couldn't retrieve E3 Java logging config file from host {}, will try default path", ipAddress); } if ((remoteLogPath == null) || (remoteLogPath.length() == 0)) { // If we can't find a logging config file with an E3Appender section, // look anyway in the usual servicemix log directory for any log files with the default name logger.warn("Instance at {} is not using E3Appender (check log-config file: {})", ipAddress, LoggingUtil.defaultConfigPath); remoteLogPath = LoggingUtil.defaultLogPath; } File localTargetDir = createLocalLogTargetDir(instanceCollectionDir, LogFileSource.JAVA); if (localTargetDir == null) { logger.warn("Couldn't create log-collection directory: {}", instanceCollectionDir + File.separator + LogFileSource.JAVA.toString()); } else { File remoteLog = new File(remoteLogPath); List<FileInfo> logList = getMatchingRemoteFileList(sshCommand, remoteLog.getParent(), remoteLog.getName()); for (FileInfo remoteFileInfo : logList) { File localCopy = new File(localTargetDir, targetNameForLogFile(remoteFileInfo, localTargetDir)); try { bytesCollected += copyRemoteFileWithWorkingTemp(sshCommand, remoteFileInfo, localCopy.getAbsolutePath(), deleteAfterCollect); } catch (Exception ex) { // Continue copy attempts if we experience an error logger.warn("Failed to copy remote file: {} ({})", remoteFileInfo.filePath, ex.getLocalizedMessage()); } } } // Now try to get the remote serviceMix log files // We use the same log-config file as the java logs to parse the path remoteLogPath = LoggingUtil.getLogFilePathFromConfigFile(localConfigFilePath, LogFileSource.SMX, false); localConfigFile.delete(); // no longer needed if ((remoteLogPath == null) || (remoteLogPath.length() == 0)) { // If we can't find a logging config file with the proper appender section, // look anyway in the usual servicemix log directory for any log files with the default name logger.warn( "Instance at {} is not using expected appender for servicemix rootLogger (check log-config file: {})", ipAddress, LoggingUtil.defaultConfigPath); remoteLogPath = LoggingUtil.defaultSMXLogPath; } localTargetDir = createLocalLogTargetDir(instanceCollectionDir, LogFileSource.SMX); if (localTargetDir == null) { logger.warn("Couldn't create log-collection directory: {}", instanceCollectionDir + File.separator + LogFileSource.SMX.toString()); } else { File remoteLog = new File(remoteLogPath); List<FileInfo> logList = getMatchingRemoteFileList(sshCommand, remoteLog.getParent(), remoteLog.getName()); for (FileInfo remoteFileInfo : logList) { File localCopy = new File(localTargetDir, targetNameForLogFile(remoteFileInfo, localTargetDir)); try { bytesCollected += copyRemoteFileWithWorkingTemp(sshCommand, remoteFileInfo, localCopy.getAbsolutePath(), deleteAfterCollect); } catch (Exception ex) { // Continue copy attempts if we experience an error logger.warn("Failed to copy remote file: {} ({})", remoteFileInfo.filePath, ex.getLocalizedMessage()); } } } // Collect the E3-specific syslog files // For syslog we parse the rsyslog config file for the log-file path localConfigFile = new File(instanceCollectionDir, "syslog.cfg"); localConfigFilePath = localConfigFile.getAbsolutePath(); if (copyRemoteConfigFile(sshCommand, localConfigFilePath, LogFileSource.SYSLOG)) { remoteLogPath = NonJavaLogger.getLogFilePathFromConfigFile(localConfigFilePath); localConfigFile.delete(); } else { logger.warn("Couldn't retrieve E3 syslog config file from host {}", ipAddress); } if ((remoteLogPath == null) || (remoteLogPath.length() == 0)) { // Try default path remoteLogPath = NonJavaLogger.defaultLogFilePath; logger.warn("Instance at {} does not specify an E3-specific syslog file, trying default: {}", ipAddress, remoteLogPath); } localTargetDir = createLocalLogTargetDir(instanceCollectionDir, LogFileSource.SYSLOG); if (localTargetDir == null) { logger.warn("Couldn't create log-collection directory: {}", instanceCollectionDir + File.separator + LogFileSource.SYSLOG.toString()); } else { File remoteLog = new File(remoteLogPath); List<FileInfo> logList = getMatchingRemoteFileList(sshCommand, remoteLog.getParent(), remoteLog.getName()); for (FileInfo remoteFileInfo : logList) { File localCopy = new File(localTargetDir, targetNameForLogFile(remoteFileInfo, localTargetDir)); try { bytesCollected += copyRemoteFileWithWorkingTemp(sshCommand, remoteFileInfo, localCopy.getAbsolutePath(), deleteAfterCollect); } catch (Exception ex) { // Continue copy attempts if we experience an error logger.warn("Failed to copy remote file: {} ({})", remoteFileInfo.filePath, ex.getLocalizedMessage()); } } } // We're done - disconnect and return number of bytes copied sshCommand.disconnect(); if (logger.isDebugEnabled()) { logger.debug("connected/disconnected!"); } return bytesCollected; } /** * Visit a particular local directory and collect all the log files that start with * a particular base filename, copying them to the specified target directory. * * @param sourceDir The directory in which the log files are located * @param targetDir The directory to put the copied files * @param baseName The basename of the log files to collect (such as "e3.log", "servicemix.log", etc) * @param logSource The type of logs to collect (JAVA, SMX, SYSLOG); the string form of the type will * be used as a destination subdirectory under targetDir * @return The number of bytes in all files collected */ private long collectLocalLogs(File sourceDir, File targetDir, final String baseName, LogFileSource logSource) { long bytesCollected = 0L; logger.debug("Collecting logs from localhost from {} with base {}", sourceDir, baseName); // New: Get all log files with a matching basename, regardless of rotation type File[] logFiles = sourceDir.listFiles(new LogFileFilter(baseName)); if (logFiles == null) { logger.warn("Error retrieving file list from {} matching name {}", sourceDir, baseName); return 0L; } // Make or use a specific subdirectory for this log type targetDir = new File(targetDir, logSource.toString()); if (!targetDir.exists()) { targetDir.mkdirs(); } else if (!targetDir.isDirectory()) { logger.error("Target for local log collection is not a directory: {}", targetDir.getAbsolutePath()); return 0L; } File logFile; for (File log : logFiles) { logFile = log; if (logger.isDebugEnabled()) { logger.debug("Copying log file {} ...", logFile.getAbsolutePath()); } String destFileName = targetNameForLogFile(logFile, targetDir); File destFile = new File(targetDir, destFileName); try { bytesCollected += LogCollector.copyLocalFileWithWorkingTemp(logFile, destFile, deleteAfterCollect); } catch (IOException ex) { // Continue copying despite single-file error logger.error("Could not copy file {}: {}", logFile.getAbsolutePath() + " to " + targetDir.getAbsolutePath(), ex.getLocalizedMessage()); } } return bytesCollected; } /** * Collects logs from the localhost for all log sources. * * @param numLines The requested number of log lines to retrieve. * Fewer lines may be returned if sufficient log entries are not available. * @return An XML-structured string with a <code><log></code> node for * each log-file found.. * @throws IOException */ private String getTailOfLocalActiveLogs(int numLines) throws IOException { // First, get the E3Appender (java) log // Parse the log-config file to find path to log file // Note: use TRACE-level logging here since our output may appear in retrieved log lines StringBuilder sb = new StringBuilder(); String logFilePath = LoggingUtil.getLocalLogFilePath(LogFileSource.JAVA); if ((logFilePath == null) || (logFilePath.length() == 0)) { // If we can't determine the log-file path from the config file, // look anyway in the usual servicemix log directory for any log files with the default name logger.warn( "Localhost is not using E3Appender, using default log-file path (check log-config file: {})", LoggingUtil.defaultConfigPath); logFilePath = LoggingUtil.defaultLogPath; } logger.trace("java log file path {}", logFilePath); File logFile = new File(logFilePath); String logLines = execTailOnFile(logFile, numLines); if (logLines != null) { sb.append(LoggingResponseBuilder.logLinesToXml(LogFileSource.JAVA, E3Constant.localhost, StringEscapeUtils.escapeXml(logLines.toString()))); } // Next, get the serviceMix log logFilePath = LoggingUtil.getLocalLogFilePath(LogFileSource.SMX); if ((logFilePath == null) || (logFilePath.length() == 0)) { // Same situation as above logger.warn("Localhost log-config file ({}) does not specify servicemix log location, using default", LoggingUtil.defaultConfigPath); logFilePath = LoggingUtil.defaultSMXLogPath; } logger.trace("local smx log file path {}", logFilePath); File smxLogFile = new File(logFilePath); logLines = execTailOnFile(smxLogFile, numLines); if (logLines != null) { sb.append(LoggingResponseBuilder.logLinesToXml(LogFileSource.SMX, E3Constant.localhost, StringEscapeUtils.escapeXml(logLines.toString()))); } // Collect the E3-specific syslog files logFilePath = NonJavaLogger.getLogFilePath(); if ((logFilePath == null) || (logFilePath.length() == 0)) { // Same situation as above logFilePath = NonJavaLogger.defaultLogFilePath; logger.warn( "Localhost syslog-config file does not specify an E3-specific log location, using default: {}", LoggingUtil.defaultLogPath); } logger.trace("local syslog file path: {}", logFilePath); logFile = new File(logFilePath); logLines = execTailOnFile(logFile, numLines); if (logLines != null) { sb.append(LoggingResponseBuilder.logLinesToXml(LogFileSource.SYSLOG, E3Constant.localhost, StringEscapeUtils.escapeXml(logLines.toString()))); } return sb.toString(); } /** * Retrieves the last <code>numLines</code> lines from the remote machine's active logs. * * @param numLines The requested number of log lines to retrieve. * Fewer lines may be returned if sufficient log entries are not available. * @return An XML-structured string with a <code><log></code> node for * each log-file found.. * @throws JSchException, IOException */ private String getTailOfRemoteActiveLogs(Instance logSource, Instance logDestination, int numLines) throws JSchException, IOException { if (logSource == null) { throw new NullPointerException("Log source cannot be null"); } if (logDestination == null) { throw new NullPointerException("Log destination cannot be null"); } StringBuilder sb = new StringBuilder(); // First, try to open a new SSH session to instance // Note: use TRACE-level logging here since our output may appear in retrieved log lines String ipAddress = logSource.getInternalIP(); logger.trace("trying to connect to {} via ssh ...", ipAddress); SSHCommand sshCommand = new SSHCommand(); sshCommand.connect(logDestination.getSSHKey(), ipAddress, 22, logSource.getUser(), logSource.getPassword(), sshSessionTimeout); // Start with E3Appender Java log first // Get a local copy of the logging config file to determine log-file path String remoteLogPath = null; File localConfigFile = File.createTempFile("java-logging", ".cfg"); boolean gotConfigFile = false; String localConfigFilePath = localConfigFile.getAbsolutePath(); if (copyRemoteConfigFile(sshCommand, localConfigFilePath, LogFileSource.JAVA)) { gotConfigFile = true; remoteLogPath = LoggingUtil.getLogFilePathFromConfigFile(localConfigFilePath, LogFileSource.JAVA, false); } else { logger.warn("Couldn't retrieve E3 Java logging config file from host {}, will try default path", ipAddress); } if ((remoteLogPath == null) || (remoteLogPath.length() == 0)) { // If we can't find a logging config file with an E3Appender section, // look anyway in the usual servicemix log directory for any log files with the default name logger.warn("Instance at {} is not using E3Appender (check log-config file: {})", ipAddress, LoggingUtil.defaultConfigPath); remoteLogPath = LoggingUtil.defaultLogPath; } logger.trace("java log file path for instance {}: {}", ipAddress, remoteLogPath); String logLines = execTailOnRemoteFile(sshCommand, remoteLogPath, numLines); if (logLines != null) { sb.append(LoggingResponseBuilder.logLinesToXml(LogFileSource.JAVA, ipAddress, StringEscapeUtils.escapeXml(logLines.toString()))); } // Next, get the serviceMix log // We use the same logging config file to get the servicemix log-file path remoteLogPath = null; if (gotConfigFile) { remoteLogPath = LoggingUtil.getLogFilePathFromConfigFile(localConfigFilePath, LogFileSource.SMX, false); localConfigFile.delete(); // We're done with the local version of the java/smx config file } if ((remoteLogPath == null) || (remoteLogPath.length() == 0)) { // Same situation as above logger.warn( "Could not parse servicemix log-file path from config file for instance at {}, using default", ipAddress); remoteLogPath = LoggingUtil.defaultSMXLogPath; } logger.trace("smx log file path for instance {}: {}", ipAddress, remoteLogPath); logLines = execTailOnRemoteFile(sshCommand, remoteLogPath, numLines); if (logLines != null) { sb.append(LoggingResponseBuilder.logLinesToXml(LogFileSource.SMX, ipAddress, StringEscapeUtils.escapeXml(logLines.toString()))); } // Collect the E3-specific syslog files // For syslog we parse the rsyslog config file for the log-file path remoteLogPath = null; localConfigFile = File.createTempFile("syslog", ".cfg"); localConfigFilePath = localConfigFile.getAbsolutePath(); if (copyRemoteConfigFile(sshCommand, localConfigFilePath, LogFileSource.SYSLOG)) { remoteLogPath = NonJavaLogger.getLogFilePathFromConfigFile(localConfigFilePath); localConfigFile.delete(); } else { logger.warn("Couldn't retrieve E3 syslog config file from host {}", ipAddress); } if ((remoteLogPath == null) || (remoteLogPath.length() == 0)) { // Same situation as above logger.warn( "Could not parse E3-specific log-file path from syslog config file for instance at {}, using default", ipAddress); remoteLogPath = NonJavaLogger.defaultLogFilePath; } logger.trace("syslog file path: {}", remoteLogPath); logLines = execTailOnRemoteFile(sshCommand, remoteLogPath, numLines); if (logLines != null) { sb.append(LoggingResponseBuilder.logLinesToXml(LogFileSource.SYSLOG, ipAddress, StringEscapeUtils.escapeXml(logLines.toString()))); } return sb.toString(); } // Code based on Stack Overflow suggestion: // http://stackoverflow.com/questions/686231/java-quickly-read-the-last-line-of-a-text-file // Pretty basic: byte-based (no Unicode) and relies on Unix-style EOL 0xA private static String getTailOfFile(File file, int numLines) throws FileNotFoundException, IOException { if (numLines < 0) { return null; } else if (numLines == 0) { return ""; } java.io.RandomAccessFile raFile = new java.io.RandomAccessFile(file, "r"); long fileLength = file.length() - 1; StringBuilder sb = new StringBuilder(); int line = 0; for (long filePointer = fileLength; filePointer >= 0; filePointer--) { raFile.seek(filePointer); int readByte = raFile.readByte(); if (readByte == 0xA) { if (filePointer < fileLength) { line = line + 1; if (line >= numLines) { break; } } } sb.append((char) readByte); } String lastLines = sb.reverse().toString(); return lastLines; } private static String execTailOnFile(File file, int numLines) throws IOException { String numLinesArg = String.valueOf(numLines); if ((numLines < 0) || (numLinesArg == null) || (numLinesArg.length() == 0) || (file == null) || !file.exists()) { return null; } else if (numLines == 0) { return ""; } ProcessBuilder processBuilder = new ProcessBuilder("/usr/bin/tail", "-n " + numLinesArg, file.getAbsolutePath()); File workingDirectory = file.getParentFile(); if (workingDirectory != null) { processBuilder.directory(workingDirectory); } Process p = processBuilder.start(); // Get tail's output: its InputStream InputStream is = p.getInputStream(); InputStreamReader isr = new InputStreamReader(is); BufferedReader reader = new BufferedReader(isr); StringBuilder sb = new StringBuilder(); String line; if (reader != null) { try { while ((line = reader.readLine()) != null) { sb.append(line).append("\n"); } } finally { try { if (reader != null) { reader.close(); } if (is != null) { is.close(); } } catch (IOException ioe) { // Nothing to do on close exception } } } /* // We only need to wait for p to finish if we want the exit value try { p.waitFor(); } catch (InterruptedException ex) { logger.warn("Tail on logfile {} interrupted!", file.getAbsoluteFile()); } logger.debug("Tail process exited with code {} ", String.valueOf(p.exitValue())); */ return sb.toString(); } /** * Count the number of newline characters in a string. * * @param logLines The string to parse (assumed to be log lines) * @return The number of newlines found */ private static int lineCount(String logLines) { return logLines.split(System.getProperty("line.separator")).length; } /** * Traverse the specified topology and return a list of instances. * Included instance types: * <ul> * <li>E3Gateway * <li>E3GatewayA * <li>E3Manager * <li>E3ManagerA * </ul> * * @param t The topology to traverse * @return A List of instances found in the topology */ private List<Instance> getInstanceList(ITopology t) { List<Instance> instances = new LinkedList<Instance>(); if (t == null) { logger.warn("topology is null when trying to retrieve instances!"); } else { instances.addAll(t.getInstancesByType("E3Gateway")); instances.addAll(t.getInstancesByType("E3Manager")); instances.addAll(t.getInstancesByType("E3GatewayA")); instances.addAll(t.getInstancesByType("E3ManagerA")); // add other types? } return instances; } /* * Local file operations */ /** * Copy a file from one location to another on the localhost, first moving * (renaming) the source file out of the way of any rotator process, and * then optionally deleting the source file after a successful copy. * Will attempt to replicate the modification time from the original file. * * @param sourceFile File to copy * @param destFile Destination file * @param deleteSource If <code>true</code>, will delete original after copy * @return The number of bytes copied * @throws IOException */ public static long copyLocalFile(File sourceFile, File destFile, boolean deleteSource) throws IOException { long bytesCopied = 0L; if ((sourceFile == null) || (destFile == null)) { throw new NullPointerException( "Source or destination file is null (source: " + sourceFile + ", dest: " + destFile + ")"); } if (!destFile.exists()) { destFile.createNewFile(); } String origSourcePath = sourceFile.getPath(); File tempFile = new File(tempNameForSourceFile(sourceFile.getPath())); FileChannel source = null; FileChannel destination = null; IOException cleanupException = null; boolean success = false; // Copy and validate result try { // Rename source file to temporary name before copying if (logger.isDebugEnabled()) { logger.debug("Renaming local file to: {}", tempFile.getPath()); } if (!sourceFile.renameTo(tempFile)) { logger.error("Could not move file to new name: {}", tempFile.getAbsolutePath()); } else { source = new FileInputStream(tempFile).getChannel(); destination = new FileOutputStream(destFile).getChannel(); bytesCopied = destination.transferFrom(source, 0, source.size()); copyModificationTime(tempFile, destFile); // Check integrity of copy success = validateFileCopy(tempFile, destFile); if (!success) { logger.warn("Copy of file {} did not pass integrity check!", origSourcePath); } } } catch (IOException ex) { // If there's been an error copying the file, we may be left with a zero-length or incomplete file if (!success) { if (logger.isDebugEnabled()) { logger.debug("Deleting failed copy of local file: {}", destFile.getAbsolutePath()); } destFile.delete(); } } finally { // Use a try-block during cleanup, but only throw exception if the // main file-copy try-block doesn't try { if (source != null) { source.close(); } if (destination != null) { destination.close(); } if (deleteSource && success) { if (logger.isDebugEnabled()) { logger.debug("Deleting local source file: {}", tempFile.getAbsolutePath()); } tempFile.delete(); } else { // Move source file back from temp name if (tempFile == null || !tempFile.renameTo(new File(origSourcePath))) { logger.error("Could not restore original filename: {}", origSourcePath); } } } catch (IOException ex) { logger.warn("IOException during local file-copy cleanup: {}", ex); cleanupException = new IOException(ex); } } if (cleanupException != null) { throw cleanupException; } return bytesCopied; } /** * Copy the modification time from one local file to another. * * @param sourceFile The file to copy the mod time from * @param destFile The file to copy the mod time to * @throws IOException */ public static void copyModificationTime(File sourceFile, File destFile) throws IOException { destFile.setLastModified(sourceFile.lastModified()); } private static boolean validateFileCopy(File sourceFile, File destFile) throws IOException { boolean success; long sourceLength = sourceFile.length(); long destLength = destFile.length(); if (sourceLength != destLength) { logger.error("File-size difference after copy of file {}: {}", sourceFile.getAbsolutePath(), "orig size: " + String.valueOf(sourceLength) + ", copy size: " + String.valueOf(destLength)); success = false; } else { // Check if last lines of files are the same String sourceTail = execTailOnFile(sourceFile, 1); String destTail = execTailOnFile(destFile, 1); success = sourceTail != null && sourceTail.equals(destTail); if (!success) { logger.error("Last lines of copied files are different: '{}' vs '{}'", sourceTail, destTail); } } return success; } /** * Make a copy of a local file, but write first to a "working" temporary file and then * rename the temporary file to the destination name. * * @param sourceFile The file to copy * @param destFile The final destination of the copy * @param deleteSource If <code>true</code>, will delete original after copy * @return The number of bytes copied * @throws IOException */ public static long copyLocalFileWithWorkingTemp(File sourceFile, File destFile, boolean deleteSource) throws IOException { File tempLocalFile = new File(tempNameForWorkingFile(destFile.getPath())); long bytesCopied = copyLocalFile(sourceFile, tempLocalFile, deleteSource); tempLocalFile.renameTo(destFile); return bytesCopied; } /* * Remote file operations */ /** * Copies the appropriate logging-config file for the logSource from the ssh remote host to * the path specified by destFilePath. May search remote host at multiple locations for * config file, or use a default path. * * @param sshCommand An connected SSHCommand session * @param destFilePath The target path for the copy of the config file * @param logSource The type of log-config file to look for * @return <ul> * <li><code>true</code> if an appropriate log-config file was found and copied * <li><code>false</code> otherwise * </ul> */ private static boolean copyRemoteConfigFile(SSHCommand sshCommand, String destFilePath, LogFileSource logSource) { boolean success = false; if (logSource.equals(LogFileSource.JAVA)) { try { // We assume there is only one location for the java-logging config file String remoteConfigPath = LoggingUtil.defaultConfigPath; success = (sshCommand.copyFrom(remoteConfigPath, destFilePath) > 0); } catch (Exception ex) { // swallow exception since failure means we try default paths, etc } } else if (logSource.equals(LogFileSource.SYSLOG)) { // There are a couple of pre-defined alternatives for syslog config file paths // And NonJavaLogger.getConfigFilePath() could return a non-default value if setConfigFilePath() has been used String[] altPaths = NonJavaLogger.getAltConfigFilePaths(); List<String> pathCandidates = new LinkedList<String>(); pathCandidates.add(NonJavaLogger.getConfigFilePath()); pathCandidates.addAll(Arrays.asList(altPaths)); //boolean foundConfigFile = false; for (String pathCand : pathCandidates) { String remoteConfigPath = pathCand; try { if (logger.isDebugEnabled()) { logger.debug("Trying syslog config path: {}", pathCand); } if (sshCommand.copyFrom(remoteConfigPath, destFilePath) > 0) { success = true; break; } } catch (Exception ex) { if (logger.isDebugEnabled()) { logger.debug("Failed on path {}", remoteConfigPath); } } } } else { logger.warn("Invalid log-file type passed to getRemoteLogPath: {}", logSource.toString()); } return success; } /** * Creates a local directory to hold collected log files, specific to the log source (JAVA, SMX, SYSLOG) * * @param instanceCollectionDir The parent directory for the new directory * @param logSource The log-file source (determines name of created directory) * @return A File object representing the new directory, <code>null</code> on error */ private static File createLocalLogTargetDir(File instanceCollectionDir, LogFileSource logSource) { File logTargetDir = new File(instanceCollectionDir, logSource.toString()); if (logTargetDir.exists()) { if (!logTargetDir.isDirectory()) { logger.error("Log collection target path is not a directory: {}", logTargetDir.getAbsolutePath()); return null; } } else if (!logTargetDir.mkdirs()) { logger.error("Could not create log collection target directory: {}", logTargetDir.getAbsolutePath()); } return logTargetDir; } /** * Searches the contents of the specified remote directory, and returns a list of FileInfo * instances that represent remote files that match the specified basename. * * @param sshCommand An active (connected) SSHCommand * @param remoteDirPath The path on the remote machine to search for matching files * @param baseName A base filename to compare against each remote filename (no path) to determine a match * @return A List of matching files in FileInfo form, empty if no matches were found * @throws JSchException * @throws IOException */ private static List<FileInfo> getMatchingRemoteFileList(SSHCommand sshCommand, String remoteDirPath, String baseName) throws JSchException, IOException { if ((sshCommand == null) || !sshCommand.isConnected()) { throw new JSchException("Not connected with a valid SSH session"); } String remoteUsername = null; if (LogCollector.collectOnlyUserOwnedLogs) { remoteUsername = sshCommand.getSessionUsername(); if (remoteUsername == null) { throw new JSchException("Username for SSH session is not valid"); } } // First get a list of *all* files in the remote directory, and then // parse the results for a match with the target basename // (Prefer to do the matching work locally rather than remotely) List<FileInfo> matches = new LinkedList<FileInfo>(); String findCmd; if (LogCollector.collectOnlyUserOwnedLogs) { findCmd = "find " + remoteDirPath + " -maxdepth 1 -user " + remoteUsername + " -perm -664 -type f"; } else { findCmd = "find " + remoteDirPath + " -maxdepth 1 -type f"; } findCmd = findCmd + " -printf '%T@\t%s\t%p\n' 2> /dev/null"; ShellCommandResult sshResult = sshCommand.execShellCommand(findCmd); String[] allFiles = sshResult.getResult().split("\n"); LogFileFilter baseFileFilter = new LogFileFilter(baseName); for (String remoteFileItem : allFiles) { String[] remoteFileItems = remoteFileItem.split("\t"); if (remoteFileItems.length != 3) { logger.warn("Got remote file entry, but not formatted as expected: {}", remoteFileItem); } else { String remoteFilePath = remoteFileItems[2]; File remoteLogFile = new File(remoteFilePath); //logger.debug("Consider {} vs {}", remoteLogFile.getName(), filenameRegex); if (baseFileFilter.accept(remoteLogFile)) { matches.add(new FileInfo(remoteFilePath, remoteFileItems[1], remoteFileItems[0])); //logger.debug("{} matches!", remoteLogFile.getName()); } else { logger.trace("No match between regex '{}' and '{}'", baseName, remoteLogFile.getName()); } } } return matches; } /** * Formats the filename for a copied log file to enable identification and sorting. * Currently transforms rolling-appender style (".1") file suffixes to * daily-rolling-appender style ("yyyy-MM-dd-HH-ss)". * * @param logFileInfo FileInfo structure for log file * @param localTargetDir The directory where this file will be placed * @return New filename (no path) for log file */ private static String targetNameForLogFile(FileInfo logFileInfo, File localTargetDir) { // Assume log files have either a rolling file extension (e.g. ".1") // or a daily-rolling file extension ("2012-05-01-14-00") // Convert log files with rolling extensions to dailyRolling for uniqueness File logFile = new File(logFileInfo.filePath); String localName = logFile.getName(); if (localName.matches(LoggingUtil.rollingFilePatternString)) { long modTime = (long) (Double.parseDouble(logFileInfo.fileModtime) * 1000.0); Date modDate = new Date(modTime); DateFormat format = new SimpleDateFormat(LoggingUtil.minRollingDatePattern); String formattedDate = format.format(modDate); localName = localName.replaceFirst(LoggingUtil.rollingFilePatternString, "$1." + formattedDate); // Uniquify in target directory // Don't uniquify if we don't delete source logs, since this will result in local duplicates if (uniquifyCopiedFilenames) { int id = 0; File targetFile = new File(localTargetDir, localName); boolean uniquified = false; while (targetFile.exists()) { if (++id > uniquifyLimit) { break; } uniquified = true; targetFile = new File(localTargetDir, localName + "." + String.valueOf(id)); } if (uniquified) { if (logger.isDebugEnabled()) { logger.debug("Uniquified filename {} to {}", logFile.getName(), targetFile.getName()); } } localName = targetFile.getName(); } if (logger.isDebugEnabled()) { logger.debug("Changed target filename {} to {}", logFile.getName(), localName); } } return localName; } /** * Formats the filename for a copied log file to enable identification and sorting. * Currently transforms rolling-appender style (".1") file suffixes to * daily-rolling-appender style ("yyyy-MM-dd-HH-ss)". * * @param logFile Log-file File object * @param localTargetDir The directory where this file will be placed * @return New filename (no path) for log file */ private static String targetNameForLogFile(File logFile, File localTargetDir) { // Assume log files have either a rolling file extension (e.g. ".1") // or a daily-rolling file extension ("2012-05-01-14-00") // Convert log files with rolling extensions to dailyRolling for uniqueness String localName = logFile.getName(); if (localName.matches(LoggingUtil.rollingFilePatternString)) { long modTime = logFile.lastModified(); Date modDate = new Date(modTime); DateFormat format = new SimpleDateFormat(LoggingUtil.minRollingDatePattern); String formattedDate = format.format(modDate); localName = localName.replaceFirst(LoggingUtil.rollingFilePatternString, "$1." + formattedDate); // Uniquify in target directory, requested // Don't uniquify if we don't delete the original (source) log file, // since we'll end up with duplicates if (uniquifyCopiedFilenames) { int id = 0; File targetFile = new File(localTargetDir, localName); while (targetFile.exists()) { if (++id > uniquifyLimit) { break; } targetFile = new File(localTargetDir, localName + "." + String.valueOf(id)); if (logger.isDebugEnabled()) { logger.debug("Uniquified filename {} to {}", logFile.getName(), targetFile.getName()); } } localName = targetFile.getName(); } if (logger.isDebugEnabled()) { logger.debug("Changed target filename {} to {}", logFile.getName(), localName); } } return localName; } /** * Copies a file on a remote host to a local path. Before copying, renames source file * with a temporary name to move it out of the way of any rotator process. * Optionally deletes the file from the remote host. * * @param sshCommand An active (connected) SSHCommand * @param remoteFileInfo A FileInfo structure representing the remote file * @param localFilePath The full path for the local file copy * @param deleteSource If <code>true</code> and the copied succeeds, the remote file will be deleted * @return The number of bytes copied. * @throws JSchException * @throws IOException */ private static long copyRemoteFile(SSHCommand sshCommand, FileInfo remoteFileInfo, String localFilePath, boolean deleteSource) throws JSchException, IOException { String remoteFilePath = remoteFileInfo.filePath; String tempRemotePath = tempNameForSourceFile(remoteFilePath); File localFile = null; long bytesCopied = 0L; IOException cleanupIOException = null; JSchException cleanupJSchException = null; boolean success = false; // Copy file and validate result try { // Rename the remote file with a temporary name before copying if (logger.isDebugEnabled()) { logger.debug("Renaming remote file to: {}", tempRemotePath); } sshCommand.execShellCommand("mv " + remoteFilePath + " " + tempRemotePath); // Copy the (renamed) remote file to local destination remoteFileInfo.filePath = tempRemotePath; // remote file name is needed during validation bytesCopied = sshCommand.copyFrom(tempRemotePath, localFilePath); localFile = new File(localFilePath); long modTime = (long) (Double.parseDouble(remoteFileInfo.fileModtime) * 1000.0); localFile.setLastModified(modTime); // Check integrity of copy success = validateRemoteFileCopy(sshCommand, remoteFileInfo, localFilePath); if (!success) { logger.warn("Copy of file {} did not pass integrity check!", remoteFilePath); } } catch (IOException ex) { // If there's been an error copying the file, we may be left with a zero-length or incomplete file if ((localFile != null) && localFile.exists() && !success) { if (logger.isDebugEnabled()) { logger.debug("Deleting failed local copy of remote file: {}", localFile.getAbsolutePath()); } localFile.delete(); } } finally { // Use a try-block during cleanup, but only throw exception if the // main file-copy try-block doesn't try { if (deleteSource && success) { if (logger.isDebugEnabled()) { logger.debug("Deleting remote file: {}", tempRemotePath); } sshCommand.execShellCommand("rm " + tempRemotePath); } else { // Move source file back from temporary name sshCommand.execShellCommand("mv " + tempRemotePath + " " + remoteFilePath); } } catch (JSchException ex) { logger.warn("JSchException during remote file copy cleanup: {}", ex); cleanupJSchException = new JSchException(ex.getMessage()); } catch (IOException ex) { logger.warn("IOException during remote file copy cleanup: {}", ex); cleanupIOException = new IOException(ex); } remoteFileInfo.filePath = remoteFilePath; // restore original file name in argument } if (cleanupJSchException != null) { throw cleanupJSchException; } else if (cleanupIOException != null) { throw cleanupIOException; } return bytesCopied; } private static boolean validateRemoteFileCopy(SSHCommand sshCommand, FileInfo remoteFileInfo, String localFilePath) throws JSchException, IOException { boolean success; File localFile = new File(localFilePath); String remoteFilePath = remoteFileInfo.filePath; long remoteLength = Long.parseLong(remoteFileInfo.fileSize); long localLength = localFile.length(); if (localLength != remoteLength) { logger.warn("File-size difference after copy of remote log file {}: {}", remoteFilePath, "remote size: " + String.valueOf(remoteLength) + ", local size: " + String.valueOf(localLength)); success = false; } else { // Check if last lines of files are the same String remoteTail = execTailOnRemoteFile(sshCommand, remoteFilePath, 1); String localTail = execTailOnFile(localFile, 1); success = remoteTail != null && remoteTail.equals(localTail); if (!success) { logger.error("Last lines of copied files are different: '{}' vs '{}'", localTail, remoteTail); } } return success; } /** * Copies a file on a remote host to a local path, using a temporary local file intermediary. Optionally deletes the file from the remote host. * * @param sshCommand An active (connected) SSHCommand * @param remoteFileInfo A FileInfo structure representing the remote file * @param localFilePath The full path for the local file copy * @param deleteSource If <code>true</code> and the copied succeeds, the remote file will be deleted * @return The number of bytes copied. * @throws JSchException * @throws IOException */ private static long copyRemoteFileWithWorkingTemp(SSHCommand sshCommand, FileInfo remoteFileInfo, String localFilePath, boolean deleteSource) throws JSchException, IOException { String tempLocalPath = tempNameForWorkingFile(localFilePath); long bytesCopied = copyRemoteFile(sshCommand, remoteFileInfo, tempLocalPath, deleteSource); (new File(tempLocalPath)).renameTo(new File(localFilePath)); return bytesCopied; } private static String execTailOnRemoteFile(SSHCommand sshCommand, String remoteFilePath, int numLines) throws JSchException, IOException { if ((sshCommand == null) || !sshCommand.isConnected()) { throw new JSchException("Not connected with a valid SSH session"); } String numLinesArg = String.valueOf(numLines); if ((numLines < 0) || (numLinesArg == null) || (numLinesArg.length() == 0) || (remoteFilePath == null) || (remoteFilePath.length() == 0)) { return null; } else if (numLines == 0) { return ""; } String tailCmd = "/usr/bin/tail -n " + numLinesArg + " " + remoteFilePath; ShellCommandResult sshResult = sshCommand.execShellCommand(tailCmd); if (logger.isDebugEnabled()) { logger.debug("Remote tail process exited with code {}", String.valueOf(sshResult.getExitStatus())); } String logLines = sshResult.getResult(); return logLines; } /** * Format an ip address string for use as a directory name. * * @param ipAddress The ip address in the usual IPv4 format * @return */ private static String ipToCollectionDirectory(String ipAddress) { return ipAddress.replace(".", "_"); } /** * Reverses the formatting performed by the <code>ipToCollectionDirectory</code> function. * * @param dirName The collection directory name * @return The name re-formatted as an IPv4 address */ private static String collectionDirectoryToIP(String dirName) { String ip = dirName.replaceAll("^(\\d{1,3})_(\\d{1,3})_(\\d{1,3})_(\\d{1,3})$", "$1\\.$2\\.$3\\.$4"); return ip; } private static String tempNameForWorkingFile(String fileNameOrPath) { return fileNameOrPath + workingFileSuffix; } private static String tempNameForSourceFile(String fileNameOrPath) { Date modDate = new Date(); DateFormat format = new SimpleDateFormat(LoggingUtil.minRollingDatePattern); String formattedDate = format.format(modDate); return fileNameOrPath + ".collected_" + formattedDate; } /** * Log an entire Instance structure. * * @param instance The Instance to send to the log */ public static void logInstance(Instance instance) { if (logger.isDebugEnabled()) { logger.debug("Got instance: {}", instance); } logger.trace("SSHKeyName: {}", instance.getSSHKeyName()); logger.trace("User: {}", instance.getUser()); logger.trace("Password: {}", instance.getPassword()); logger.trace("Port: {}", instance.getPort()); logger.trace("Area: {}", instance.getArea()); logger.trace("SSHKey: {}", instance.getSSHKey()); } /** * Gets a specified number of most-recent log lines from the localhost's * rotated logs. * * This method has been supplanted by the other log-collection methods in this class. * * @param numLines * @return * @throws IOException */ public static String getLocalLogLines(LogFileSource logSource, int numLines) throws IOException { if (!LogFileSource.JAVA.equals(logSource) && !LogFileSource.SMX.equals(logSource)) { logger.warn("Unexpected log-source value: {}", logSource == null ? "(null)" : logSource); return null; } StringBuilder logLines = null; String logFilePath = LoggingUtil.getLocalLogFilePath(logSource); int lineCount = 0; File logDir = (new File(logFilePath)).getParentFile(); if (!logDir.exists() || !logDir.isDirectory()) { return null; } // Try getting lines from the current log file(s) ... logLines = new StringBuilder(); int logGen = 1; try { while (lineCount < numLines) { File logFile = new File(logFilePath + "." + String.valueOf(logGen)); if (!logFile.exists()) { break; } String logContent = getTailOfFile(logFile, numLines - lineCount); logLines.insert(0, logContent); lineCount += lineCount(logContent); logGen++; } } catch (IOException ex) { logger.warn("Couldn't read from log file {}", logFilePath); } return LoggingResponseBuilder.logLinesToXml(LogFileSource.JAVA, E3Constant.localhost, logLines.toString()); } }