Java tutorial
/* * Autopsy Forensic Browser * * Copyright 2015 Basis Technology Corp. * Contact: carrier <at> sleuthkit <dot> org * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.sleuthkit.autopsy.experimental.configuration; import java.io.File; import java.io.IOException; import java.io.PrintWriter; import java.io.BufferedReader; import java.io.FileReader; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.UnsupportedEncodingException; import java.util.zip.CRC32; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.logging.Level; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.HashMap; import java.util.concurrent.TimeUnit; import java.util.prefs.BackingStoreException; import org.apache.commons.io.FileUtils; import org.sleuthkit.autopsy.core.UserPreferences; import org.sleuthkit.autopsy.coreutils.Logger; import org.sleuthkit.autopsy.coreutils.PlatformUtil; import org.sleuthkit.autopsy.coreutils.FileUtil; import org.sleuthkit.autopsy.ingest.IngestJobSettings; import org.sleuthkit.autopsy.keywordsearch.KeywordListsManager; import org.sleuthkit.autopsy.modules.hashdatabase.HashDbManager; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.autopsy.core.ServicesMonitor; import org.sleuthkit.autopsy.modules.hashdatabase.HashDbManager.HashDb; import org.sleuthkit.autopsy.experimental.configuration.AutoIngestSettingsPanel.UpdateConfigSwingWorker; import org.sleuthkit.autopsy.experimental.coordinationservice.CoordinationService; import org.sleuthkit.autopsy.experimental.coordinationservice.CoordinationService.Lock; import org.sleuthkit.autopsy.experimental.coordinationservice.CoordinationService.CoordinationServiceException; /* * A utility class for loading and saving shared configuration data */ public class SharedConfiguration { // Files private static final String AUTO_MODE_CONTEXT_FILE = "AutoModeContext.properties"; //NON-NLS private static final String USER_DEFINED_TYPE_DEFINITIONS_FILE = "UserFileTypeDefinitions.settings"; //NON-NLS private static final String USER_DEFINED_TYPE_DEFINITIONS_FILE_LEGACY = "UserFileTypeDefinitions.xml"; //NON-NLS private static final String INTERESTING_FILES_SET_DEFS_FILE = "InterestingFileSets.settings"; //NON-NLS private static final String INTERESTING_FILES_SET_DEFS_FILE_LEGACY = "InterestingFilesSetDefs.xml"; //NON-NLS private static final String KEYWORD_SEARCH_SETTINGS = "keywords.settings"; //NON-NLS private static final String KEYWORD_SEARCH_SETTINGS_LEGACY = "keywords.xml"; //NON-NLS private static final String KEYWORD_SEARCH_GENERAL_LEGACY = "KeywordSearch.properties"; //NON-NLS private static final String KEYWORD_SEARCH_NSRL_LEGACY = "KeywordSearch_NSRL.properties"; //NON-NLS private static final String KEYWORD_SEARCH_OPTIONS_LEGACY = "KeywordSearch_Options.properties"; //NON-NLS private static final String KEYWORD_SEARCH_SCRIPTS_LEGACY = "KeywordSearch_Scripts.properties"; //NON-NLS private static final String FILE_EXT_MISMATCH_SETTINGS = "mismatch_config.settings"; //NON-NLS private static final String FILE_EXT_MISMATCH_SETTINGS_LEGACY = "mismatch_config.xml"; //NON-NLS private static final String ANDROID_TRIAGE = "AndroidTriage_Options.properties"; //NON-NLS private static final String GENERAL_PROPERTIES = "core.properties"; //NON-NLS private static final String AUTO_INGEST_PROPERTIES = "AutoIngest.properties"; //NON-NLS private static final String HASHDB_CONFIG_FILE_NAME = "hashLookup.settings"; //NON-NLS private static final String HASHDB_CONFIG_FILE_NAME_LEGACY = "hashsets.xml"; //NON-NLS public static final String FILE_EXPORTER_SETTINGS_FILE = "fileexporter.settings"; //NON-NLS private static final String SHARED_CONFIG_VERSIONS = "SharedConfigVersions.txt"; //NON-NLS // Folders private static final String AUTO_MODE_FOLDER = "AutoModeContext"; //NON-NLS private static final String REMOTE_HASH_FOLDER = "hashDb"; //NON-NLS private static final String PREFERENCES_FOLDER = "Preferences"; //NON-NLS public static final String FILE_EXPORTER_FOLDER = "Automated File Exporter"; //NON-NLS private static final String LOCK_ROOT = "/autopsy"; // NON-NLS private static final String UPLOAD_IN_PROGRESS_FILE = "uploadInProgress"; // NON-NLS private static final String moduleDirPath = PlatformUtil.getUserConfigDirectory(); private static final Logger logger = Logger.getLogger(SharedConfiguration.class.getName()); private final UpdateConfigSwingWorker swingWorker; private AutoIngestUserPreferences.SelectedMode mode; private String sharedConfigFolder; private int fileIngestThreads; private boolean sharedConfigMaster; private boolean showToolsWarning; private boolean displayLocalTime; private boolean hideKnownFilesInDataSource; private boolean hideKnownFilesInViews; private boolean hideSlackFilesInDataSource; private boolean hideSlackFilesInViews; private boolean keepPreferredViewer; /** * Exception type thrown by shared configuration. */ public final static class SharedConfigurationException extends Exception { private static final long serialVersionUID = 1L; private SharedConfigurationException(String message) { super(message); } private SharedConfigurationException(String message, Throwable cause) { super(message, cause); } } // Using this so we can indicate whether a read/write failed because the lock file is present, // which we need to know to display the correct message in the auto-ingest dashboard about why // processing has been paused. public enum SharedConfigResult { SUCCESS, LOCKED } public SharedConfiguration() { swingWorker = null; } /** * Construct with a SwingWorker reference to allow status update messages to * go to the GUI * * @param worker */ public SharedConfiguration(UpdateConfigSwingWorker worker) { this.swingWorker = worker; } private void publishTask(String task) { if (swingWorker != null) { swingWorker.publishStatus(task); } } /** * Upload the current multi-user ingest settings to a shared folder. * * @throws SharedConfigurationException * @throws CoordinationServiceException * @throws InterruptedException */ public SharedConfigResult uploadConfiguration() throws SharedConfigurationException, CoordinationServiceException, InterruptedException { publishTask("Starting shared configuration upload"); File remoteFolder = getSharedFolder(); try (Lock writeLock = CoordinationService.getInstance(LOCK_ROOT).tryGetExclusiveLock( CoordinationService.CategoryNode.CONFIG, remoteFolder.getAbsolutePath(), 30, TimeUnit.MINUTES)) { if (writeLock == null) { logger.log(Level.INFO, String.format( "Failed to lock %s - another node is currently uploading or downloading configuration", remoteFolder.getAbsolutePath())); return SharedConfigResult.LOCKED; } // Make sure the local settings have been initialized if (!isConfigFolderPopulated(new File(moduleDirPath), false)) { logger.log(Level.INFO, "Local configuration has not been initialized."); throw new SharedConfigurationException( "Local configuration has not been initialized. Please verify/update and save the settings using the Ingest Module Settings button and then retry the upload."); } // Write a file to indicate that uploading is in progress. If we crash or // have an error, this file will remain in the shared folder. File uploadInProgress = new File(remoteFolder, UPLOAD_IN_PROGRESS_FILE); if (!uploadInProgress.exists()) { try { Files.createFile(uploadInProgress.toPath()); } catch (IOException ex) { throw new SharedConfigurationException( String.format("Failed to create %s", uploadInProgress.toPath()), ex); } } // Make sure all recent changes are saved to the preference file // Current testing suggests that we do not need to do this for the ingest settings // because there is a longer delay between setting them and copying the files. try { // Make sure all recent changes are saved to the preference file // Current testing suggests that we do not need to do this for the ingest settings // because there is a longer delay between setting them and copying the files. UserPreferences.saveToStorage(); } catch (BackingStoreException ex) { throw new SharedConfigurationException("Failed to save shared configuration settings", ex); } uploadAutoModeContextSettings(remoteFolder); uploadEnabledModulesSettings(remoteFolder); uploadFileTypeSettings(remoteFolder); uploadInterestingFilesSettings(remoteFolder); uploadKeywordSearchSettings(remoteFolder); uploadFileExtMismatchSettings(remoteFolder); uploadAndroidTriageSettings(remoteFolder); uploadMultiUserAndGeneralSettings(remoteFolder); uploadHashDbSettings(remoteFolder); uploadFileExporterSettings(remoteFolder); try { Files.deleteIfExists(uploadInProgress.toPath()); } catch (IOException ex) { throw new SharedConfigurationException( String.format("Failed to delete %s", uploadInProgress.toPath()), ex); } } return SharedConfigResult.SUCCESS; } /** * Download the multi-user settings from a shared folder. * * @throws SharedConfigurationException * @throws InterruptedException */ public synchronized SharedConfigResult downloadConfiguration() throws SharedConfigurationException, InterruptedException { publishTask("Starting shared configuration download"); // Save local settings that should not get overwritten saveNonSharedSettings(); File remoteFolder = getSharedFolder(); try (Lock readLock = CoordinationService.getInstance(LOCK_ROOT).tryGetSharedLock( CoordinationService.CategoryNode.CONFIG, remoteFolder.getAbsolutePath(), 30, TimeUnit.MINUTES)) { if (readLock == null) { return SharedConfigResult.LOCKED; } // Make sure the shared configuration was last uploaded successfully File uploadInProgress = new File(remoteFolder, UPLOAD_IN_PROGRESS_FILE); if (uploadInProgress.exists()) { logger.log(Level.INFO, String.format("Shared configuration folder %s is corrupt - re-upload configuration", remoteFolder.getAbsolutePath())); throw new SharedConfigurationException( String.format("Shared configuration folder %s is corrupt - re-upload configuration", remoteFolder.getAbsolutePath())); } // Make sure the shared configuration folder isn't empty if (!isConfigFolderPopulated(remoteFolder, true)) { logger.log(Level.INFO, String.format( "Shared configuration folder %s is missing files / may be empty. Aborting download.", remoteFolder.getAbsolutePath())); throw new SharedConfigurationException(String.format( "Shared configuration folder %s is missing files / may be empty. Aborting download.", remoteFolder.getAbsolutePath())); } try { /* Make sure all recent changes are saved to the preference file. This also releases open file handles to the preference files. If this is not done, then occasionally downloading of shared configuration fails silently, likely because Java/OS is still holding the file handle. The problem manifests itself by some of the old/original configuration files sticking around after shared configuration has seemingly been successfully updated. */ UserPreferences.saveToStorage(); } catch (BackingStoreException ex) { throw new SharedConfigurationException("Failed to save shared configuration settings", ex); } downloadAutoModeContextSettings(remoteFolder); downloadEnabledModuleSettings(remoteFolder); downloadFileTypeSettings(remoteFolder); downloadInterestingFilesSettings(remoteFolder); downloadKeywordSearchSettings(remoteFolder); downloadFileExtMismatchSettings(remoteFolder); downloadAndroidTriageSettings(remoteFolder); downloadFileExporterSettings(remoteFolder); // Download general settings, then restore the current // values for the unshared fields downloadMultiUserAndGeneralSettings(remoteFolder); try { UserPreferences.reloadFromStorage(); } catch (BackingStoreException ex) { throw new SharedConfigurationException("Failed to read shared configuration settings", ex); } restoreNonSharedSettings(); downloadHashDbSettings(remoteFolder); } catch (CoordinationServiceException ex) { throw new SharedConfigurationException(String.format( "Coordination service error acquiring exclusive lock on shared configuration source %s", remoteFolder.getAbsolutePath()), ex); } // Check Solr service if (!isServiceUp(ServicesMonitor.Service.REMOTE_KEYWORD_SEARCH.toString())) { throw new SharedConfigurationException("Keyword search service is down"); } // Check PostgreSQL service if (!isServiceUp(ServicesMonitor.Service.REMOTE_CASE_DATABASE.toString())) { throw new SharedConfigurationException("Case database server is down"); } // Check ActiveMQ service if (!isServiceUp(ServicesMonitor.Service.MESSAGING.toString())) { throw new SharedConfigurationException("Messaging service is down"); } // Check input folder permissions String inputFolder = AutoIngestUserPreferences.getAutoModeImageFolder(); if (!FileUtil.hasReadWriteAccess(Paths.get(inputFolder))) { throw new SharedConfigurationException("Cannot read input folder " + inputFolder + ". Check that the folder exists and that you have permissions to access it."); } // Check output folder permissions String outputFolder = AutoIngestUserPreferences.getAutoModeResultsFolder(); if (!FileUtil.hasReadWriteAccess(Paths.get(outputFolder))) { throw new SharedConfigurationException("Cannot read output folder " + outputFolder + ". Check that the folder exists and that you have permissions to access it."); } return SharedConfigResult.SUCCESS; } /** * Tests service of interest to verify that it is running. * * @param serviceName Name of the service. * * @return True if the service is running, false otherwise. */ private boolean isServiceUp(String serviceName) { try { return (ServicesMonitor.getInstance().getServiceStatus(serviceName) .equals(ServicesMonitor.ServiceStatus.UP.toString())); } catch (ServicesMonitor.ServicesMonitorException ex) { logger.log(Level.SEVERE, String.format("Problem checking service status for %s", serviceName), ex); return false; } } /** * Save any settings that should not be overwritten by the shared * configuration. */ private void saveNonSharedSettings() { sharedConfigMaster = AutoIngestUserPreferences.getSharedConfigMaster(); sharedConfigFolder = AutoIngestUserPreferences.getSharedConfigFolder(); showToolsWarning = AutoIngestUserPreferences.getShowToolsWarning(); displayLocalTime = UserPreferences.displayTimesInLocalTime(); hideKnownFilesInDataSource = UserPreferences.hideKnownFilesInDataSourcesTree(); hideKnownFilesInViews = UserPreferences.hideKnownFilesInViewsTree(); keepPreferredViewer = UserPreferences.keepPreferredContentViewer(); fileIngestThreads = UserPreferences.numberOfFileIngestThreads(); hideSlackFilesInDataSource = UserPreferences.hideSlackFilesInDataSourcesTree(); hideSlackFilesInViews = UserPreferences.hideSlackFilesInViewsTree(); } /** * Restore the settings that may have been overwritten. */ private void restoreNonSharedSettings() { AutoIngestUserPreferences.setSharedConfigFolder(sharedConfigFolder); AutoIngestUserPreferences.setSharedConfigMaster(sharedConfigMaster); AutoIngestUserPreferences.setShowToolsWarning(showToolsWarning); UserPreferences.setDisplayTimesInLocalTime(displayLocalTime); UserPreferences.setHideKnownFilesInDataSourcesTree(hideKnownFilesInDataSource); UserPreferences.setHideKnownFilesInViewsTree(hideKnownFilesInViews); UserPreferences.setKeepPreferredContentViewer(keepPreferredViewer); UserPreferences.setNumberOfFileIngestThreads(fileIngestThreads); UserPreferences.setHideSlackFilesInDataSourcesTree(hideSlackFilesInDataSource); UserPreferences.setHideSlackFilesInViewsTree(hideSlackFilesInViews); } /** * Get the base folder being used to store the shared config settings. * * @return The shared configuration folder * * @throws SharedConfigurationException */ private static File getSharedFolder() throws SharedConfigurationException { // Check that the shared folder is set and exists String remoteConfigFolderPath = AutoIngestUserPreferences.getSharedConfigFolder(); if (remoteConfigFolderPath.isEmpty()) { logger.log(Level.SEVERE, "Shared configuration folder is not set."); throw new SharedConfigurationException("Shared configuration folder is not set."); } File remoteFolder = new File(remoteConfigFolderPath); if (!remoteFolder.exists()) { logger.log(Level.SEVERE, "Shared configuration folder {0} does not exist", remoteConfigFolderPath); throw new SharedConfigurationException( "Shared configuration folder " + remoteConfigFolderPath + " does not exist"); } return remoteFolder; } /** * Do a basic check to determine whether settings have been stored to the * given folder. There may still be missing files/errors, but this will stop * empty/corrupt settings from overwriting local settings or being uploaded. * * Currently we check for: - A non-empty AutoModeContext folder - An * AutoModeContext properties file * * @param folder Folder to check the contents of * @param isSharedFolder True if the folder being tested is the shared * folder, false if its the local folder * * @return true if the folder appears to have been initialized, false * otherwise */ private static boolean isConfigFolderPopulated(File folder, boolean isSharedFolder) { if (!folder.exists()) { return false; } // Check that the context directory exists and is not empty File contextDir; if (isSharedFolder) { contextDir = new File(folder, AUTO_MODE_FOLDER); } else { IngestJobSettings ingestJobSettings = new IngestJobSettings( AutoIngestUserPreferences.getAutoModeIngestModuleContextString()); contextDir = ingestJobSettings.getSavedModuleSettingsFolder().toFile(); } if ((!contextDir.exists()) || (!contextDir.isDirectory())) { return false; } if (contextDir.listFiles().length == 0) { return false; } // Check that the automode context properties file exists File contextProperties = new File(folder, AUTO_MODE_CONTEXT_FILE); return contextProperties.exists(); } /** * Copy a local settings file to the remote folder. * * @param fileName Name of the file to copy * @param localFolder Local settings folder * @param remoteFolder Shared settings folder * @param missingFileOk True if it's not an error if the source file is not * found * * @throws SharedConfigurationException */ private static void copyToRemoteFolder(String fileName, String localFolder, File remoteFolder, boolean missingFileOk) throws SharedConfigurationException { logger.log(Level.INFO, "Uploading {0} to {1}", new Object[] { fileName, remoteFolder.getAbsolutePath() }); File localFile = new File(localFolder, fileName); if (!localFile.exists()) { Path deleteRemote = Paths.get(remoteFolder.toString(), fileName); try { if (deleteRemote.toFile().exists()) { deleteRemote.toFile().delete(); } } catch (SecurityException ex) { logger.log(Level.SEVERE, "Shared configuration {0} does not exist on local node, but unable to remove remote copy", fileName); throw new SharedConfigurationException( "Shared configuration file " + deleteRemote.toString() + " could not be deleted."); } if (!missingFileOk) { logger.log(Level.SEVERE, "Local configuration file {0} does not exist", localFile.getAbsolutePath()); throw new SharedConfigurationException( "Local configuration file " + localFile.getAbsolutePath() + " does not exist"); } else { logger.log(Level.INFO, "Local configuration file {0} does not exist", localFile.getAbsolutePath()); return; } } try { FileUtils.copyFileToDirectory(localFile, remoteFolder); } catch (IOException ex) { throw new SharedConfigurationException(String.format("Failed to copy %s to %s", localFile.getAbsolutePath(), remoteFolder.getAbsolutePath()), ex); } } /** * Copy a shared settings file to the local settings folder. * * @param fileName Name of the file to copy * @param localFolder Local settings folder * @param remoteFolder Shared settings folder * @param missingFileOk True if it's not an error if the source file is not * found * * @throws SharedConfigurationException */ private static void copyToLocalFolder(String fileName, String localFolder, File remoteFolder, boolean missingFileOk) throws SharedConfigurationException { logger.log(Level.INFO, "Downloading {0} from {1}", new Object[] { fileName, remoteFolder.getAbsolutePath() }); File remoteFile = new File(remoteFolder, fileName); if (!remoteFile.exists()) { Path deleteLocal = Paths.get(localFolder, fileName); try { if (deleteLocal.toFile().exists()) { deleteLocal.toFile().delete(); } } catch (SecurityException ex) { logger.log(Level.SEVERE, "Shared configuration {0} does not exist on remote node, but unable to remove local copy", fileName); throw new SharedConfigurationException( "Shared configuration file " + deleteLocal.toString() + " could not be deleted."); } if (!missingFileOk) { logger.log(Level.SEVERE, "Shared configuration file {0} does not exist", remoteFile.getAbsolutePath()); throw new SharedConfigurationException( "Shared configuration file " + remoteFile.getAbsolutePath() + " does not exist"); } else { logger.log(Level.INFO, "Shared configuration file {0} does not exist", remoteFile.getAbsolutePath()); return; } } File localSettingsFolder = new File(localFolder); try { FileUtils.copyFileToDirectory(remoteFile, localSettingsFolder); } catch (IOException ex) { throw new SharedConfigurationException(String.format("Failed to copy %s to %s", remoteFile.getAbsolutePath(), localSettingsFolder.getAbsolutePath()), ex); } } /** * Upload the basic set of auto-ingest settings to the shared folder. * * @param remoteFolder Shared settings folder * * @throws Exception */ private void uploadAutoModeContextSettings(File remoteFolder) throws SharedConfigurationException { logger.log(Level.INFO, "Uploading shared configuration to {0}", remoteFolder.getAbsolutePath()); publishTask("Uploading AutoModeContext configuration files"); // Make a subfolder File remoteAutoConfFolder = new File(remoteFolder, AUTO_MODE_FOLDER); try { if (remoteAutoConfFolder.exists()) { FileUtils.deleteDirectory(remoteAutoConfFolder); } Files.createDirectories(remoteAutoConfFolder.toPath()); } catch (IOException | SecurityException ex) { logger.log(Level.SEVERE, "Failed to create clean shared configuration subfolder " + remoteAutoConfFolder.getAbsolutePath(), ex); //NON-NLS throw new SharedConfigurationException("Failed to create clean shared configuration subfolder " + remoteAutoConfFolder.getAbsolutePath()); } IngestJobSettings ingestJobSettings = new IngestJobSettings( AutoIngestUserPreferences.getAutoModeIngestModuleContextString()); File localFolder = ingestJobSettings.getSavedModuleSettingsFolder().toFile(); if (!localFolder.exists()) { logger.log(Level.SEVERE, "Local configuration folder {0} does not exist", localFolder.getAbsolutePath()); throw new SharedConfigurationException( "Local configuration folder " + localFolder.getAbsolutePath() + " does not exist"); } try { FileUtils.copyDirectory(localFolder, remoteAutoConfFolder); } catch (IOException ex) { throw new SharedConfigurationException(String.format("Failed to copy %s to %s", localFolder.getAbsolutePath(), remoteAutoConfFolder.getAbsolutePath()), ex); } } /** * Download the basic set of auto-ingest settings from the shared folder * * @param remoteFolder Shared settings folder * * @throws SharedConfigurationException */ private void downloadAutoModeContextSettings(File remoteFolder) throws SharedConfigurationException { logger.log(Level.INFO, "Downloading shared configuration from {0}", remoteFolder.getAbsolutePath()); publishTask("Downloading AutoModeContext configuration files"); // Check that the remote subfolder exists File remoteAutoConfFolder = new File(remoteFolder, AUTO_MODE_FOLDER); if (!remoteAutoConfFolder.exists()) { logger.log(Level.SEVERE, "Shared configuration folder {0} does not exist", remoteAutoConfFolder.getAbsolutePath()); throw new SharedConfigurationException( "Shared configuration folder " + remoteAutoConfFolder.getAbsolutePath() + " does not exist"); } // Get/create the local subfolder IngestJobSettings ingestJobSettings = new IngestJobSettings( AutoIngestUserPreferences.getAutoModeIngestModuleContextString()); File localFolder = ingestJobSettings.getSavedModuleSettingsFolder().toFile(); try { if (localFolder.exists()) { FileUtils.deleteDirectory(localFolder); } Files.createDirectories(localFolder.toPath()); } catch (IOException | SecurityException ex) { logger.log(Level.SEVERE, "Failed to create clean local configuration folder " + localFolder.getAbsolutePath(), ex); //NON-NLS throw new SharedConfigurationException( "Failed to create clean local configuration folder " + localFolder.getAbsolutePath()); } try { FileUtils.copyDirectory(remoteAutoConfFolder, localFolder); } catch (IOException ex) { throw new SharedConfigurationException(String.format("Failed to copy %s to %s", remoteFolder.getAbsolutePath(), localFolder.getAbsolutePath()), ex); } } /** * Upload settings file containing enabled ingest modules. * * @param remoteFolder Shared settings folder * * @throws SharedConfigurationException */ private void uploadEnabledModulesSettings(File remoteFolder) throws SharedConfigurationException { publishTask("Uploading enabled module configuration"); copyToRemoteFolder(AUTO_MODE_CONTEXT_FILE, moduleDirPath, remoteFolder, false); } /** * Download settings file containing enabled ingest modules. * * @param remoteFolder Shared settings folder * * @throws SharedConfigurationException */ private void downloadEnabledModuleSettings(File remoteFolder) throws SharedConfigurationException { publishTask("Downloading enabled module configuration"); copyToLocalFolder(AUTO_MODE_CONTEXT_FILE, moduleDirPath, remoteFolder, false); } /** * Upload settings file containing file type settings. * * @param remoteFolder Shared settings folder * * @throws SharedConfigurationException */ private void uploadFileTypeSettings(File remoteFolder) throws SharedConfigurationException { publishTask("Uploading FileType module configuration"); copyToRemoteFolder(USER_DEFINED_TYPE_DEFINITIONS_FILE, moduleDirPath, remoteFolder, true); copyToRemoteFolder(USER_DEFINED_TYPE_DEFINITIONS_FILE_LEGACY, moduleDirPath, remoteFolder, true); } /** * Download settings file containing file type settings. * * @param remoteFolder Shared settings folder * * @throws SharedConfigurationException */ private void downloadFileTypeSettings(File remoteFolder) throws SharedConfigurationException { publishTask("Downloading FileType module configuration"); copyToLocalFolder(USER_DEFINED_TYPE_DEFINITIONS_FILE, moduleDirPath, remoteFolder, true); copyToLocalFolder(USER_DEFINED_TYPE_DEFINITIONS_FILE_LEGACY, moduleDirPath, remoteFolder, true); } /** * Upload settings for the interesting files module. * * @param remoteFolder Shared settings folder * * @throws SharedConfigurationException */ private void uploadInterestingFilesSettings(File remoteFolder) throws SharedConfigurationException { publishTask("Uploading InterestingFiles module configuration"); copyToRemoteFolder(INTERESTING_FILES_SET_DEFS_FILE_LEGACY, moduleDirPath, remoteFolder, true); copyToRemoteFolder(INTERESTING_FILES_SET_DEFS_FILE, moduleDirPath, remoteFolder, true); } /** * Download settings for the interesting files module. * * @param remoteFolder Shared settings folder * * @throws SharedConfigurationException */ private void downloadInterestingFilesSettings(File remoteFolder) throws SharedConfigurationException { publishTask("Downloading InterestingFiles module configuration"); copyToLocalFolder(INTERESTING_FILES_SET_DEFS_FILE_LEGACY, moduleDirPath, remoteFolder, true); copyToLocalFolder(INTERESTING_FILES_SET_DEFS_FILE, moduleDirPath, remoteFolder, true); } /** * Upload settings for the keyword search module. * * @param remoteFolder Shared settings folder * * @throws SharedConfigurationException */ private void uploadKeywordSearchSettings(File remoteFolder) throws SharedConfigurationException { publishTask("Uploading KeywordSearch module configuration"); copyToRemoteFolder(KEYWORD_SEARCH_SETTINGS, moduleDirPath, remoteFolder, true); copyToRemoteFolder(KEYWORD_SEARCH_SETTINGS_LEGACY, moduleDirPath, remoteFolder, true); copyToRemoteFolder(KEYWORD_SEARCH_GENERAL_LEGACY, moduleDirPath, remoteFolder, true); copyToRemoteFolder(KEYWORD_SEARCH_NSRL_LEGACY, moduleDirPath, remoteFolder, true); copyToRemoteFolder(KEYWORD_SEARCH_OPTIONS_LEGACY, moduleDirPath, remoteFolder, true); copyToRemoteFolder(KEYWORD_SEARCH_SCRIPTS_LEGACY, moduleDirPath, remoteFolder, true); } /** * Download settings for the keyword search module. * * @param remoteFolder Shared settings folder * * @throws SharedConfigurationException */ private void downloadKeywordSearchSettings(File remoteFolder) throws SharedConfigurationException { publishTask("Downloading KeywordSearch module configuration"); copyToLocalFolder(KEYWORD_SEARCH_SETTINGS, moduleDirPath, remoteFolder, true); copyToLocalFolder(KEYWORD_SEARCH_SETTINGS_LEGACY, moduleDirPath, remoteFolder, true); copyToLocalFolder(KEYWORD_SEARCH_GENERAL_LEGACY, moduleDirPath, remoteFolder, true); copyToLocalFolder(KEYWORD_SEARCH_NSRL_LEGACY, moduleDirPath, remoteFolder, true); copyToLocalFolder(KEYWORD_SEARCH_OPTIONS_LEGACY, moduleDirPath, remoteFolder, true); copyToLocalFolder(KEYWORD_SEARCH_SCRIPTS_LEGACY, moduleDirPath, remoteFolder, true); KeywordListsManager.reloadKeywordLists(); } /** * Upload settings for the file extension mismatch module. * * @param remoteFolder Shared settings folder * * @throws SharedConfigurationException */ private void uploadFileExtMismatchSettings(File remoteFolder) throws SharedConfigurationException { publishTask("Uploading File Extension Mismatch module configuration"); copyToRemoteFolder(FILE_EXT_MISMATCH_SETTINGS, moduleDirPath, remoteFolder, true); copyToRemoteFolder(FILE_EXT_MISMATCH_SETTINGS_LEGACY, moduleDirPath, remoteFolder, false); } /** * Download settings for the file extension mismatch module. * * @param remoteFolder Shared settings folder * * @throws SharedConfigurationException */ private void downloadFileExtMismatchSettings(File remoteFolder) throws SharedConfigurationException { publishTask("Downloading File Extension Mismatch module configuration"); copyToLocalFolder(FILE_EXT_MISMATCH_SETTINGS, moduleDirPath, remoteFolder, true); copyToLocalFolder(FILE_EXT_MISMATCH_SETTINGS_LEGACY, moduleDirPath, remoteFolder, false); } /** * Upload settings for the android triage module. * * @param remoteFolder Shared settings folder * * @throws SharedConfigurationException */ private void uploadAndroidTriageSettings(File remoteFolder) throws SharedConfigurationException { publishTask("Uploading Android Triage module configuration"); copyToRemoteFolder(ANDROID_TRIAGE, moduleDirPath, remoteFolder, true); } /** * Download settings for the android triage module. * * @param remoteFolder Shared settings folder * * @throws SharedConfigurationException */ private void downloadAndroidTriageSettings(File remoteFolder) throws SharedConfigurationException { publishTask("Downloading Android Triage module configuration"); copyToLocalFolder(ANDROID_TRIAGE, moduleDirPath, remoteFolder, true); } /** * Upload File Exporter settings. * * @param remoteFolder Shared settings folder * * @throws SharedConfigurationException */ private void uploadFileExporterSettings(File remoteFolder) throws SharedConfigurationException { publishTask("Uploading File Exporter configuration"); File fileExporterFolder = new File(moduleDirPath, FILE_EXPORTER_FOLDER); copyToRemoteFolder(FILE_EXPORTER_SETTINGS_FILE, fileExporterFolder.getAbsolutePath(), remoteFolder, true); } /** * Download File Exporter settings. * * @param remoteFolder Shared settings folder * * @throws SharedConfigurationException */ private void downloadFileExporterSettings(File remoteFolder) throws SharedConfigurationException { publishTask("Downloading File Exporter configuration"); File fileExporterFolder = new File(moduleDirPath, FILE_EXPORTER_FOLDER); copyToLocalFolder(FILE_EXPORTER_SETTINGS_FILE, fileExporterFolder.getAbsolutePath(), remoteFolder, true); } /** * Upload multi-user settings and other general Autopsy settings * * @param remoteFolder Shared settings folder * * @throws SharedConfigurationException */ private void uploadMultiUserAndGeneralSettings(File remoteFolder) throws SharedConfigurationException { publishTask("Uploading multi user configuration"); File generalSettingsFolder = Paths.get(moduleDirPath, PREFERENCES_FOLDER, "org", "sleuthkit", "autopsy") .toFile(); copyToRemoteFolder(GENERAL_PROPERTIES, generalSettingsFolder.getAbsolutePath(), remoteFolder, false); copyToRemoteFolder(AUTO_INGEST_PROPERTIES, moduleDirPath, remoteFolder, false); } /** * Download multi-user settings and other general Autopsy settings * * @param remoteFolder Shared settings folder * * @throws SharedConfigurationException */ private void downloadMultiUserAndGeneralSettings(File remoteFolder) throws SharedConfigurationException { publishTask("Downloading multi user configuration"); File generalSettingsFolder = Paths.get(moduleDirPath, PREFERENCES_FOLDER, "org", "sleuthkit", "autopsy") .toFile(); copyToLocalFolder(GENERAL_PROPERTIES, generalSettingsFolder.getAbsolutePath(), remoteFolder, false); copyToLocalFolder(AUTO_INGEST_PROPERTIES, moduleDirPath, remoteFolder, false); } /** * Upload settings and hash databases to the shared folder. The general * algorithm is: - Copy the general settings in hashsets.xml - For each hash * database listed in hashsets.xml: - Calculate the CRC of the database - If * the CRC does not match the one listed for that database in the shared * folder, (or if no entry exists), copy the database - Store the CRCs for * each database in the shared folder and locally * * @param remoteFolder Shared settings folder * * @throws SharedConfigurationException */ private void uploadHashDbSettings(File remoteFolder) throws SharedConfigurationException { publishTask("Uploading HashDb module configuration"); // Keep track of everything being uploaded File localVersionFile = new File(moduleDirPath, SHARED_CONFIG_VERSIONS); File sharedVersionFile = new File(remoteFolder, SHARED_CONFIG_VERSIONS); Map<String, String> newVersions = new HashMap<>(); Map<String, String> sharedVersions = readVersionsFromFile(sharedVersionFile); // Copy the settings file copyToRemoteFolder(HASHDB_CONFIG_FILE_NAME, moduleDirPath, remoteFolder, true); copyToRemoteFolder(HASHDB_CONFIG_FILE_NAME_LEGACY, moduleDirPath, remoteFolder, true); // Get the list of databases from the file List<String> databases = getHashFileNamesFromSettingsFile(); for (String fullPathToDbFile : databases) { // Compare the CRC of the local copy with what is stored in the shared folder publishTask("Deciding whether to upload " + fullPathToDbFile); String crc = calculateCRC(fullPathToDbFile); // Determine full path to db file in remote folder String sharedName = convertLocalDbPathToShared(fullPathToDbFile); File sharedDbBaseFolder = new File(remoteFolder, REMOTE_HASH_FOLDER); File sharedDb = new File(sharedDbBaseFolder, sharedName); if (!(sharedVersions.containsKey(fullPathToDbFile) && sharedVersions.get(fullPathToDbFile).equals(crc) && sharedDb.exists())) { publishTask("Uploading " + fullPathToDbFile); File sharedDbPath = sharedDb.getParentFile(); if (!sharedDbPath.exists()) { if (!sharedDbPath.mkdirs()) { throw new SharedConfigurationException( "Error creating shared hash database directory " + sharedDbPath.getAbsolutePath()); } } File dbFile = new File(fullPathToDbFile); // copy hash db file to the remote folder copyFile(sharedDbPath, dbFile); // check whether the hash db has an index file (.idx) that should also be copied. // NOTE: only text hash databases (.txt, .hash, .Hash) can have index file. // it is possible that the hash db file itself is the index file String fullPathToIndexFile = ""; if (fullPathToDbFile.endsWith(".txt") || fullPathToDbFile.endsWith(".hash") || fullPathToDbFile.endsWith(".Hash")) { // check whether index file for this text database is present // For example, if text db name is "hash_db.txt" then index file name will be "hash_db.txt-md5.idx" fullPathToIndexFile = fullPathToDbFile + "-md5.idx"; // if index file exists, copy it to the remote location File dbIndexFile = new File(fullPathToIndexFile); if (dbIndexFile.exists()) { // copy index file to the remote folder copyFile(sharedDbPath, dbIndexFile); } else { fullPathToIndexFile = ""; } } else if (fullPathToDbFile.endsWith(".idx")) { // hash db file itself is the index file and it has already been copied to the remote location fullPathToIndexFile = fullPathToDbFile; } // check whether "index of the index" file exists for this hash DB's index file. // NOTE: "index of the index" file may only exist // for text hash database index files (i.e ".idx" extension). The index of the // index file will always have the same name as the index file, // distinguished only by the "2" in the extension. "index of the index" file // is optional and may not be present. if (fullPathToIndexFile.endsWith(".idx")) { String fullPathToIndexOfIndexFile = fullPathToIndexFile + "2"; // "index of the index" file has same file name and extension ".idx2" File dbIndexOfIndexFile = new File(fullPathToIndexOfIndexFile); if (dbIndexOfIndexFile.exists()) { // copy index of the index file to the remote folder copyFile(sharedDbPath, dbIndexOfIndexFile); } } } newVersions.put(fullPathToDbFile, crc); } // Write the versions of all uploaded files to a file (make local and shared copies) writeVersionsToFile(localVersionFile, newVersions); writeVersionsToFile(sharedVersionFile, newVersions); } /** * Utility method to copy a file * * @param sharedDbPath File object of the folder to copy to * @param dbFile File object of the file to copy * * @throws * org.sleuthkit.autopsy.configuration.SharedConfiguration.SharedConfigurationException */ private void copyFile(File sharedDbPath, File dbFile) throws SharedConfigurationException { try { Path path = Paths.get(sharedDbPath.toString(), dbFile.getName()); if (path.toFile().exists()) { path.toFile().delete(); } FileUtils.copyFileToDirectory(dbFile, sharedDbPath); } catch (IOException | SecurityException ex) { throw new SharedConfigurationException(String.format("Failed to copy %s to %s", dbFile.getAbsolutePath(), sharedDbPath.getAbsolutePath()), ex); } } /** * Upload settings and hash databases to the shared folder. The general * algorithm is: - Copy the general settings in hashsets.xml - For each hash * database listed in hashsets.xml: - Compare the recorded CRC in the shared * directory with the one in the local directory - If different, download * the database - Update the local list of database CRCs Note that databases * are downloaded to the exact path they were uploaded from. * * @param remoteFolder Shared settings folder * * @throws SharedConfigurationException */ private void downloadHashDbSettings(File remoteFolder) throws SharedConfigurationException { publishTask("Downloading HashDb module configuration"); // Read in the current local and shared database versions File localVersionFile = new File(moduleDirPath, SHARED_CONFIG_VERSIONS); File remoteVersionFile = new File(remoteFolder, SHARED_CONFIG_VERSIONS); Map<String, String> localVersions = readVersionsFromFile(localVersionFile); Map<String, String> remoteVersions = readVersionsFromFile(remoteVersionFile); /* Iterate through remote list If local needs it, download Download remote settings files to local Download remote versions file to local HashDbManager reload */ File localDb = new File(""); File sharedDb = new File(""); try { for (String path : remoteVersions.keySet()) { localDb = new File(path); if ((!localVersions.containsKey(path)) || (!localVersions.get(path).equals(remoteVersions.get(path))) || !localDb.exists()) { // Need to download a new copy if // - We have no entry for the database in the local versions file // - The CRC in the local versions file does not match the one in the shared file // - Local copy of the database does not exist if (localDb.exists()) { String crc = calculateCRC(path); if (crc.equals(remoteVersions.get(path))) { // Can skip the download if the local disk has it // but it's just not in the versions file. This will // be populated just before refreshing HashDbManager. continue; } } publishTask("Downloading " + path); String sharedName = convertLocalDbPathToShared(path); File sharedDbBaseFolder = new File(remoteFolder, REMOTE_HASH_FOLDER); sharedDb = new File(sharedDbBaseFolder, sharedName); if (!localDb.getParentFile().exists()) { if (!localDb.getParentFile().mkdirs()) { throw new SharedConfigurationException("Error creating hash database directory " + localDb.getParentFile().getAbsolutePath()); } } // If a copy of the database is loaded, close it before deleting and copying. if (localDb.exists()) { List<HashDbManager.HashDb> hashDbs = HashDbManager.getInstance().getAllHashSets(); HashDbManager.HashDb matchingDb = null; for (HashDbManager.HashDb db : hashDbs) { try { if (localDb.getAbsolutePath().equals(db.getDatabasePath()) || localDb.getAbsolutePath().equals(db.getIndexPath())) { matchingDb = db; break; } } catch (TskCoreException ex) { throw new SharedConfigurationException( String.format("Error getting hash database path info for %s", localDb.getParentFile().getAbsolutePath()), ex); } } if (matchingDb != null) { try { HashDbManager.getInstance().removeHashDatabase(matchingDb); } catch (HashDbManager.HashDbManagerException ex) { throw new SharedConfigurationException(String.format( "Error updating hash database info for %s", localDb.getAbsolutePath()), ex); } } if (localDb.exists()) { localDb.delete(); } } FileUtils.copyFile(sharedDb, localDb); // check whether the hash db has an index file (.idx) that should also be copied. // NOTE: only text hash databases (.txt, .hash, .Hash) can have index file. // it is possible that the hash db file itself is the index file String fullPathToRemoteDbFile = sharedDb.getPath(); String fullPathToRemoteIndexFile = ""; String fullPathToLocalIndexFile = ""; if (fullPathToRemoteDbFile.endsWith(".txt") || fullPathToRemoteDbFile.toLowerCase().endsWith(".hash")) { // check whether index file for this text database is present // For example, if text db name is "hash_db.txt" then index file name will be "hash_db.txt-md5.idx" fullPathToRemoteIndexFile = fullPathToRemoteDbFile + "-md5.idx"; // if index file exists, copy it to the remote location File remoteDbIndexFile = new File(fullPathToRemoteIndexFile); if (remoteDbIndexFile.exists()) { // delete local copy of "index of the index" file if one exists fullPathToLocalIndexFile = localDb.getPath() + "-md5.idx"; File localIndexFile = new File(fullPathToLocalIndexFile); if (localIndexFile.exists()) { localIndexFile.delete(); } // copy index file to the remote folder FileUtils.copyFile(remoteDbIndexFile, localIndexFile); } else { // index file doesn't exist at remote location fullPathToRemoteIndexFile = ""; } } else if (fullPathToRemoteDbFile.endsWith(".idx")) { // hash db file itself is the index file and it has already been copied to the remote location fullPathToRemoteIndexFile = fullPathToRemoteDbFile; fullPathToLocalIndexFile = localDb.getPath(); } // check whether "index of the index" file exists for this hash DB index file. // NOTE: "index of the index" file may only exist for hash database index files (.idx files). // For example, hash_db.txt-md5.idx index file will have hash_db.txt-md5.idx2 "index of the index" file. // "index of the index" file is optional and may not be present. if (fullPathToRemoteIndexFile.endsWith(".idx")) { // check if "index of the index" file exists in remote shared config folder String fullPathToRemoteIndexOfIndexFile = fullPathToRemoteIndexFile + "2"; // "index of the index" file has same file name with extension ".idx2" File remoteIndexOfIndexFile = new File(fullPathToRemoteIndexOfIndexFile); if (remoteIndexOfIndexFile.exists()) { // delete local copy of "index of the index" file if one exists String fullPathToLocalIndexOfIndexFile = fullPathToLocalIndexFile + "2"; // "index of the index" file has same file name with extension ".idx2" File localIndexOfIndexFile = new File(fullPathToLocalIndexOfIndexFile); if (localIndexOfIndexFile.exists()) { localIndexOfIndexFile.delete(); } // copy index of the index file to the local folder FileUtils.copyFile(remoteIndexOfIndexFile, localIndexOfIndexFile); } } } } } catch (IOException | SecurityException ex) { throw new SharedConfigurationException( String.format("Failed to copy %s to %s", sharedDb.getAbsolutePath(), localDb.getAbsolutePath()), ex); } // Copy the settings filey copyToLocalFolder(HASHDB_CONFIG_FILE_NAME, moduleDirPath, remoteFolder, true); copyToLocalFolder(HASHDB_CONFIG_FILE_NAME_LEGACY, moduleDirPath, remoteFolder, true); copyToLocalFolder(SHARED_CONFIG_VERSIONS, moduleDirPath, remoteFolder, true); // Refresh HashDbManager with the new settings HashDbManager.getInstance().loadLastSavedConfiguration(); } /** * Read in the hashsets settings to pull out the names of the databases. * * @return List of all hash databases * * @throws SharedConfigurationException */ private static List<String> getHashFileNamesFromSettingsFile() throws SharedConfigurationException { List<String> results = new ArrayList<>(); try { HashDbManager hashDbManager = HashDbManager.getInstance(); hashDbManager.loadLastSavedConfiguration(); for (HashDb hashDb : hashDbManager.getAllHashSets()) { if (hashDb.hasIndexOnly()) { results.add(hashDb.getIndexPath()); } else { results.add(hashDb.getDatabasePath()); } } } catch (TskCoreException ex) { throw new SharedConfigurationException("Unable to read hash databases", ex); } return results; } /** * Change the database path into a form that can be used to create * subfolders in the shared folder. * * @param localName Database name from the XML file * * @return Path with the initial colon removed */ private static String convertLocalDbPathToShared(String localName) { // Replace the colon String sharedName = localName.replace(":", "__colon__"); return sharedName; } /** * Write the list of database paths and versions to a file. * * @param versionFile File to write to * @param versions Map of database name -> version (current using CRCs as * versions) * * @throws SharedConfigurationException */ private static void writeVersionsToFile(File versionFile, Map<String, String> versions) throws SharedConfigurationException { try (PrintWriter writer = new PrintWriter(versionFile.getAbsoluteFile(), "UTF-8")) { for (String filename : versions.keySet()) { writer.println(versions.get(filename) + " " + filename); } } catch (FileNotFoundException | UnsupportedEncodingException ex) { throw new SharedConfigurationException(String.format("Failed to write version info to %s", versionFile), ex); } } /** * Read the map of database paths to versions from a file. * * @param versionFile File containing the version information * * @return Map of database name -> version * * @throws SharedConfigurationException */ private static Map<String, String> readVersionsFromFile(File versionFile) throws SharedConfigurationException { Map<String, String> versions = new HashMap<>(); // If the file does not exist, return an empty map if (!versionFile.exists()) { return versions; } // Read in and store each pair try (BufferedReader reader = new BufferedReader(new FileReader(versionFile))) { String currentLine = reader.readLine(); while (null != currentLine) { if (!currentLine.isEmpty()) { int index = currentLine.indexOf(' '); // Find the first space String crc = currentLine.substring(0, index); String path = currentLine.substring(index + 1); versions.put(path, crc); } currentLine = reader.readLine(); } } catch (FileNotFoundException ex) { throw new SharedConfigurationException(String.format("Failed to find version file %s", versionFile), ex); } catch (IOException ex) { throw new SharedConfigurationException( String.format("Failed to read version info from %s", versionFile), ex); } return versions; } /** * Calculate the CRC of a file to use to determine if it has changed. * * @param filePath File to get the CRC for * * @return String containing the CRC * * @throws SharedConfigurationException */ private static String calculateCRC(String filePath) throws SharedConfigurationException { File file = new File(filePath); try { FileInputStream fileStream = new FileInputStream(file); CRC32 crc = new CRC32(); byte[] buffer = new byte[65536]; int bytesRead = fileStream.read(buffer); while (-1 != bytesRead) { crc.update(buffer, 0, bytesRead); bytesRead = fileStream.read(buffer); } return String.valueOf(crc.getValue()); } catch (IOException ex) { throw new SharedConfigurationException( String.format("Failed to calculate CRC for %s", file.getAbsolutePath()), ex); } } }