Java tutorial
/******************************************************************************* * Copyright 2010-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. * A copy of the License is located at * * http://aws.amazon.com/apache2.0 * * or in the "license" file accompanying this file. This file is distributed * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either * express or implied. See the License for the specific language governing * permissions and limitations under the License. ******************************************************************************/ package com.amazonaws.services.cloudtrail.processinglibrary; import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import com.amazonaws.ClientConfiguration; import com.amazonaws.regions.Region; import com.amazonaws.regions.Regions; import com.amazonaws.services.cloudtrail.processinglibrary.configuration.ProcessingConfiguration; import com.amazonaws.services.cloudtrail.processinglibrary.configuration.PropertiesFileConfiguration; import com.amazonaws.services.cloudtrail.processinglibrary.factory.EventReaderFactory; import com.amazonaws.services.cloudtrail.processinglibrary.factory.ThreadPoolFactory; import com.amazonaws.services.cloudtrail.processinglibrary.impl.DefaultExceptionHandler; import com.amazonaws.services.cloudtrail.processinglibrary.impl.DefaultProgressReporter; import com.amazonaws.services.cloudtrail.processinglibrary.impl.DefaultEventFilter; import com.amazonaws.services.cloudtrail.processinglibrary.impl.DefaultEventsProcessor; import com.amazonaws.services.cloudtrail.processinglibrary.impl.DefaultSourceFilter; import com.amazonaws.services.cloudtrail.processinglibrary.interfaces.ExceptionHandler; import com.amazonaws.services.cloudtrail.processinglibrary.interfaces.ProgressReporter; import com.amazonaws.services.cloudtrail.processinglibrary.interfaces.EventFilter; import com.amazonaws.services.cloudtrail.processinglibrary.interfaces.EventsProcessor; import com.amazonaws.services.cloudtrail.processinglibrary.interfaces.SourceFilter; import com.amazonaws.services.cloudtrail.processinglibrary.manager.S3Manager; import com.amazonaws.services.cloudtrail.processinglibrary.manager.SqsManager; import com.amazonaws.services.cloudtrail.processinglibrary.model.CloudTrailSource; import com.amazonaws.services.cloudtrail.processinglibrary.reader.EventReader; import com.amazonaws.services.cloudtrail.processinglibrary.utils.LibraryUtils; import com.amazonaws.services.s3.AmazonS3Client; import com.amazonaws.services.sqs.AmazonSQSClient; /** * AWS CloudTrail Processing Library's main execution logic. This class loads a user's configuration and * creates an {@link EventReaderFactory} object which spawns a {@link EventReader} to process log files. * <p> * It has two thread pools: <code>scheduledThreadPool</code>, which is a single-threaded scheduled * thread pool used to poll SQS for messages, and <code>mainThreadPool</code>, which has a * configurable size and processes each {@link CloudTrailSource} in parallel. */ public class AWSCloudTrailProcessingExecutor { private static final Log logger = LogFactory.getLog(AWSCloudTrailProcessingExecutor.class); /** * The delay between starting time stamp of each execution, minimum 1 nanosecond. * In such a case it behaves as if it were continuously running. */ private static final int EXECUTION_DELAY = 1; //1 nanosecond private static final String ERROR_CONFIGURATION_NULL = "ProcessingConfiguration object is null. " + "Either pass in a class path property file path or directly pass in a ProcessingConfiguration object"; private ProcessingConfiguration config; private SourceFilter sourceFilter; private EventFilter eventFilter; private EventsProcessor eventsProcessor; private ProgressReporter progressReporter; private ExceptionHandler exceptionHandler; /** * Scheduled thread pool used to continuously poll queue and enqueue jobs * into our main thread pool executorService. */ private ScheduledExecutorService scheduledThreadPool; /** * The thread pool that processes the log files. */ private ExecutorService mainThreadPool; private EventReaderFactory readerFactory; private AWSCloudTrailProcessingExecutor(Builder builder) { this.config = builder.config; this.sourceFilter = builder.sourceFilter; this.eventFilter = builder.eventFilter; this.eventsProcessor = builder.eventsProcessor; this.progressReporter = builder.progressReporter; this.exceptionHandler = builder.exceptionHandler; this.scheduledThreadPool = builder.scheduledThreadPool; this.mainThreadPool = builder.mainThreadPool; this.readerFactory = builder.readerFactory; } /** * Start processing AWS CloudTrail logs. */ public void start() { logger.info("Started AWSCloudTrailProcessingLibrary."); this.ValidateBeforeStart(); scheduledThreadPool.scheduleAtFixedRate(new ScheduledJob(this.readerFactory), 0L, EXECUTION_DELAY, TimeUnit.MICROSECONDS); } /** * Stop processing AWS CloudTrail logs. */ public void stop() { stopThreadPool(this.mainThreadPool); stopThreadPool(this.scheduledThreadPool); logger.info("Stopped AWSCloudTrailProcessingLibrary."); } /** * Helper function to gracefully stop an {@link ExecutorService}. * * @param threadPool the thread pool to stop. */ private void stopThreadPool(ExecutorService threadPool) { LibraryUtils.checkCondition(threadPool == null, "Thread pool is null when calling stop"); if (threadPool.isShutdown()) { logger.debug(threadPool.toString() + " is already stopped."); } else { logger.debug(threadPool.toString() + " is about to shutdown."); threadPool.shutdown(); // Shutdown thread pool try { // Wait for shutdown threadPool.awaitTermination(this.config.getThreadTerminationDelaySeconds(), TimeUnit.SECONDS); } catch (InterruptedException e) { logger.debug("Wait thread pool termination is interrupted."); } if (!threadPool.isShutdown()) { // ShutdownNow after waiting logger.debug(threadPool.toString() + " is force to shutdown now."); threadPool.shutdownNow(); } logger.debug(threadPool.toString() + " is stopped."); } } /** * A job that runs in a scheduled thread pool. * * If any execution of the task raises an exception, then {@link ScheduledExecutorService} will * suppress any subsequent executions. Therefore, we try/catch a Throwable here. */ private class ScheduledJob implements Runnable { private EventReaderFactory eventReaderFactory; public ScheduledJob(EventReaderFactory eventReaderFactory) { this.eventReaderFactory = eventReaderFactory; } /** * Run the scheduled job. */ public void run() { try { final EventReader reader = eventReaderFactory.createReader(); List<CloudTrailSource> sources = reader.getSources(); for (final CloudTrailSource source : sources) { // process each CloudTrailSource in main thread pool mainThreadPool.execute(new Runnable() { public void run() { reader.processSource(source); } }); } } catch (Throwable t) { logger.error("Executor failed to process a task. " + t.getMessage(), t); } } } /** * A class that builds an {@link AWSCloudTrailProcessingExecutor} object. */ public static class Builder { private static final int SDK_TIME_OUT = 10000; // 10 seconds private ProcessingConfiguration config; //provide default implementation to AWSCloudTrailProcessingLibrary interfaces. private SourceFilter sourceFilter = new DefaultSourceFilter(); private EventFilter eventFilter = new DefaultEventFilter(); private EventsProcessor eventsProcessor = new DefaultEventsProcessor(); private ProgressReporter progressReporter = new DefaultProgressReporter(); private ExceptionHandler exceptionHandler = new DefaultExceptionHandler(); private ScheduledExecutorService scheduledThreadPool; private ExecutorService mainThreadPool; private EventReaderFactory readerFactory; private String propertyFilePath; private AmazonS3Client s3Client; private AmazonSQSClient sqsClient; /** * Builder for {@link AWSCloudTrailProcessingExecutor}. * * @param eventsProcessor The {@link interfaces.EventsProcessor} that will process * {@link model.CloudTrailEvent}s. * @param propertyFilePath The path to a property file containing the AWS CloudTrail Processing Library's * configuration. */ public Builder(EventsProcessor eventsProcessor, String propertyFilePath) { this.eventsProcessor = eventsProcessor; this.propertyFilePath = propertyFilePath; } /** * Builder for {@link AWSCloudTrailProcessingExecutor}. * * @param eventsProcessor The {@link interfaces.EventsProcessor} instance that will process * {@link model.CloudTrailEvent}s. * @param config An {@link configuration.ProcessingConfiguration} instance that provides the library's * configuration details. */ public Builder(EventsProcessor eventsProcessor, ProcessingConfiguration config) { this.eventsProcessor = eventsProcessor; this.config = config; } /** * Applies a user-defined {@link interfaces.SourceFilter} to this instance. * * @param sourceFilter The <code>SourceFilter</code> that will be used to filter * {@link model.CloudTrailSource} source. * @return This <code>Builder</code> instance, using the specified <code>SourceFilter</code>. */ public Builder withSourceFilter(SourceFilter sourceFilter) { this.sourceFilter = sourceFilter; return this; } /** * Applies a user-defined {@link interfaces.EventFilter} to this instance. * * @param eventFilter The <code>EventFilter</code> that will be used to filter * {@link model.CloudTrailEvent}s. * @return This <code>Builder</code> instance, using the specified <code>EventFilter</code>. */ public Builder withEventFilter(EventFilter eventFilter) { this.eventFilter = eventFilter; return this; } /** * Applies a user-defined {@link interfaces.ProgressReporter} to this instance. * * @param progressReporter The <code>ProgressReporter</code> that will report * the state of the AWSCloudTrailProcessingLibrary processing process * * @return This <code>Builder</code> instance, using the specified <code>ProgressReporter</code>. */ public Builder withProgressReporter(ProgressReporter progressReporter) { this.progressReporter = progressReporter; return this; } /** * Applies a user-defined {@link interfaces.ExceptionHandler} to this instance. * * @param exceptionHandler The <code>ExceptionHandler</code> that will handle exceptions for * this instance. * * @return This <code>Builder</code> instance, using the specified * <code>ExceptionHandler</code>. */ public Builder withExceptionHandler(ExceptionHandler exceptionHandler) { this.exceptionHandler = exceptionHandler; return this; } /** * Applies a user-defined <a * href="http://docs.oracle.com/javase/7/docs/api/java/util/concurrent/ExecutorService.html">ExecutorService</a> * thread pool to this instance. * * @param mainThreadPool The <code>ExecutorService</code> thread pool that will be used to * process CloudTrailSource * * @return This <code>Builder</code> instance, using the specified thread pool. */ public Builder withThreadPool(ExecutorService mainThreadPool) { this.mainThreadPool = mainThreadPool; return this; } /** * Applies a user-defined <a * href="http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/sqs/AmazonS3Client.html">AmazonS3Client</a> * to this instance. * * @param s3Client the <code>AmazonS3Client</code> object used to download CloudTrail log files * * @return This <code>Builder</code> instance, using the specified * <code>AmazonS3Client</code>. */ public Builder withS3Client(AmazonS3Client s3Client) { this.s3Client = s3Client; return this; } /** * Applies a user-defined <a * href="http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/sqs/AmazonSQSClient.html">AmazonSQSClient</a> * to this instance. * * @param sqsClient The <code>AmazonSQSClient</code> that will be used to poll messages from * the SQS queue. * * @return This <code>Builder</code> instance, using the specified * <code>AmazonSQSClient</code>. */ public Builder withSQSClient(AmazonSQSClient sqsClient) { this.sqsClient = sqsClient; return this; } /** * Build an {@link AWSCloudTrailProcessingExecutor} using the classpath property file. * * @return an AWSCloudTrailProcessingExecutor instance. */ public AWSCloudTrailProcessingExecutor build() { // passed in configuration as property file if (this.config == null && this.propertyFilePath != null) { this.config = new PropertiesFileConfiguration(propertyFilePath); } LibraryUtils.checkArgumentNotNull(this.config, ERROR_CONFIGURATION_NULL); LibraryUtils.checkArgumentNotNull(this.config.getAwsCredentialsProvider(), "ProcessingConfiguration missing AWSCredentialsProvider attribute"); LibraryUtils.checkArgumentNotNull(this.eventsProcessor, "eventsProcessor is null."); LibraryUtils.checkArgumentNotNull(this.sourceFilter, "sourceFilter is null."); LibraryUtils.checkArgumentNotNull(this.eventFilter, "eventFilter is null."); LibraryUtils.checkArgumentNotNull(this.progressReporter, "progressReporter is null."); LibraryUtils.checkArgumentNotNull(this.exceptionHandler, "exceptionHandler is null."); // override default timeout for S3Client ClientConfiguration clientConfiguration = new ClientConfiguration(); clientConfiguration.setConnectionTimeout(SDK_TIME_OUT); clientConfiguration.setSocketTimeout(SDK_TIME_OUT); if (this.s3Client == null) { AmazonS3Client s3Client = new AmazonS3Client(this.config.getAwsCredentialsProvider(), clientConfiguration); s3Client.setRegion(Region.getRegion(Regions.fromName(this.config.getS3Region()))); this.s3Client = s3Client; } if (this.sqsClient == null) { AmazonSQSClient sqsClient = new AmazonSQSClient(config.getAwsCredentialsProvider()); sqsClient.setRegion(Region.getRegion(Regions.fromName(this.config.getSqsRegion()))); this.sqsClient = sqsClient; } LibraryUtils.checkArgumentNotNull(this.s3Client, "s3Client is null."); LibraryUtils.checkArgumentNotNull(this.sqsClient, "sqsClient is null."); SqsManager sqsManager = new SqsManager(sqsClient, this.config, this.exceptionHandler, this.progressReporter); S3Manager s3Manager = new S3Manager(s3Client, this.config, this.exceptionHandler, this.progressReporter); this.readerFactory = new EventReaderFactory.Builder(this.config) .withEventsProcessor(this.eventsProcessor).withSourceFilter(this.sourceFilter) .withEventFilter(this.eventFilter).withProgressReporter(this.progressReporter) .withExceptionHandler(this.exceptionHandler).withS3Manager(s3Manager).withSQSManager(sqsManager) .build(); LibraryUtils.checkArgumentNotNull(this.readerFactory, "readerFactory is null."); ThreadPoolFactory threadFactory = new ThreadPoolFactory(this.config.getThreadCount(), this.exceptionHandler); this.scheduledThreadPool = threadFactory.createScheduledThreadPool(); if (this.mainThreadPool == null) { this.mainThreadPool = threadFactory.createMainThreadPool(); } LibraryUtils.checkArgumentNotNull(this.scheduledThreadPool, "scheduledThreadPoll is null."); LibraryUtils.checkArgumentNotNull(this.mainThreadPool, "mainThreadPool is null."); return new AWSCloudTrailProcessingExecutor(this); } } /** * Validate the user's input before processing logs. */ private void ValidateBeforeStart() { LibraryUtils.checkArgumentNotNull(this.config, "Configuration is null."); this.config.validate(); LibraryUtils.checkArgumentNotNull(this.eventsProcessor, "eventsProcessor is null."); LibraryUtils.checkArgumentNotNull(this.sourceFilter, "sourceFilter is null."); LibraryUtils.checkArgumentNotNull(this.eventFilter, "eventFilter is null."); LibraryUtils.checkArgumentNotNull(this.progressReporter, "progressReporter is null."); LibraryUtils.checkArgumentNotNull(this.exceptionHandler, "exceptionHandler is null."); LibraryUtils.checkArgumentNotNull(this.scheduledThreadPool, "scheduledThreadPool is null."); LibraryUtils.checkArgumentNotNull(this.mainThreadPool, "mainThreadPool is null."); LibraryUtils.checkArgumentNotNull(this.readerFactory, "readerFactory is null."); } }