Java tutorial
/* * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"). * You may not use this file except in compliance with the License. * A copy of the License is located at * * http://aws.amazon.com/apache2.0 * * or in the "license" file accompanying this file. This file is distributed * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either * express or implied. See the License for the specific language governing * permissions and limitations under the License. */ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.util.List; import java.util.Properties; import java.util.concurrent.TimeUnit; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import com.amazonaws.AmazonClientException; import com.amazonaws.AmazonServiceException; import com.amazonaws.services.kinesisfirehose.model.BufferingHints; import com.amazonaws.services.kinesisfirehose.model.CompressionFormat; import com.amazonaws.services.kinesisfirehose.model.CreateDeliveryStreamRequest; import com.amazonaws.services.kinesisfirehose.model.DeliveryStreamDescription; import com.amazonaws.services.kinesisfirehose.model.EncryptionConfiguration; import com.amazonaws.services.kinesisfirehose.model.KMSEncryptionConfig; import com.amazonaws.services.kinesisfirehose.model.NoEncryptionConfig; import com.amazonaws.services.kinesisfirehose.model.S3DestinationConfiguration; import com.amazonaws.services.kinesisfirehose.model.S3DestinationUpdate; import com.amazonaws.services.kinesisfirehose.model.UpdateDestinationRequest; import com.amazonaws.util.StringUtils; /** * Amazon Kinesis Firehose is a fully managed service for real-time streaming data delivery * to destinations such as Amazon S3 and Amazon Redshift. Firehose is part of the Amazon Kinesis * streaming data family, along with Amazon Kinesis Streams. With Firehose, you do not need to * write any applications or manage any resources. You configure your data producers to send data * to Firehose and it automatically delivers the data to the destination that you specified. * * Detailed Amazon Kinesis Firehose documentation can be found here: * https://aws.amazon.com/documentation/firehose/ * * This is a sample java application to deliver data to Amazon S3 destination. */ public class AmazonKinesisFirehoseToS3Sample extends AbstractAmazonKinesisFirehoseDelivery { /* * Before running the code: * * Step 1: Please check you have AWS access credentials set under * (~/.aws/credentials). If not, fill in your AWS access credentials in the * provided credentials file template, and be sure to move the file to the * default location (~/.aws/credentials) where the sample code will load the * credentials from. * https://console.aws.amazon.com/iam/home?#security_credential * * WARNING: To avoid accidental leakage of your credentials, DO NOT keep the * credentials file in your source directory. * * Step 2: Update the firehosetos3sample.properties file with the required parameters. */ // DeliveryStream properties private static String updateS3ObjectPrefix; private static Integer updateSizeInMBs; private static Integer updateIntervalInSeconds; // Properties File private static final String CONFIG_FILE = "firehosetos3sample.properties"; // Logger private static final Log LOG = LogFactory.getLog(AmazonKinesisFirehoseToS3Sample.class); /** * Initialize the parameters. * * @throws Exception */ private static void init() throws Exception { // Load the parameters from properties file loadConfig(); // Initialize the clients initClients(); // Validate AccountId parameter is set if (StringUtils.isNullOrEmpty(accountId)) { throw new IllegalArgumentException( "AccountId is empty. Please enter the accountId in " + CONFIG_FILE + " file"); } } /** * Load the input parameters from properties file. * * @throws FileNotFoundException * @throws IOException */ private static void loadConfig() throws FileNotFoundException, IOException { try (InputStream configStream = Thread.currentThread().getContextClassLoader() .getResourceAsStream(CONFIG_FILE)) { if (configStream == null) { throw new FileNotFoundException(); } properties = new Properties(); properties.load(configStream); } // Read properties accountId = properties.getProperty("customerAccountId"); createS3Bucket = Boolean.valueOf(properties.getProperty("createS3Bucket")); s3RegionName = properties.getProperty("s3RegionName"); s3BucketName = properties.getProperty("s3BucketName").trim(); s3BucketARN = getBucketARN(s3BucketName); s3ObjectPrefix = properties.getProperty("s3ObjectPrefix").trim(); String sizeInMBsProperty = properties.getProperty("destinationSizeInMBs"); s3DestinationSizeInMBs = StringUtils.isNullOrEmpty(sizeInMBsProperty) ? null : Integer.parseInt(sizeInMBsProperty.trim()); String intervalInSecondsProperty = properties.getProperty("destinationIntervalInSeconds"); s3DestinationIntervalInSeconds = StringUtils.isNullOrEmpty(intervalInSecondsProperty) ? null : Integer.parseInt(intervalInSecondsProperty.trim()); deliveryStreamName = properties.getProperty("deliveryStreamName"); s3DestinationAWSKMSKeyId = properties.getProperty("destinationAWSKMSKeyId"); firehoseRegion = properties.getProperty("firehoseRegion"); iamRoleName = properties.getProperty("iamRoleName"); iamRegion = properties.getProperty("iamRegion"); // Update Delivery Stream Destination related properties enableUpdateDestination = Boolean.valueOf(properties.getProperty("updateDestination")); updateS3ObjectPrefix = properties.getProperty("updateS3ObjectPrefix").trim(); String updateSizeInMBsProperty = properties.getProperty("updateSizeInMBs"); updateSizeInMBs = StringUtils.isNullOrEmpty(updateSizeInMBsProperty) ? null : Integer.parseInt(updateSizeInMBsProperty.trim()); String updateIntervalInSecondsProperty = properties.getProperty("updateIntervalInSeconds"); updateIntervalInSeconds = StringUtils.isNullOrEmpty(updateIntervalInSecondsProperty) ? null : Integer.parseInt(updateIntervalInSecondsProperty.trim()); } public static void main(String[] args) throws Exception { init(); try { // Create S3 bucket for deliveryStream to deliver data createS3Bucket(); // Create the DeliveryStream createDeliveryStream(); // Print the list of delivery streams printDeliveryStreams(); // Put records into deliveryStream LOG.info("Putting records in deliveryStream : " + deliveryStreamName + " via Put Record method."); putRecordIntoDeliveryStream(); // Batch Put records into deliveryStream LOG.info("Putting records in deliveryStream : " + deliveryStreamName + " via Put Record Batch method. Now you can check your S3 bucket " + s3BucketName + " for the data delivered by deliveryStream."); putRecordBatchIntoDeliveryStream(); // Wait for some interval for the firehose to write data to the S3 bucket int waitTimeSecs = s3DestinationIntervalInSeconds == null ? DEFAULT_WAIT_INTERVAL_FOR_DATA_DELIVERY_SECS : s3DestinationIntervalInSeconds; waitForDataDelivery(waitTimeSecs); // Update the deliveryStream and Put records into updated deliveryStream, only if the flag is set if (enableUpdateDestination) { // Update the deliveryStream updateDeliveryStream(); // Wait for some interval to propagate the updated configuration options before ingesting data LOG.info("Waiting for few seconds to propagate the updated configuration options."); TimeUnit.SECONDS.sleep(60); // Put records into updated deliveryStream. Records will be delivered to new S3 prefix location LOG.info("Putting records in updated deliveryStream : " + deliveryStreamName + " via Put Record method."); putRecordIntoDeliveryStream(); // Batch Put records into updated deliveryStream. Records will be delivered to new S3 prefix location LOG.info("Putting records in updated deliveryStream : " + deliveryStreamName + " via Put Record Batch method."); putRecordBatchIntoDeliveryStream(); // Wait for some interval for the deliveryStream to write data to new prefix location in S3 bucket waitTimeSecs = updateIntervalInSeconds == null ? waitTimeSecs : updateIntervalInSeconds; waitForDataDelivery(waitTimeSecs); } } catch (AmazonServiceException ase) { LOG.error("Caught Amazon Service Exception"); LOG.error("Status Code " + ase.getErrorCode()); LOG.error("Message: " + ase.getErrorMessage(), ase); } catch (AmazonClientException ace) { LOG.error("Caught Amazon Client Exception"); LOG.error("Exception Message " + ace.getMessage(), ace); } } /** * Method to create delivery stream for S3 destination configuration. * * @throws Exception */ private static void createDeliveryStream() throws Exception { boolean deliveryStreamExists = false; LOG.info("Checking if " + deliveryStreamName + " already exits"); List<String> deliveryStreamNames = listDeliveryStreams(); if (deliveryStreamNames != null && deliveryStreamNames.contains(deliveryStreamName)) { deliveryStreamExists = true; LOG.info("DeliveryStream " + deliveryStreamName + " already exists. Not creating the new delivery stream"); } else { LOG.info("DeliveryStream " + deliveryStreamName + " does not exist"); } if (!deliveryStreamExists) { // Create deliveryStream CreateDeliveryStreamRequest createDeliveryStreamRequest = new CreateDeliveryStreamRequest(); createDeliveryStreamRequest.setDeliveryStreamName(deliveryStreamName); S3DestinationConfiguration s3DestinationConfiguration = new S3DestinationConfiguration(); s3DestinationConfiguration.setBucketARN(s3BucketARN); s3DestinationConfiguration.setPrefix(s3ObjectPrefix); // Could also specify GZIP or ZIP s3DestinationConfiguration.setCompressionFormat(CompressionFormat.UNCOMPRESSED); // Encryption configuration is optional EncryptionConfiguration encryptionConfiguration = new EncryptionConfiguration(); if (!StringUtils.isNullOrEmpty(s3DestinationAWSKMSKeyId)) { encryptionConfiguration.setKMSEncryptionConfig( new KMSEncryptionConfig().withAWSKMSKeyARN(s3DestinationAWSKMSKeyId)); } else { encryptionConfiguration.setNoEncryptionConfig(NoEncryptionConfig.NoEncryption); } s3DestinationConfiguration.setEncryptionConfiguration(encryptionConfiguration); BufferingHints bufferingHints = null; if (s3DestinationSizeInMBs != null || s3DestinationIntervalInSeconds != null) { bufferingHints = new BufferingHints(); bufferingHints.setSizeInMBs(s3DestinationSizeInMBs); bufferingHints.setIntervalInSeconds(s3DestinationIntervalInSeconds); } s3DestinationConfiguration.setBufferingHints(bufferingHints); // Create and set IAM role so that firehose service has access to the S3Buckets to put data // and KMS keys (if provided) to encrypt data. Please check the trustPolicyDocument.json and // permissionsPolicyDocument.json files for the trust and permissions policies set for the role. String iamRoleArn = createIamRole(s3ObjectPrefix); s3DestinationConfiguration.setRoleARN(iamRoleArn); createDeliveryStreamRequest.setS3DestinationConfiguration(s3DestinationConfiguration); firehoseClient.createDeliveryStream(createDeliveryStreamRequest); // The Delivery Stream is now being created. LOG.info("Creating DeliveryStream : " + deliveryStreamName); waitForDeliveryStreamToBecomeAvailable(deliveryStreamName); } } /** * Method to update s3 destination with updated s3Prefix, and buffering hints values. * * @throws Exception */ private static void updateDeliveryStream() throws Exception { DeliveryStreamDescription deliveryStreamDescription = describeDeliveryStream(deliveryStreamName); LOG.info("Updating DeliveryStream Destination: " + deliveryStreamName + " with new configuration options"); // get(0) -> DeliveryStream currently supports only one destination per DeliveryStream UpdateDestinationRequest updateDestinationRequest = new UpdateDestinationRequest() .withDeliveryStreamName(deliveryStreamName) .withCurrentDeliveryStreamVersionId(deliveryStreamDescription.getVersionId()) .withDestinationId(deliveryStreamDescription.getDestinations().get(0).getDestinationId()); S3DestinationUpdate s3DestinationUpdate = new S3DestinationUpdate(); s3DestinationUpdate.withPrefix(updateS3ObjectPrefix); BufferingHints bufferingHints = null; if (updateSizeInMBs != null || updateIntervalInSeconds != null) { bufferingHints = new BufferingHints(); bufferingHints.setSizeInMBs(updateSizeInMBs); bufferingHints.setIntervalInSeconds(updateIntervalInSeconds); } s3DestinationUpdate.setBufferingHints(bufferingHints); // Update the role policy with new s3Prefix configuration putRolePolicy(updateS3ObjectPrefix); updateDestinationRequest.setS3DestinationUpdate(s3DestinationUpdate); // Update deliveryStream destination with new configuration options such as s3Prefix and Buffering Hints. // Can also update Compression format, KMS key values and IAM Role. firehoseClient.updateDestination(updateDestinationRequest); } }