com.datatorrent.contrib.kinesis.KinesisUtil.java Source code

Introduction

Here is the source code for com.datatorrent.contrib.kinesis.KinesisUtil.java

Source

/*
 * Copyright (c) 2013 DataTorrent, Inc. ALL Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.datatorrent.contrib.kinesis;

import com.amazonaws.AmazonClientException;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.kinesis.AmazonKinesisClient;
import com.amazonaws.services.kinesis.model.*;

import java.util.List;

/**
 * A util class for Amazon Kinesis. Contains the wrappers for creating client, get the shard list and records from shard.
 *
 * @since 2.0.0
 */
public class KinesisUtil {

    private transient AmazonKinesisClient client = null;

    private static KinesisUtil instance = null;

    private KinesisUtil() {
    }

    public static KinesisUtil getInstance() {
        if (instance == null) {
            instance = new KinesisUtil();
        }
        return instance;
    }

    /**
     * Create the AmazonKinesisClient with the given credentials
     * @param accessKey AWS accessKeyId
     * @param secretKey AWS secretAccessKey
     * @throws Exception
     */
    public void createKinesisClient(String accessKey, String secretKey, String endPoint) throws Exception {
        if (client == null) {
            try {
                client = new AmazonKinesisClient(new BasicAWSCredentials(accessKey, secretKey));
                if (endPoint != null) {
                    client.setEndpoint(endPoint);
                }
            } catch (Exception e) {
                throw new AmazonClientException("Unable to load credentials", e);
            }
        }
    }

    /**
     * Get the available shards from the kinesis
     * @param streamName Name of the stream from where the shards to be accessed
     * @return the list of shards from the given stream
     */
    public List<Shard> getShardList(String streamName) {
        assert client != null : "Illegal client";
        DescribeStreamRequest describeRequest = new DescribeStreamRequest();
        describeRequest.setStreamName(streamName);

        DescribeStreamResult describeResponse = client.describeStream(describeRequest);
        return describeResponse.getStreamDescription().getShards();
    }

    /**
     * Get the records from the particular shard
     * @param streamName Name of the stream from where the records to be accessed
     * @param recordsLimit Number of records to return from shard
     * @param shId Shard Id of the shard
     * @param iteratorType Shard iterator type
     * @param seqNo Record sequence number
     * @return the list of records from the given shard
     * @throws AmazonClientException
     */
    public List<Record> getRecords(String streamName, Integer recordsLimit, Shard shId,
            ShardIteratorType iteratorType, String seqNo) throws AmazonClientException {
        assert client != null : "Illegal client";
        try {
            // Create the GetShardIteratorRequest instance and sets streamName, shardId and iteratorType to it
            GetShardIteratorRequest iteratorRequest = new GetShardIteratorRequest();
            iteratorRequest.setStreamName(streamName);
            iteratorRequest.setShardId(shId.getShardId());
            iteratorRequest.setShardIteratorType(iteratorType);

            // If the iteratorType is AFTER_SEQUENCE_NUMBER, set the sequence No to the iteratorRequest
            if (ShardIteratorType.AFTER_SEQUENCE_NUMBER.equals(iteratorType))
                iteratorRequest.setStartingSequenceNumber(seqNo);

            // Get the Response from the getShardIterator service method & get the shardIterator from that response
            GetShardIteratorResult iteratorResponse = client.getShardIterator(iteratorRequest);
            // getShardIterator() specifies the position in the shard
            String iterator = iteratorResponse.getShardIterator();

            // Create the GetRecordsRequest instance and set the recordsLimit and iterator
            GetRecordsRequest getRequest = new GetRecordsRequest();
            getRequest.setLimit(recordsLimit);
            getRequest.setShardIterator(iterator);

            // Get the Response from the getRecords service method and get the data records from that response.
            GetRecordsResult getResponse = client.getRecords(getRequest);
            return getResponse.getRecords();
        } catch (AmazonClientException e) {
            throw new RuntimeException(e);
        }
    }

    public AmazonKinesisClient getClient() {
        return client;
    }

    public void setClient(AmazonKinesisClient client) {
        this.client = client;
    }
}