Example usage for com.amazonaws.services.kinesis AmazonKinesisClient AmazonKinesisClient

Introduction

In this page you can find the example usage for com.amazonaws.services.kinesis AmazonKinesisClient AmazonKinesisClient.

Prototype

AmazonKinesisClient(AwsSyncClientParams clientParams) 

Source Link

Document

Constructs a new client to invoke service methods on Kinesis using the specified parameters.

Usage

From source file:com.amazon.services.awsrum.utils.KinesisUtils.java

License:Open Source License

/**
 * Deletes the input stream specified by config.KINESIS_INPUT_STREAM
 * // w  w  w.j  ava  2s  .c  om
 * @param config
 *            The configuration containing the stream name and {@link AWSCredentialsProvider}
 */
public static void deleteInputStream(KinesisConnectorConfiguration config) {
    AmazonKinesisClient kinesisClient = new AmazonKinesisClient(config.AWS_CREDENTIALS_PROVIDER);
    kinesisClient.setEndpoint(config.KINESIS_ENDPOINT);
    deleteStream(kinesisClient, config.KINESIS_INPUT_STREAM);
}

---

From source file:com.amazon.services.awsrum.utils.KinesisUtils.java

License:Open Source License

/**
 * Deletes the output stream specified by config.KINESIS_OUTPUT_STREAM
 * //from w  ww  . ja v  a2 s .co m
 * @param config
 *            The configuration containing the stream name and {@link AWSCredentialsProvider}
 */
public static void deleteOutputStream(KinesisConnectorConfiguration config) {
    AmazonKinesisClient kinesisClient = new AmazonKinesisClient(config.AWS_CREDENTIALS_PROVIDER);
    kinesisClient.setEndpoint(config.KINESIS_ENDPOINT);
    deleteStream(kinesisClient, config.KINESIS_OUTPUT_STREAM);
}

---

From source file:com.boundary.aws.kinesis.Sample.java

License:Open Source License

private static void init() throws Exception {

    /*//  w ww. j  a v  a  2s  .  c  om
     * The ProfileCredentialsProvider will return your [default] credential
     * profile by reading from the credentials file located at
     * (HOME/.aws/credentials).
     */
    AWSCredentials credentials = null;
    try {
        credentials = new ProfileCredentialsProvider("default").getCredentials();
    } catch (Exception e) {
        throw new AmazonClientException("Cannot load the credentials from the credential profiles file. ", e);
    }

    kinesisClient = new AmazonKinesisClient(credentials);
    Region region = Region.getRegion(Regions.US_WEST_2);
    kinesisClient.setRegion(region);
}

---

From source file:com.datatorrent.contrib.kinesis.KinesisOperatorTestBase.java

License:Open Source License

private void createClient() {
    AWSCredentialsProvider credentials = new DefaultAWSCredentialsProviderChain();
    client = new AmazonKinesisClient(credentials);
}

---

From source file:com.datatorrent.contrib.kinesis.KinesisUtil.java

License:Open Source License

/**
 * Create the AmazonKinesisClient with the given credentials
 * @param accessKey AWS accessKeyId//from   ww  w .  j a v a 2  s  .c  om
 * @param secretKey AWS secretAccessKey
 * @throws Exception
 */
public void createKinesisClient(String accessKey, String secretKey, String endPoint) throws Exception {
    if (client == null) {
        try {
            client = new AmazonKinesisClient(new BasicAWSCredentials(accessKey, secretKey));
            if (endPoint != null) {
                client.setEndpoint(endPoint);
            }
        } catch (Exception e) {
            throw new AmazonClientException("Unable to load credentials", e);
        }
    }
}

---

From source file:com.example.base.StreamSource.java

License:Open Source License

/**
 * Creates a new StreamSource.//  ww w. j a v a  2s .  com
 * 
 * @param config
 *        Configuration to determine which stream to put records to and get {@link AWSCredentialsProvider}
 * @param inputFile
 *        File containing record data to emit on each line
 * @param loopOverStreamSource
 *        Loop over the stream source to continually put records
 */
public StreamSource(String content, KinesisConnectorConfiguration config, String inputFile,
        boolean loopOverStreamSource) {
    this.config = config;
    this.content = content;
    this.inputFile = inputFile;
    this.loopOverInputFile = loopOverStreamSource;
    this.objectMapper = new ObjectMapper();
    kinesisClient = new AmazonKinesisClient(new AWSCredentialsProvider() {
        @Override
        public AWSCredentials getCredentials() {
            return new AWSCredentials() {
                @Override
                public String getAWSAccessKeyId() {
                    return "AKIAJMOK3KHKIRPDXQBA";
                }

                @Override
                public String getAWSSecretKey() {
                    return "s2MEQ6k4acTakezkqFdoYWw909DqLwIro8ZeR9Iy";
                }
            };
        }

        @Override
        public void refresh() {

        }
    });
    kinesisClient.setRegion(RegionUtils.getRegion(config.REGION_NAME));
    if (config.KINESIS_ENDPOINT != null) {
        kinesisClient.setEndpoint(config.KINESIS_ENDPOINT);
    }
    KinesisUtils.createInputStream(config);
}

---

From source file:com.example.JavaKinesisWordCountASL.java

License:Apache License

public static void main(String[] args) {

    // Populate the appropriate variables from the given args
    String kinesisAppName = "testApp";
    String streamName = "test";
    String endpointUrl = "kinesis.us-east-1.amazonaws.com";

    // Create a Kinesis client in order to determine the number of shards for the given stream
    AmazonKinesisClient kinesisClient = new AmazonKinesisClient(
            CredentialsProvider.getAwsSessionCredentialsProvider());
    kinesisClient.setEndpoint(endpointUrl);
    int numShards = kinesisClient.describeStream(streamName).getStreamDescription().getShards().size();

    // In this com, we're going to create 1 Kinesis Receiver/input DStream for each shard.
    // This is not a necessity; if there are less receivers/DStreams than the number of shards,
    // then the shards will be automatically distributed among the receivers and each receiver
    // will receive data from multiple shards.
    int numStreams = numShards;

    // Spark Streaming batch interval
    Duration batchInterval = new Duration(2000);

    // Kinesis checkpoint interval.  Same as batchInterval for this com.
    Duration kinesisCheckpointInterval = batchInterval;

    // Get the region name from the endpoint URL to save Kinesis Client Library metadata in
    // DynamoDB of the same region as the Kinesis stream
    String regionName = RegionUtils.getRegionByEndpoint(endpointUrl).getName();

    // Setup the Spark config and StreamingContext
    SparkConf sparkConfig = new SparkConf().setAppName("JavaKinesisWordCountASL").setMaster("local[2]");
    JavaStreamingContext jssc = new JavaStreamingContext(sparkConfig, batchInterval);

    // Create the Kinesis DStreams
    List<JavaDStream<byte[]>> streamsList = new ArrayList<JavaDStream<byte[]>>(numStreams);
    for (int i = 0; i < numStreams; i++) {
        streamsList.add(KinesisUtils.createStream(jssc, kinesisAppName, streamName, endpointUrl, regionName,
                InitialPositionInStream.TRIM_HORIZON, kinesisCheckpointInterval,
                StorageLevel.MEMORY_AND_DISK_2()));
    }/*from   ww w. ja v  a 2 s .com*/

    // Union all the streams if there is more than 1 stream
    JavaDStream<byte[]> unionStreams;
    if (streamsList.size() > 1) {
        unionStreams = jssc.union(streamsList.get(0), streamsList.subList(1, streamsList.size()));
    } else {
        // Otherwise, just use the 1 stream
        unionStreams = streamsList.get(0);
    }

    // Convert each line of Array[Byte] to String, and split into words
    JavaDStream<String> words = unionStreams.flatMap(new FlatMapFunction<byte[], String>() {
        public Iterable<String> call(byte[] line) {
            return Lists.newArrayList(WORD_SEPARATOR.split(new String(line)));
        }
    });

    // Map each word to a (word, 1) tuple so we can reduce by key to count the words
    JavaPairDStream<String, Integer> wordCounts = words.mapToPair(new PairFunction<String, String, Integer>() {
        public Tuple2<String, Integer> call(String s) {
            return new Tuple2<String, Integer>(s, 1);
        }
    }).reduceByKey(new Function2<Integer, Integer, Integer>() {
        public Integer call(Integer i1, Integer i2) {
            return i1 + i2;
        }
    });

    // Print the first 10 wordCounts
    wordCounts.print();

    // Start the streaming context and await termination
    jssc.start();
    jssc.awaitTermination();
}

---

From source file:com.example.utils.KinesisUtils.java

License:Open Source License

/**
 * Creates the Amazon Kinesis stream specified by config.KINESIS_INPUT_STREAM
 * /*  www  .  j  a  v a2 s . com*/
 * @param config
 *        The configuration with the specified input stream name and {@link AWSCredentialsProvider}
 * @param shardCount
 *        The shard count to create the stream with
 */
public static void createInputStream(KinesisConnectorConfiguration config) {
    AmazonKinesisClient kinesisClient = new AmazonKinesisClient(config.AWS_CREDENTIALS_PROVIDER);
    kinesisClient.setRegion(RegionUtils.getRegion(config.REGION_NAME));
    if (config.KINESIS_ENDPOINT != null) {
        kinesisClient.setEndpoint(config.KINESIS_ENDPOINT);
    }
    createAndWaitForStreamToBecomeAvailable(kinesisClient, config.KINESIS_INPUT_STREAM,
            config.KINESIS_INPUT_STREAM_SHARD_COUNT);
}

---

From source file:com.example.utils.KinesisUtils.java

License:Open Source License

/**
 * Creates the Amazon Kinesis stream specified by config.KINESIS_OUTPUT_STREAM.
 * /*from w  w w .  ja  v a  2 s  . c om*/
 * @param config
 *        The configuration with the specified output stream name and {@link AWSCredentialsProvider}
 * @param shardCount
 *        The shard count to create the stream with
 */
public static void createOutputStream(KinesisConnectorConfiguration config) {
    AmazonKinesisClient kinesisClient = new AmazonKinesisClient(config.AWS_CREDENTIALS_PROVIDER);
    kinesisClient.setRegion(RegionUtils.getRegion(config.REGION_NAME));
    if (config.KINESIS_ENDPOINT != null) {
        kinesisClient.setEndpoint(config.KINESIS_ENDPOINT);
    }
    createAndWaitForStreamToBecomeAvailable(kinesisClient, config.KINESIS_OUTPUT_STREAM,
            config.KINESIS_OUTPUT_STREAM_SHARD_COUNT);
}

---

From source file:com.example.utils.KinesisUtils.java

License:Open Source License

/**
 * Deletes the input stream specified by config.KINESIS_INPUT_STREAM
 * //  w  w w.jav a 2s  .  c  o  m
 * @param config
 *        The configuration containing the stream name and {@link AWSCredentialsProvider}
 */
public static void deleteInputStream(KinesisConnectorConfiguration config) {
    AmazonKinesisClient kinesisClient = new AmazonKinesisClient(config.AWS_CREDENTIALS_PROVIDER);
    kinesisClient.setRegion(RegionUtils.getRegion(config.REGION_NAME));
    if (config.KINESIS_ENDPOINT != null) {
        kinesisClient.setEndpoint(config.KINESIS_ENDPOINT);
    }
    deleteStream(kinesisClient, config.KINESIS_INPUT_STREAM);
}

---

• «
• 1
• 2
• 3
• 4
• »