List of usage examples for com.amazonaws.services.kinesis AmazonKinesisClient AmazonKinesisClient
AmazonKinesisClient(AwsSyncClientParams clientParams)
From source file:edu.hawaii.kscmfeedprocessor.AmazonKinesisApplicationSample.java
License:Open Source License
public void deleteResources() { AWSCredentials credentials = credentialsProvider.getCredentials(); // Delete the stream AmazonKinesis kinesis = new AmazonKinesisClient(credentials); System.out.printf("Deleting the Amazon Kinesis stream used by the sample. Stream Name = %s.\n", SAMPLE_APPLICATION_STREAM_NAME); try {/* w w w . ja v a 2 s . c o m*/ kinesis.deleteStream(SAMPLE_APPLICATION_STREAM_NAME); } catch (ResourceNotFoundException ex) { // The stream doesn't exist. } // Delete the table AmazonDynamoDBClient dynamoDB = new AmazonDynamoDBClient(credentialsProvider.getCredentials()); System.out.printf( "Deleting the Amazon DynamoDB table used by the Amazon Kinesis Client Library. Table Name = %s.\n", SAMPLE_APPLICATION_NAME); try { dynamoDB.deleteTable(SAMPLE_APPLICATION_NAME); } catch (com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException ex) { // The table doesn't exist. } }
From source file:org.apache.apex.malhar.contrib.kinesis.KinesisOperatorTestBase.java
License:Apache License
private void createClient() { credentials = new DefaultAWSCredentialsProviderChain(); client = new AmazonKinesisClient(credentials); }
From source file:org.apache.beam.sdk.io.kinesis.KinesisUploader.java
License:Apache License
public static void uploadAll(List<String> data, KinesisTestOptions options) { AmazonKinesisClient client = new AmazonKinesisClient(new StaticCredentialsProvider( new BasicAWSCredentials(options.getAwsAccessKey(), options.getAwsSecretKey()))) .withRegion(Regions.fromName(options.getAwsKinesisRegion())); List<List<String>> partitions = Lists.partition(data, MAX_NUMBER_OF_RECORDS_IN_BATCH); for (List<String> partition : partitions) { List<PutRecordsRequestEntry> allRecords = newArrayList(); for (String row : partition) { allRecords.add(new PutRecordsRequestEntry().withData(ByteBuffer.wrap(row.getBytes(Charsets.UTF_8))) .withPartitionKey(Integer.toString(row.hashCode())) );//from w w w.jav a 2 s .c o m } PutRecordsResult result; do { result = client.putRecords(new PutRecordsRequest().withStreamName(options.getAwsKinesisStream()) .withRecords(allRecords)); List<PutRecordsRequestEntry> failedRecords = newArrayList(); int i = 0; for (PutRecordsResultEntry row : result.getRecords()) { if (row.getErrorCode() != null) { failedRecords.add(allRecords.get(i)); } ++i; } allRecords = failedRecords; } while (result.getFailedRecordCount() > 0); } }
From source file:org.apache.beam.sdk.io.kinesis.TalendKinesisProvider.java
License:Open Source License
@Override public AmazonKinesis get() { AWSCredentialsProviderChain credentials = null; if (specifyCredentials) { credentials = new AWSCredentialsProviderChain(new BasicAWSCredentialsProvider(accessKey, secretKey), new DefaultAWSCredentialsProviderChain(), new AnonymousAWSCredentialsProvider()); } else {//from w ww . j ava2 s. c om // do not be polluted by hidden accessKey/secretKey credentials = new AWSCredentialsProviderChain(new DefaultAWSCredentialsProviderChain(), new AnonymousAWSCredentialsProvider()); } if (specifySTS) { STSAssumeRoleSessionCredentialsProvider.Builder builder = new STSAssumeRoleSessionCredentialsProvider.Builder( roleArn, roleSessionName).withLongLivedCredentialsProvider(credentials); if (specifyRoleExternalId) { builder = builder.withExternalId(roleExternalId); } if (specifySTSEndpoint) { builder = builder.withServiceEndpoint(stsEndpoint); } credentials = new AWSCredentialsProviderChain(builder.build()); } AmazonKinesisClient client = new AmazonKinesisClient(credentials); client.withRegion(region); if (specifyEndpoint) { client.setEndpoint(endpoint); } return client; }
From source file:org.apache.spark.examples.streaming.JavaKinesisWordCountASL.java
License:Apache License
public static void main(String[] args) throws Exception { // Check that all required args were passed in. if (args.length != 3) { System.err.println("Usage: JavaKinesisWordCountASL <stream-name> <endpoint-url>\n\n" + " <app-name> is the name of the app, used to track the read data in DynamoDB\n" + " <stream-name> is the name of the Kinesis stream\n" + " <endpoint-url> is the endpoint of the Kinesis service\n" + " (e.g. https://kinesis.us-east-1.amazonaws.com)\n" + "Generate data for the Kinesis stream using the example KinesisWordProducerASL.\n" + "See http://spark.apache.org/docs/latest/streaming-kinesis-integration.html for more\n" + "details.\n"); System.exit(1);//from ww w . ja va 2s . c om } // Set default log4j logging level to WARN to hide Spark logs StreamingExamples.setStreamingLogLevels(); // Populate the appropriate variables from the given args String kinesisAppName = args[0]; String streamName = args[1]; String endpointUrl = args[2]; // Create a Kinesis client in order to determine the number of shards for the given stream AmazonKinesisClient kinesisClient = new AmazonKinesisClient(new DefaultAWSCredentialsProviderChain()); kinesisClient.setEndpoint(endpointUrl); int numShards = kinesisClient.describeStream(streamName).getStreamDescription().getShards().size(); // In this example, we're going to create 1 Kinesis Receiver/input DStream for each shard. // This is not a necessity; if there are less receivers/DStreams than the number of shards, // then the shards will be automatically distributed among the receivers and each receiver // will receive data from multiple shards. int numStreams = numShards; // Spark Streaming batch interval Duration batchInterval = new Duration(2000); // Kinesis checkpoint interval. Same as batchInterval for this example. Duration kinesisCheckpointInterval = batchInterval; // Get the region name from the endpoint URL to save Kinesis Client Library metadata in // DynamoDB of the same region as the Kinesis stream String regionName = KinesisExampleUtils.getRegionNameByEndpoint(endpointUrl); // Setup the Spark config and StreamingContext SparkConf sparkConfig = new SparkConf().setAppName("JavaKinesisWordCountASL"); JavaStreamingContext jssc = new JavaStreamingContext(sparkConfig, batchInterval); // Create the Kinesis DStreams List<JavaDStream<byte[]>> streamsList = new ArrayList<>(numStreams); for (int i = 0; i < numStreams; i++) { streamsList.add(KinesisUtils.createStream(jssc, kinesisAppName, streamName, endpointUrl, regionName, InitialPositionInStream.LATEST, kinesisCheckpointInterval, StorageLevel.MEMORY_AND_DISK_2())); } // Union all the streams if there is more than 1 stream JavaDStream<byte[]> unionStreams; if (streamsList.size() > 1) { unionStreams = jssc.union(streamsList.get(0), streamsList.subList(1, streamsList.size())); } else { // Otherwise, just use the 1 stream unionStreams = streamsList.get(0); } // Convert each line of Array[Byte] to String, and split into words JavaDStream<String> words = unionStreams.flatMap(new FlatMapFunction<byte[], String>() { @Override public Iterator<String> call(byte[] line) { String s = new String(line, StandardCharsets.UTF_8); return Arrays.asList(WORD_SEPARATOR.split(s)).iterator(); } }); // Map each word to a (word, 1) tuple so we can reduce by key to count the words JavaPairDStream<String, Integer> wordCounts = words.mapToPair(new PairFunction<String, String, Integer>() { @Override public Tuple2<String, Integer> call(String s) { return new Tuple2<>(s, 1); } }).reduceByKey(new Function2<Integer, Integer, Integer>() { @Override public Integer call(Integer i1, Integer i2) { return i1 + i2; } }); // Print the first 10 wordCounts wordCounts.print(); // Start the streaming context and await termination jssc.start(); jssc.awaitTermination(); }
From source file:org.selman.tweetamo.TweetamoClient.java
License:Apache License
public static void main(String[] args) throws Exception { if (args.length != 2) { System.out.println("Usage: [language] [search topic]"); }/* ww w .j av a 2s .co m*/ kinesisClient = new AmazonKinesisClient(new ClasspathPropertiesFileCredentialsProvider()); waitForStreamToBecomeAvailable(STREAM_NAME); LOG.info("Publishing tweets to stream : " + STREAM_NAME); StatusListener listener = new StatusListener() { public void onStatus(Status status) { try { PutRecordRequest putRecordRequest = new PutRecordRequest(); putRecordRequest.setStreamName(STREAM_NAME); putRecordRequest.setData(TweetSerializer.toBytes(status)); putRecordRequest.setPartitionKey(status.getUser().getScreenName()); PutRecordResult putRecordResult = kinesisClient.putRecord(putRecordRequest); LOG.info("Successfully putrecord, partition key : " + putRecordRequest.getPartitionKey() + ", ShardID : " + putRecordResult.getShardId()); } catch (Exception e) { LOG.error("Failed to putrecord", e); } } public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) { } public void onTrackLimitationNotice(int numberOfLimitedStatuses) { } public void onException(Exception ex) { ex.printStackTrace(); } @Override public void onScrubGeo(long arg0, long arg1) { } @Override public void onStallWarning(StallWarning arg0) { } }; ClasspathTwitterCredentialsProvider provider = new ClasspathTwitterCredentialsProvider(); TwitterCredentials credentials = provider.getTwitterCredentials(); ConfigurationBuilder cb = new ConfigurationBuilder(); cb.setDebugEnabled(true).setOAuthConsumerKey(credentials.getConsumerKey()) .setOAuthConsumerSecret(credentials.getConsumerSecret()) .setOAuthAccessToken(credentials.getAccessToken()) .setOAuthAccessTokenSecret(credentials.getAccessTokenSecret()); TwitterStream twitterStream = new TwitterStreamFactory(cb.build()).getInstance(); twitterStream.addListener(listener); FilterQuery filterQuery = new FilterQuery(); filterQuery.language(new String[] { args[0] }); filterQuery.track(new String[] { args[1] }); twitterStream.filter(filterQuery); }
From source file:producer.SensorReadingProducer.java
License:Open Source License
private void run(final int events, final OutputFormat format, final String streamName, final String region) throws Exception { AmazonKinesis kinesisClient = new AmazonKinesisClient(new DefaultAWSCredentialsProviderChain()); kinesisClient.setRegion(Region.getRegion(Regions.fromName(region))); int count = 0; SensorReading r = null;//w w w . j a v a 2 s .c o m do { r = nextSensorReading(format); try { PutRecordRequest req = new PutRecordRequest().withPartitionKey("" + rand.nextLong()) .withStreamName(streamName).withData(ByteBuffer.wrap(r.toString().getBytes())); kinesisClient.putRecord(req); } catch (ProvisionedThroughputExceededException e) { Thread.sleep(BACKOFF); } System.out.println(r); count++; } while (count < events); }
From source file:utils.kinesis.OldStyleKinesisUploader.java
License:Apache License
public OldStyleKinesisUploader() { client = new AmazonKinesisClient(TestUtils.getTestAwsCredentialsProvider()) .withRegion(Regions.fromName(TestConfiguration.get().getTestRegion())); }