List of usage examples for com.amazonaws ClientConfiguration ClientConfiguration
public ClientConfiguration()
From source file:com.alertlogic.aws.analytics.poc.DeleteResources.java
License:Open Source License
public static void main(String[] args) { if (args.length != 4) { System.err.println("Usage: " + DeleteResources.class.getSimpleName() + " <application name> <stream name> <DynamoDB table name> <region>"); System.exit(1);// ww w. j a v a 2 s . co m } String applicationName = args[0]; String streamName = args[1]; String countsTableName = args[2]; Region region = Utils.parseRegion(args[3]); AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain(); ClientConfiguration clientConfig = Utils.configureUserAgentForSample(new ClientConfiguration()); AmazonKinesis kinesis = new AmazonKinesisClient(credentialsProvider, clientConfig); kinesis.setRegion(region); AmazonDynamoDB dynamoDB = new AmazonDynamoDBClient(credentialsProvider, clientConfig); dynamoDB.setRegion(region); StreamUtils streamUtils = new StreamUtils(kinesis); DynamoDBUtils dynamoDBUtils = new DynamoDBUtils(dynamoDB); LOG.info("Removing Amazon Kinesis and DynamoDB resources used by the sample application..."); streamUtils.deleteStream(streamName); // The Kinesis Client Library creates a table to manage shard leases and uses the application name for its name. dynamoDBUtils.deleteTable(applicationName); dynamoDBUtils.deleteTable(countsTableName); }
From source file:com.alertlogic.aws.kinesis.test1.StreamProcessor.java
License:Open Source License
/** * Start the Kinesis Client application. * //from w ww . j a v a 2s .c o m * @param args Expecting 4 arguments: Application name to use for the Kinesis Client Application, Stream name to * read from, DynamoDB table name to persist counts into, and the AWS region in which these resources * exist or should be created. */ public static void main(String[] args) throws UnknownHostException { if (args.length != 4) { System.err.println("Usage: " + StreamProcessor.class.getSimpleName() + " <application name> <stream name> <DynamoDB table name> <region>"); System.exit(1); } String applicationName = args[0]; String streamName = args[1]; String countsTableName = args[2]; Region region = SampleUtils.parseRegion(args[3]); AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain(); ClientConfiguration clientConfig = SampleUtils.configureUserAgentForSample(new ClientConfiguration()); AmazonKinesis kinesis = new AmazonKinesisClient(credentialsProvider, clientConfig); kinesis.setRegion(region); AmazonDynamoDB dynamoDB = new AmazonDynamoDBClient(credentialsProvider, clientConfig); dynamoDB.setRegion(region); // Creates a stream to write to, if it doesn't exist StreamUtils streamUtils = new StreamUtils(kinesis); streamUtils.createStreamIfNotExists(streamName, 2); LOG.info(String.format("%s stream is ready for use", streamName)); DynamoDBUtils dynamoDBUtils = new DynamoDBUtils(dynamoDB); dynamoDBUtils.createCountTableIfNotExists(countsTableName); LOG.info(String.format("%s DynamoDB table is ready for use", countsTableName)); String workerId = String.valueOf(UUID.randomUUID()); LOG.info(String.format("Using working id: %s", workerId)); KinesisClientLibConfiguration kclConfig = new KinesisClientLibConfiguration(applicationName, streamName, credentialsProvider, workerId); kclConfig.withCommonClientConfig(clientConfig); kclConfig.withRegionName(region.getName()); kclConfig.withInitialPositionInStream(InitialPositionInStream.LATEST); // Persist counts to DynamoDB DynamoDBCountPersister persister = new DynamoDBCountPersister( dynamoDBUtils.createMapperForTable(countsTableName)); IRecordProcessorFactory recordProcessor = new CountingRecordProcessorFactory<HttpReferrerPair>( HttpReferrerPair.class, persister, COMPUTE_RANGE_FOR_COUNTS_IN_MILLIS, COMPUTE_INTERVAL_IN_MILLIS); Worker worker = new Worker(recordProcessor, kclConfig); int exitCode = 0; try { worker.run(); } catch (Throwable t) { LOG.error("Caught throwable while processing data.", t); exitCode = 1; } System.exit(exitCode); }
From source file:com.alertlogic.aws.kinesis.test1.StreamWriter.java
License:Open Source License
/** * Start a number of threads and send randomly generated {@link HttpReferrerPair}s to a Kinesis Stream until the * program is terminated./* w w w . j a v a2s . co m*/ * * @param args Expecting 3 arguments: A numeric value indicating the number of threads to use to send * data to Kinesis and the name of the stream to send records to, and the AWS region in which these resources * exist or should be created. * @throws InterruptedException If this application is interrupted while sending records to Kinesis. */ public static void main(String[] args) throws InterruptedException { if (args.length != 3) { System.err.println( "Usage: " + StreamWriter.class.getSimpleName() + " <number of threads> <stream name> <region>"); System.exit(1); } int numberOfThreads = Integer.parseInt(args[0]); String streamName = args[1]; Region region = SampleUtils.parseRegion(args[2]); AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain(); ClientConfiguration clientConfig = SampleUtils.configureUserAgentForSample(new ClientConfiguration()); AmazonKinesis kinesis = new AmazonKinesisClient(credentialsProvider, clientConfig); kinesis.setRegion(region); // The more resources we declare the higher write IOPS we need on our DynamoDB table. // We write a record for each resource every interval. // If interval = 500ms, resource count = 7 we need: (1000/500 * 7) = 14 write IOPS minimum. List<String> resources = new ArrayList<>(); resources.add("/index.html"); // These are the possible referrers to use when generating pairs List<String> referrers = new ArrayList<>(); referrers.add("http://www.amazon.com"); referrers.add("http://www.google.com"); referrers.add("http://www.yahoo.com"); referrers.add("http://www.bing.com"); referrers.add("http://www.stackoverflow.com"); referrers.add("http://www.reddit.com"); HttpReferrerPairFactory pairFactory = new HttpReferrerPairFactory(resources, referrers); // Creates a stream to write to with 2 shards if it doesn't exist StreamUtils streamUtils = new StreamUtils(kinesis); streamUtils.createStreamIfNotExists(streamName, 2); LOG.info(String.format("%s stream is ready for use", streamName)); final HttpReferrerKinesisPutter putter = new HttpReferrerKinesisPutter(pairFactory, kinesis, streamName); ExecutorService es = Executors.newCachedThreadPool(); Runnable pairSender = new Runnable() { @Override public void run() { try { putter.sendPairsIndefinitely(DELAY_BETWEEN_RECORDS_IN_MILLIS, TimeUnit.MILLISECONDS); } catch (Exception ex) { LOG.warn( "Thread encountered an error while sending records. Records will no longer be put by this thread.", ex); } } }; for (int i = 0; i < numberOfThreads; i++) { es.submit(pairSender); } LOG.info(String.format("Sending pairs with a %dms delay between records with %d thread(s).", DELAY_BETWEEN_RECORDS_IN_MILLIS, numberOfThreads)); es.shutdown(); es.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS); }
From source file:com.alertlogic.aws.kinesis.test1.utils.DeleteSampleResources.java
License:Open Source License
public static void main(String[] args) { if (args.length != 4) { System.err.println("Usage: " + DeleteSampleResources.class.getSimpleName() + " <application name> <stream name> <DynamoDB table name> <region>"); System.exit(1);//from w w w .j a v a2s .c o m } String applicationName = args[0]; String streamName = args[1]; String countsTableName = args[2]; Region region = SampleUtils.parseRegion(args[3]); AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain(); ClientConfiguration clientConfig = SampleUtils.configureUserAgentForSample(new ClientConfiguration()); AmazonKinesis kinesis = new AmazonKinesisClient(credentialsProvider, clientConfig); kinesis.setRegion(region); AmazonDynamoDB dynamoDB = new AmazonDynamoDBClient(credentialsProvider, clientConfig); dynamoDB.setRegion(region); StreamUtils streamUtils = new StreamUtils(kinesis); DynamoDBUtils dynamoDBUtils = new DynamoDBUtils(dynamoDB); LOG.info("Removing Amazon Kinesis and DynamoDB resources used by the sample application..."); streamUtils.deleteStream(streamName); // The Kinesis Client Library creates a table to manage shard leases and uses the application name for its name. dynamoDBUtils.deleteTable(applicationName); dynamoDBUtils.deleteTable(countsTableName); }
From source file:com.alertlogic.aws.kinesis.test1.WebServer.java
License:Open Source License
/** * Start an embedded web server.//from ww w .ja va 2 s . co m * * @param args Expecting 4 arguments: Port number, File path to static content, the name of the * DynamoDB table where counts are persisted to, and the AWS region in which these resources * exist or should be created. * @throws Exception Error starting the web server. */ public static void main(String[] args) throws Exception { if (args.length != 4) { System.err.println("Usage: " + WebServer.class + " <port number> <directory for static content> <DynamoDB table name> <region>"); System.exit(1); } Server server = new Server(Integer.parseInt(args[0])); String wwwroot = args[1]; String countsTableName = args[2]; Region region = SampleUtils.parseRegion(args[3]); // Servlet context ServletContextHandler context = new ServletContextHandler( ServletContextHandler.NO_SESSIONS | ServletContextHandler.NO_SECURITY); context.setContextPath("/api"); // Static resource context ResourceHandler resources = new ResourceHandler(); resources.setDirectoriesListed(false); resources.setWelcomeFiles(new String[] { "graph.html" }); resources.setResourceBase(wwwroot); // Create the servlet to handle /GetCounts AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain(); ClientConfiguration clientConfig = SampleUtils.configureUserAgentForSample(new ClientConfiguration()); AmazonDynamoDB dynamoDB = new AmazonDynamoDBClient(credentialsProvider, clientConfig); dynamoDB.setRegion(region); DynamoDBUtils dynamoDBUtils = new DynamoDBUtils(dynamoDB); context.addServlet( new ServletHolder(new GetCountsServlet(dynamoDBUtils.createMapperForTable(countsTableName))), "/GetCounts/*"); HandlerList handlers = new HandlerList(); handlers.addHandler(context); handlers.addHandler(resources); handlers.addHandler(new DefaultHandler()); server.setHandler(handlers); server.start(); server.join(); }
From source file:com.amazon.janusgraph.diskstorage.dynamodb.Client.java
License:Open Source License
public Client(final Configuration config) { final String credentialsClassName = config.get(Constants.DYNAMODB_CREDENTIALS_CLASS_NAME); final Class<?> clazz; try {// ww w. j a v a 2 s .com clazz = Class.forName(credentialsClassName); } catch (ClassNotFoundException e) { throw new IllegalArgumentException(VALIDATE_CREDENTIALS_CLASS_NAME, e); } final String[] credentialsConstructorArgsValues = config .get(Constants.DYNAMODB_CREDENTIALS_CONSTRUCTOR_ARGS); final List<String> filteredArgList = new ArrayList<>(); for (Object obj : credentialsConstructorArgsValues) { final String str = obj.toString(); if (!str.isEmpty()) { filteredArgList.add(str); } } final AWSCredentialsProvider credentialsProvider; if (AWSCredentials.class.isAssignableFrom(clazz)) { final AWSCredentials credentials = createCredentials(clazz, filteredArgList.toArray(new String[filteredArgList.size()])); credentialsProvider = new AWSStaticCredentialsProvider(credentials); } else if (AWSCredentialsProvider.class.isAssignableFrom(clazz)) { credentialsProvider = createCredentialsProvider(clazz, credentialsConstructorArgsValues); } else { throw new IllegalArgumentException(VALIDATE_CREDENTIALS_CLASS_NAME); } //begin adaptation of constructor at //https://github.com/buka/titan/blob/master/src/main/java/com/thinkaurelius/titan/diskstorage/dynamodb/DynamoDBClient.java#L77 final ClientConfiguration clientConfig = new ClientConfiguration(); clientConfig.withConnectionTimeout(config.get(Constants.DYNAMODB_CLIENT_CONN_TIMEOUT)) .withConnectionTTL(config.get(Constants.DYNAMODB_CLIENT_CONN_TTL)) .withMaxConnections(config.get(Constants.DYNAMODB_CLIENT_MAX_CONN)) .withMaxErrorRetry(config.get(Constants.DYNAMODB_CLIENT_MAX_ERROR_RETRY)) .withGzip(config.get(Constants.DYNAMODB_CLIENT_USE_GZIP)) .withReaper(config.get(Constants.DYNAMODB_CLIENT_USE_REAPER)) .withUserAgentSuffix(config.get(Constants.DYNAMODB_CLIENT_USER_AGENT)) .withSocketTimeout(config.get(Constants.DYNAMODB_CLIENT_SOCKET_TIMEOUT)) .withSocketBufferSizeHints(config.get(Constants.DYNAMODB_CLIENT_SOCKET_BUFFER_SEND_HINT), config.get(Constants.DYNAMODB_CLIENT_SOCKET_BUFFER_RECV_HINT)) .withProxyDomain(config.get(Constants.DYNAMODB_CLIENT_PROXY_DOMAIN)) .withProxyWorkstation(config.get(Constants.DYNAMODB_CLIENT_PROXY_WORKSTATION)) .withProxyHost(config.get(Constants.DYNAMODB_CLIENT_PROXY_HOST)) .withProxyPort(config.get(Constants.DYNAMODB_CLIENT_PROXY_PORT)) .withProxyUsername(config.get(Constants.DYNAMODB_CLIENT_PROXY_USERNAME)) .withProxyPassword(config.get(Constants.DYNAMODB_CLIENT_PROXY_PASSWORD)); forceConsistentRead = config.get(Constants.DYNAMODB_FORCE_CONSISTENT_READ); //end adaptation of constructor at //https://github.com/buka/titan/blob/master/src/main/java/com/thinkaurelius/titan/diskstorage/dynamodb/DynamoDBClient.java#L77 enableParallelScan = config.get(Constants.DYNAMODB_ENABLE_PARALLEL_SCAN); prefix = config.get(Constants.DYNAMODB_TABLE_PREFIX); final String metricsPrefix = config.get(Constants.DYNAMODB_METRICS_PREFIX); final long maxRetries = config.get(Constants.DYNAMODB_MAX_SELF_THROTTLED_RETRIES); Preconditions.checkArgument(maxRetries >= 0, Constants.DYNAMODB_MAX_SELF_THROTTLED_RETRIES.getName() + " must be at least 0"); final long retryMillis = config.get(Constants.DYNAMODB_INITIAL_RETRY_MILLIS); Preconditions.checkArgument(retryMillis > 0, Constants.DYNAMODB_INITIAL_RETRY_MILLIS.getName() + " must be at least 1"); final double controlPlaneRate = config.get(Constants.DYNAMODB_CONTROL_PLANE_RATE); Preconditions.checkArgument(controlPlaneRate >= 0, "must have a positive control plane rate"); final RateLimiter controlPlaneRateLimiter = RateLimiter.create(controlPlaneRate); final Map<String, RateLimiter> readRateLimit = new HashMap<>(); final Map<String, RateLimiter> writeRateLimit = new HashMap<>(); final Set<String> storeNames = new HashSet<>(Constants.REQUIRED_BACKEND_STORES); storeNames.add(config.get(GraphDatabaseConfiguration.IDS_STORE_NAME)); storeNames.addAll(config.getContainedNamespaces(Constants.DYNAMODB_STORES_NAMESPACE)); storeNames.forEach(storeName -> setupStore(config, readRateLimit, writeRateLimit, storeName)); delegate = new DynamoDbDelegate( JanusGraphConfigUtil.getNullableConfigValue(config, Constants.DYNAMODB_CLIENT_ENDPOINT), JanusGraphConfigUtil.getNullableConfigValue(config, Constants.DYNAMODB_CLIENT_SIGNING_REGION), credentialsProvider, clientConfig, config, readRateLimit, writeRateLimit, maxRetries, retryMillis, prefix, metricsPrefix, controlPlaneRateLimiter); }
From source file:com.amazon.kinesis.streaming.agent.AgentContext.java
License:Open Source License
public ClientConfiguration getAwsClientConfiguration() { ClientConfiguration config = new ClientConfiguration(); config.setUserAgent(userAgent(config)); config.setMaxConnections(maxConnections()); config.setConnectionTimeout(connectionTimeoutMillis()); config.setSocketTimeout(socketTimeoutMillis()); config.setUseTcpKeepAlive(useTcpKeepAlive()); config.setConnectionTTL(connectionTTLMillis()); config.setUseGzip(useHttpGzip());//from w ww . j a va 2s . co m return config; }
From source file:com.appdynamics.extensions.cloudwatch.configuration.Configuration.java
License:Apache License
public Configuration() { disabledMetrics = Collections.synchronizedMap(new HashMap<String, Set<String>>()); metricTypes = Collections.synchronizedMap(new HashMap<String, Map<String, MetricType>>()); availableNamespaces = new HashSet<String>(); availableRegions = new HashSet<String>(); clientConfiguration = new ClientConfiguration(); proxyParams = new HashMap<String, String>(); }
From source file:com.AWSManagementAntTask.dao.RDSDao.java
public RDSDao(AWSMATDto dto) { AWSCredentialsProvider provider = new ProfileCredentialsProvider(dto.getProfilename()); rds = Region.getRegion(dto.getRegionid()).createClient(AmazonRDSClient.class, provider, new ClientConfiguration()); this.dto = dto; }
From source file:com.bigstep.S3Sampler.java
License:Apache License
@Override public SampleResult runTest(JavaSamplerContext context) { // pull parameters String bucket = context.getParameter("bucket"); String object = context.getParameter("object"); String method = context.getParameter("method"); String local_file_path = context.getParameter("local_file_path"); String key_id = context.getParameter("key_id"); String secret_key = context.getParameter("secret_key"); String proxy_host = context.getParameter("proxy_host"); String proxy_port = context.getParameter("proxy_port"); String endpoint = context.getParameter("endpoint"); log.debug("runTest:method=" + method + " local_file_path=" + local_file_path + " bucket=" + bucket + " object=" + object); SampleResult result = new SampleResult(); result.sampleStart(); // start stopwatch try {//ww w.j a va 2 s . c o m ClientConfiguration config = new ClientConfiguration(); if (proxy_host != null && !proxy_host.isEmpty()) { config.setProxyHost(proxy_host); } if (proxy_port != null && !proxy_port.isEmpty()) { config.setProxyPort(Integer.parseInt(proxy_port)); } //config.setProtocol(Protocol.HTTP); AWSCredentials credentials = new BasicAWSCredentials(key_id, secret_key); AmazonS3 s3Client = new AmazonS3Client(credentials, config); if (endpoint != null && !endpoint.isEmpty()) { s3Client.setEndpoint(endpoint); } ObjectMetadata meta = null; if (method.equals("GET")) { File file = new File(local_file_path); //meta= s3Client.getObject(new GetObjectRequest(bucket, object), file); S3Object s3object = s3Client.getObject(bucket, object); S3ObjectInputStream stream = s3object.getObjectContent(); //while(stream.skip(1024*1024)>0); stream.close(); } else if (method.equals("PUT")) { File file = new File(local_file_path); s3Client.putObject(bucket, object, file); } result.sampleEnd(); // stop stopwatch result.setSuccessful(true); if (meta != null) { result.setResponseMessage( "OK on url:" + bucket + "/" + object + ". Length=" + meta.getContentLength()); } else { result.setResponseMessage("OK on url:" + bucket + "/" + object + ".No metadata"); } result.setResponseCodeOK(); // 200 code } catch (Exception e) { result.sampleEnd(); // stop stopwatch result.setSuccessful(false); result.setResponseMessage("Exception: " + e); // get stack trace as a String to return as document data java.io.StringWriter stringWriter = new java.io.StringWriter(); e.printStackTrace(new java.io.PrintWriter(stringWriter)); result.setResponseData(stringWriter.toString()); result.setDataType(org.apache.jmeter.samplers.SampleResult.TEXT); result.setResponseCode("500"); } return result; }