List of usage examples for com.amazonaws AmazonServiceException getStatusCode
public int getStatusCode()
From source file:getting_started.InlineGettingStartedCodeSampleApp.java
License:Open Source License
/** * @param args//from w w w. j a v a 2s. c om */ public static void main(String[] args) { //============================================================================================// //=============================== Submitting a Request =======================================// //============================================================================================// // Retrieves the credentials from an AWSCredentials.properties file. AWSCredentials credentials = null; try { credentials = new PropertiesCredentials( InlineGettingStartedCodeSampleApp.class.getResourceAsStream("AwsCredentials.properties")); } catch (IOException e1) { System.out.println("Credentials were not properly entered into AwsCredentials.properties."); System.out.println(e1.getMessage()); System.exit(-1); } // Create the AmazonEC2Client object so we can call various APIs. AmazonEC2 ec2 = new AmazonEC2Client(credentials); // Initializes a Spot Instance Request RequestSpotInstancesRequest requestRequest = new RequestSpotInstancesRequest(); // Request 1 x t1.micro instance with a bid price of $0.03. requestRequest.setSpotPrice("0.03"); requestRequest.setInstanceCount(Integer.valueOf(1)); // Setup the specifications of the launch. This includes the instance type (e.g. t1.micro) // and the latest Amazon Linux AMI id available. Note, you should always use the latest // Amazon Linux AMI id or another of your choosing. LaunchSpecification launchSpecification = new LaunchSpecification(); launchSpecification.setImageId("ami-8c1fece5"); launchSpecification.setInstanceType("t1.micro"); // Add the security group to the request. ArrayList<String> securityGroups = new ArrayList<String>(); securityGroups.add("GettingStartedGroup"); launchSpecification.setSecurityGroups(securityGroups); // Add the launch specifications to the request. requestRequest.setLaunchSpecification(launchSpecification); //============================================================================================// //=========================== Getting the Request ID from the Request ========================// //============================================================================================// // Call the RequestSpotInstance API. RequestSpotInstancesResult requestResult = ec2.requestSpotInstances(requestRequest); List<SpotInstanceRequest> requestResponses = requestResult.getSpotInstanceRequests(); // Setup an arraylist to collect all of the request ids we want to watch hit the running // state. ArrayList<String> spotInstanceRequestIds = new ArrayList<String>(); // Add all of the request ids to the hashset, so we can determine when they hit the // active state. for (SpotInstanceRequest requestResponse : requestResponses) { System.out.println("Created Spot Request: " + requestResponse.getSpotInstanceRequestId()); spotInstanceRequestIds.add(requestResponse.getSpotInstanceRequestId()); } //============================================================================================// //=========================== Determining the State of the Spot Request ======================// //============================================================================================// // Create a variable that will track whether there are any requests still in the open state. boolean anyOpen; // Initialize variables. ArrayList<String> instanceIds = new ArrayList<String>(); do { // Create the describeRequest with tall of the request id to monitor (e.g. that we started). DescribeSpotInstanceRequestsRequest describeRequest = new DescribeSpotInstanceRequestsRequest(); describeRequest.setSpotInstanceRequestIds(spotInstanceRequestIds); // Initialize the anyOpen variable to false which assumes there are no requests open unless // we find one that is still open. anyOpen = false; try { // Retrieve all of the requests we want to monitor. DescribeSpotInstanceRequestsResult describeResult = ec2 .describeSpotInstanceRequests(describeRequest); List<SpotInstanceRequest> describeResponses = describeResult.getSpotInstanceRequests(); // Look through each request and determine if they are all in the active state. for (SpotInstanceRequest describeResponse : describeResponses) { // If the state is open, it hasn't changed since we attempted to request it. // There is the potential for it to transition almost immediately to closed or // cancelled so we compare against open instead of active. if (describeResponse.getState().equals("open")) { anyOpen = true; break; } // Add the instance id to the list we will eventually terminate. instanceIds.add(describeResponse.getInstanceId()); } } catch (AmazonServiceException e) { // If we have an exception, ensure we don't break out of the loop. // This prevents the scenario where there was blip on the wire. anyOpen = true; } try { // Sleep for 60 seconds. Thread.sleep(60 * 1000); } catch (Exception e) { // Do nothing because it woke up early. } } while (anyOpen); //============================================================================================// //====================================== Canceling the Request ==============================// //============================================================================================// try { // Cancel requests. CancelSpotInstanceRequestsRequest cancelRequest = new CancelSpotInstanceRequestsRequest( spotInstanceRequestIds); ec2.cancelSpotInstanceRequests(cancelRequest); } catch (AmazonServiceException e) { // Write out any exceptions that may have occurred. System.out.println("Error cancelling instances"); System.out.println("Caught Exception: " + e.getMessage()); System.out.println("Reponse Status Code: " + e.getStatusCode()); System.out.println("Error Code: " + e.getErrorCode()); System.out.println("Request ID: " + e.getRequestId()); } //============================================================================================// //=================================== Terminating any Instances ==============================// //============================================================================================// try { // Terminate instances. TerminateInstancesRequest terminateRequest = new TerminateInstancesRequest(instanceIds); ec2.terminateInstances(terminateRequest); } catch (AmazonServiceException e) { // Write out any exceptions that may have occurred. System.out.println("Error terminating instances"); System.out.println("Caught Exception: " + e.getMessage()); System.out.println("Reponse Status Code: " + e.getStatusCode()); System.out.println("Error Code: " + e.getErrorCode()); System.out.println("Request ID: " + e.getRequestId()); } }
From source file:getting_started.SimpleQueueServiceSample.java
License:Open Source License
public static void main(String[] args) throws Exception { /*//ww w. j a va 2s .co m * Important: Be sure to fill in your AWS access credentials in the * AwsCredentials.properties file before you try to run this * sample. * http://aws.amazon.com/security-credentials */ AmazonSQS sqs = new AmazonSQSClient(new PropertiesCredentials( SimpleQueueServiceSample.class.getResourceAsStream("AwsCredentials.properties"))); System.out.println("==========================================="); System.out.println("Getting Started with Amazon SQS"); System.out.println("===========================================\n"); try { // Create a queue System.out.println("Creating a new SQS queue called MyQueue.\n"); CreateQueueRequest createQueueRequest = new CreateQueueRequest("MyQueue"); String myQueueUrl = sqs.createQueue(createQueueRequest).getQueueUrl(); // List queues System.out.println("Listing all queues in your account.\n"); for (String queueUrl : sqs.listQueues().getQueueUrls()) { System.out.println(" QueueUrl: " + queueUrl); } System.out.println(); // Send a message System.out.println("Sending a message to MyQueue.\n"); sqs.sendMessage(new SendMessageRequest(myQueueUrl, "This is my message text.")); // Receive messages System.out.println("Receiving messages from MyQueue.\n"); ReceiveMessageRequest receiveMessageRequest = new ReceiveMessageRequest(myQueueUrl); List<Message> messages = sqs.receiveMessage(receiveMessageRequest).getMessages(); for (Message message : messages) { System.out.println(" Message"); System.out.println(" MessageId: " + message.getMessageId()); System.out.println(" ReceiptHandle: " + message.getReceiptHandle()); System.out.println(" MD5OfBody: " + message.getMD5OfBody()); System.out.println(" Body: " + message.getBody()); for (Entry<String, String> entry : message.getAttributes().entrySet()) { System.out.println(" Attribute"); System.out.println(" Name: " + entry.getKey()); System.out.println(" Value: " + entry.getValue()); } } System.out.println(); // Delete a message System.out.println("Deleting a message.\n"); String messageRecieptHandle = messages.get(0).getReceiptHandle(); sqs.deleteMessage(new DeleteMessageRequest(myQueueUrl, messageRecieptHandle)); // Delete a queue System.out.println("Deleting the test queue.\n"); sqs.deleteQueue(new DeleteQueueRequest(myQueueUrl)); } catch (AmazonServiceException ase) { System.out.println("Caught an AmazonServiceException, which means your request made it " + "to Amazon SQS, but was rejected with an error response for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { System.out.println("Caught an AmazonClientException, which means the client encountered " + "a serious internal problem while trying to communicate with SQS, such as not " + "being able to access the network."); System.out.println("Error Message: " + ace.getMessage()); } }
From source file:gov.noaa.pfel.coastwatch.util.FileVisitorDNLS.java
License:Open Source License
/** * This is a convenience method for using this class. * <p>This works with Amazon AWS S3 bucket URLs. Internal /'s in the keys will be * treated as folder separators. If there aren't any /'s, all the keys will * be in the root directory./* ww w . ja v a2 s. co m*/ * * @param tDir The starting directory, with \\ or /, with or without trailing slash. * The resulting directoryPA will contain dirs with matching slashes and trailing slash. * @param tPathRegex a regex to constrain which subdirs to include. * This is ignored if recursive is false. * null or "" is treated as .* (i.e., match everything). * @param tDirectoriesToo if true, each directory name will get its own rows * in the results. * @return a table with columns with DIRECTORY, NAME, LASTMODIFIED, and SIZE columns. * LASTMODIFIED and SIZE are LongArrays -- For directories when the values * are otherwise unknown, the value will be Long.MAX_VALUE. * If directoriesToo=true, the original dir won't be included and any * directory's file NAME will be "". * @throws IOException if trouble */ public static Table oneStep(String tDir, String tFileNameRegex, boolean tRecursive, String tPathRegex, boolean tDirectoriesToo) throws IOException { long time = System.currentTimeMillis(); //is tDir an http URL? if (tDir.matches(FileVisitorDNLS.HTTP_REGEX)) { //Is it an S3 bucket with "files"? //If testing a "dir", url should have a trailing slash. Matcher matcher = AWS_S3_PATTERN.matcher(File2.addSlash(tDir)); //force trailing slash if (matcher.matches()) { //http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGET.html //If files have file-system-like names, e.g., // http://bucketname.s3.amazonaws.com/dir1/dir2/fileName.ext) // http://nasanex.s3.amazonaws.com/NEX-DCP30/BCSD/rcp26/mon/atmos/tasmin/r1i1p1/v1.0/CONUS/tasmin_amon_BCSD_rcp26_r1i1p1_CONUS_NorESM1-M_209601-209912.nc // you still can't request just dir2 info because they aren't directories. // They are just object keys with internal slashes. //So specify prefix in request. Table table = makeEmptyTable(); StringArray directoryPA = (StringArray) table.getColumn(DIRECTORY); StringArray namePA = (StringArray) table.getColumn(NAME); LongArray lastModifiedPA = (LongArray) table.getColumn(LASTMODIFIED); LongArray sizePA = (LongArray) table.getColumn(SIZE); String bucketName = matcher.group(1); String prefix = matcher.group(2); String baseURL = tDir.substring(0, matcher.start(2)); AmazonS3 s3client = new AmazonS3Client(new ProfileCredentialsProvider()); try { if (verbose) String2.log("FileVisitorDNLS.oneStep getting info from AWS S3 at" + "\nURL=" + tDir); //"\nbucket=" + bucketName + " prefix=" + prefix); //I wanted to generate lastMod for dir based on lastMod of files //but it would be inconsistent for different requests (recursive, fileNameRegex). //so just a set of dir names. HashSet<String> dirHashSet = new HashSet(); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName) .withPrefix(prefix); ObjectListing objectListing; do { objectListing = s3client.listObjects(listObjectsRequest); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { String keyFullName = objectSummary.getKey(); String keyDir = File2.getDirectory(baseURL + keyFullName); String keyName = File2.getNameAndExtension(keyFullName); if (debugMode) String2.log( "keyFullName=" + keyFullName + "\nkeyDir=" + keyDir + "\n tDir=" + tDir); if (keyDir.startsWith(tDir) && //it should (tRecursive || keyDir.length() == tDir.length())) { //store this dir if (tDirectoriesToo) { //S3 only returns object keys. I must infer/collect directories. //Store this dir and parents back to tDir. String choppedKeyDir = keyDir; while (choppedKeyDir.length() >= tDir.length()) { if (!dirHashSet.add(choppedKeyDir)) break; //hash set already had this, so will already have parents //chop off last subdirectory choppedKeyDir = File2.getDirectory( choppedKeyDir.substring(0, choppedKeyDir.length() - 1)); //remove trailing / } } //store this file's information //Sometimes directories appear as files are named "" with size=0. //I don't store those as files. if (debugMode) String2.log("keyName=" + keyFullName + "\n tFileNameRegex=" + tFileNameRegex + " matches=" + keyName.matches(tFileNameRegex)); if (keyName.length() > 0 && keyName.matches(tFileNameRegex)) { directoryPA.add(keyDir); namePA.add(keyName); lastModifiedPA.add(objectSummary.getLastModified().getTime()); //epoch millis sizePA.add(objectSummary.getSize()); //long } } } listObjectsRequest.setMarker(objectListing.getNextMarker()); } while (objectListing.isTruncated()); //add directories to the table if (tDirectoriesToo) { Iterator<String> it = dirHashSet.iterator(); while (it.hasNext()) { directoryPA.add(it.next()); namePA.add(""); lastModifiedPA.add(Long.MAX_VALUE); sizePA.add(Long.MAX_VALUE); } } table.leftToRightSortIgnoreCase(2); return table; } catch (AmazonServiceException ase) { throw new IOException("AmazonServiceException: " + ase.getErrorType() + " ERROR, HTTP Code=" + ase.getStatusCode() + ": " + ase.getMessage(), ase); } catch (AmazonClientException ace) { throw new IOException(ace.getMessage(), ace); } } //HYRAX before THREDDS //http://dods.jpl.nasa.gov/opendap/ocean_wind/ccmp/L3.5a/data/flk/1988/ matcher = HYRAX_PATTERN.matcher(tDir); if (matcher.matches()) { try { if (verbose) String2.log("FileVisitorDNLS.oneStep getting info from Hyrax at" + "\nURL=" + tDir); Table table = makeEmptyTable(); StringArray directoryPA = (StringArray) table.getColumn(DIRECTORY); StringArray namePA = (StringArray) table.getColumn(NAME); LongArray lastModifiedPA = (LongArray) table.getColumn(LASTMODIFIED); LongArray sizePA = (LongArray) table.getColumn(SIZE); DoubleArray lastModDA = new DoubleArray(); addToHyraxUrlList(tDir, tFileNameRegex, tRecursive, tPathRegex, tDirectoriesToo, namePA, lastModDA, sizePA); lastModifiedPA.append(lastModDA); int n = namePA.size(); for (int i = 0; i < n; i++) { String fn = namePA.get(i); directoryPA.add(File2.getDirectory(fn)); namePA.set(i, File2.getNameAndExtension(fn)); } table.leftToRightSortIgnoreCase(2); return table; } catch (Throwable t) { throw new IOException(t.getMessage(), t); } } //THREDDS matcher = THREDDS_PATTERN.matcher(tDir); if (matcher.matches()) { try { if (verbose) String2.log("FileVisitorDNLS.oneStep getting info from THREDDS at" + "\nURL=" + tDir); Table table = makeEmptyTable(); StringArray directoryPA = (StringArray) table.getColumn(DIRECTORY); StringArray namePA = (StringArray) table.getColumn(NAME); LongArray lastModifiedPA = (LongArray) table.getColumn(LASTMODIFIED); LongArray sizePA = (LongArray) table.getColumn(SIZE); DoubleArray lastModDA = new DoubleArray(); addToThreddsUrlList(tDir, tFileNameRegex, tRecursive, tPathRegex, tDirectoriesToo, namePA, lastModDA, sizePA); lastModifiedPA.append(lastModDA); int n = namePA.size(); for (int i = 0; i < n; i++) { String fn = namePA.get(i); directoryPA.add(File2.getDirectory(fn)); namePA.set(i, File2.getNameAndExtension(fn)); } table.leftToRightSortIgnoreCase(2); return table; } catch (Throwable t) { throw new IOException(t.getMessage(), t); } } //default: Apache-style WAF try { if (verbose) String2.log("FileVisitorDNLS.oneStep getting info from Apache-style WAF at" + "\nURL=" + tDir); Table table = makeEmptyTable(); StringArray directorySA = (StringArray) table.getColumn(DIRECTORY); StringArray nameSA = (StringArray) table.getColumn(NAME); LongArray lastModLA = (LongArray) table.getColumn(LASTMODIFIED); LongArray sizeLA = (LongArray) table.getColumn(SIZE); addToWAFUrlList(tDir, tFileNameRegex, tRecursive, tPathRegex, tDirectoriesToo, directorySA, nameSA, lastModLA, sizeLA); table.leftToRightSortIgnoreCase(2); return table; } catch (Throwable t) { throw new IOException(t.getMessage(), t); } } //local files //follow symbolic links: https://docs.oracle.com/javase/7/docs/api/java/nio/file/FileVisitor.html //But this doesn't follow Windows symbolic link .lnk's: // http://bugs.java.com/bugdatabase/view_bug.do?bug_id=4237760 FileVisitorDNLS fv = new FileVisitorDNLS(tDir, tFileNameRegex, tRecursive, tPathRegex, tDirectoriesToo); EnumSet<FileVisitOption> opts = EnumSet.of(FileVisitOption.FOLLOW_LINKS); Files.walkFileTree(FileSystems.getDefault().getPath(tDir), opts, //follow symbolic links Integer.MAX_VALUE, //maxDepth fv); fv.table.leftToRightSortIgnoreCase(2); if (verbose) String2.log("FileVisitorDNLS.oneStep(local) finished successfully. n=" + fv.directoryPA.size() + " time=" + (System.currentTimeMillis() - time) + "ms"); return fv.table; }
From source file:hu.cloud.edu.SimpleQueueServiceSample.java
License:Open Source License
public static void main(String[] args) throws Exception { /*/*from ww w .j av a2 s . c o m*/ * The ProfileCredentialsProvider will return your [default] * credential profile by reading from the credentials file located at * (C:\\Users\\Isaac\\.aws\\credentials). */ AWSCredentials credentials = null; try { credentials = new ProfileCredentialsProvider("default").getCredentials(); } catch (Exception e) { throw new AmazonClientException("Cannot load the credentials from the credential profiles file. " + "Please make sure that your credentials file is at the correct " + "location (C:\\Users\\Isaac\\.aws\\credentials), and is in valid format.", e); } AmazonSQS sqs = new AmazonSQSClient(credentials); Region usWest2 = Region.getRegion(Regions.US_WEST_2); sqs.setRegion(usWest2); System.out.println("==========================================="); System.out.println("Getting Started with Amazon SQS"); System.out.println("===========================================\n"); try { // Create a queue System.out.println("Creating a new SQS queue called MyQueue.\n"); CreateQueueRequest createQueueRequest = new CreateQueueRequest("MyQueue"); String myQueueUrl = sqs.createQueue(createQueueRequest).getQueueUrl(); // List queues System.out.println("Listing all queues in your account.\n"); for (String queueUrl : sqs.listQueues().getQueueUrls()) { System.out.println(" QueueUrl: " + queueUrl); } System.out.println(); // Send a message System.out.println("Sending a message to MyQueue.\n"); sqs.sendMessage(new SendMessageRequest(myQueueUrl, "This is my message text.")); // Receive messages System.out.println("Receiving messages from MyQueue.\n"); ReceiveMessageRequest receiveMessageRequest = new ReceiveMessageRequest(myQueueUrl); List<Message> messages = sqs.receiveMessage(receiveMessageRequest).getMessages(); for (Message message : messages) { System.out.println(" Message"); System.out.println(" MessageId: " + message.getMessageId()); System.out.println(" ReceiptHandle: " + message.getReceiptHandle()); System.out.println(" MD5OfBody: " + message.getMD5OfBody()); System.out.println(" Body: " + message.getBody()); for (Entry<String, String> entry : message.getAttributes().entrySet()) { System.out.println(" Attribute"); System.out.println(" Name: " + entry.getKey()); System.out.println(" Value: " + entry.getValue()); } } System.out.println(); // Delete a message System.out.println("Deleting a message.\n"); String messageRecieptHandle = messages.get(0).getReceiptHandle(); sqs.deleteMessage(new DeleteMessageRequest(myQueueUrl, messageRecieptHandle)); // Delete a queue System.out.println("Deleting the test queue.\n"); sqs.deleteQueue(new DeleteQueueRequest(myQueueUrl)); } catch (AmazonServiceException ase) { System.out.println("Caught an AmazonServiceException, which means your request made it " + "to Amazon SQS, but was rejected with an error response for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { System.out.println("Caught an AmazonClientException, which means the client encountered " + "a serious internal problem while trying to communicate with SQS, such as not " + "being able to access the network."); System.out.println("Error Message: " + ace.getMessage()); } }
From source file:hydrograph.engine.spark.datasource.utils.AWSS3Util.java
License:Apache License
public void download(RunFileTransferEntity runFileTransferEntity) { log.debug("Start AWSS3Util download"); File filecheck = new File(runFileTransferEntity.getLocalPath()); if (runFileTransferEntity.getFailOnError()) if (!(filecheck.exists() && filecheck.isDirectory()) && !(runFileTransferEntity.getLocalPath().contains("hdfs://"))) { throw new AWSUtilException("Invalid local path"); }/*from ww w .j a v a2 s . c o m*/ boolean fail_if_exist = false; int retryAttempt = 0; int i; String amazonFileUploadLocationOriginal = null; String keyName = null; if (runFileTransferEntity.getRetryAttempt() == 0) retryAttempt = 1; else retryAttempt = runFileTransferEntity.getRetryAttempt(); for (i = 0; i < retryAttempt; i++) { log.info("connection attempt: " + (i + 1)); try { AmazonS3 s3Client = null; ClientConfiguration clientConf = new ClientConfiguration(); clientConf.setProtocol(Protocol.HTTPS); if (runFileTransferEntity.getCrediationalPropertiesFile() == null) { BasicAWSCredentials creds = new BasicAWSCredentials(runFileTransferEntity.getAccessKeyID(), runFileTransferEntity.getSecretAccessKey()); s3Client = AmazonS3ClientBuilder.standard().withClientConfiguration(clientConf) .withRegion(runFileTransferEntity.getRegion()) .withCredentials(new AWSStaticCredentialsProvider(creds)).build(); } else { File securityFile = new File(runFileTransferEntity.getCrediationalPropertiesFile()); PropertiesCredentials creds = new PropertiesCredentials(securityFile); s3Client = AmazonS3ClientBuilder.standard().withClientConfiguration(clientConf) .withRegion(runFileTransferEntity.getRegion()) .withCredentials(new AWSStaticCredentialsProvider(creds)).build(); } String s3folderName = null; String filepath = runFileTransferEntity.getFolder_name_in_bucket(); if (filepath.lastIndexOf("/") != -1) { s3folderName = filepath.substring(0, filepath.lastIndexOf("/")); keyName = filepath.substring(filepath.lastIndexOf("/") + 1); } else { keyName = filepath; } log.debug("keyName is: " + keyName); log.debug("bucket name is:" + runFileTransferEntity.getBucketName()); log.debug("Folder Name is" + runFileTransferEntity.getFolder_name_in_bucket()); if (s3folderName != null) { amazonFileUploadLocationOriginal = runFileTransferEntity.getBucketName() + "/" + s3folderName; } else { amazonFileUploadLocationOriginal = runFileTransferEntity.getBucketName(); } if (runFileTransferEntity.getLocalPath().contains("hdfs://")) { String outputPath = runFileTransferEntity.getLocalPath(); String s1 = outputPath.substring(7, outputPath.length()); String s2 = s1.substring(0, s1.indexOf("/")); File f = new File("/tmp"); if (!f.exists()) f.mkdir(); GetObjectRequest request = new GetObjectRequest(amazonFileUploadLocationOriginal, keyName); S3Object object = s3Client.getObject(request); if (runFileTransferEntity.getEncoding() != null) object.getObjectMetadata().setContentEncoding(runFileTransferEntity.getEncoding()); File fexist = new File(runFileTransferEntity.getLocalPath() + File.separatorChar + keyName); if (runFileTransferEntity.getOverwrite().trim().equalsIgnoreCase("Overwrite If Exists")) { S3ObjectInputStream objectContent = object.getObjectContent(); IOUtils.copyLarge(objectContent, new FileOutputStream("/tmp/" + keyName)); } else { if (!(fexist.exists() && !fexist.isDirectory())) { S3ObjectInputStream objectContent = object.getObjectContent(); IOUtils.copyLarge(objectContent, new FileOutputStream( runFileTransferEntity.getLocalPath() + File.separatorChar + keyName)); } else { fail_if_exist = true; Log.error("File already exists"); throw new AWSUtilException("File already exists"); } } Configuration conf = new Configuration(); conf.set("fs.defaultFS", "hdfs://" + s2); FileSystem hdfsFileSystem = FileSystem.get(conf); String s = outputPath.substring(7, outputPath.length()); String hdfspath = s.substring(s.indexOf("/"), s.length()); Path local = new Path("/tmp/" + keyName); Path hdfs = new Path(hdfspath); hdfsFileSystem.copyFromLocalFile(local, hdfs); } else { GetObjectRequest request = new GetObjectRequest(amazonFileUploadLocationOriginal, keyName); S3Object object = s3Client.getObject(request); if (runFileTransferEntity.getEncoding() != null) object.getObjectMetadata().setContentEncoding(runFileTransferEntity.getEncoding()); File fexist = new File(runFileTransferEntity.getLocalPath() + File.separatorChar + keyName); if (runFileTransferEntity.getOverwrite().trim().equalsIgnoreCase("Overwrite If Exists")) { S3ObjectInputStream objectContent = object.getObjectContent(); IOUtils.copyLarge(objectContent, new FileOutputStream( runFileTransferEntity.getLocalPath() + File.separatorChar + keyName)); } else { if (!(fexist.exists() && !fexist.isDirectory())) { S3ObjectInputStream objectContent = object.getObjectContent(); IOUtils.copyLarge(objectContent, new FileOutputStream( runFileTransferEntity.getLocalPath() + File.separatorChar + keyName)); } else { fail_if_exist = true; Log.error("File already exists"); throw new AWSUtilException("File already exists"); } } } } catch (AmazonServiceException e) { log.error("Amazon Service Exception", e); if (e.getStatusCode() == 403 || e.getStatusCode() == 404) { if (runFileTransferEntity.getFailOnError()) { Log.error("Incorrect details provided.Please provide correct details", e); throw new AWSUtilException("Incorrect details provided"); } else { Log.error("Unknown amezon exception occured", e); } } { try { Thread.sleep(runFileTransferEntity.getRetryAfterDuration()); } catch (Exception e1) { Log.error("Exception occured while sleeping the thread"); } continue; } } catch (Error e) { Log.error("Error occured while sleeping the thread"); throw new AWSUtilException(e); } catch (Exception e) { log.error("error while transfering file", e); try { Thread.sleep(runFileTransferEntity.getRetryAfterDuration()); } catch (Exception e1) { } catch (Error err) { Log.error("Error occured while downloading"); throw new AWSUtilException(err); } continue; } done = true; break; } if (runFileTransferEntity.getFailOnError() && !done) { log.error("File transfer failed"); throw new AWSUtilException("File transfer failed"); } else if (!done) { log.error("File transfer failed but mentioned fail on error as false"); } if (i == runFileTransferEntity.getRetryAttempt()) { if (runFileTransferEntity.getFailOnError()) { throw new AWSUtilException("File transfer failed"); } } log.debug("Finished AWSS3Util download"); }
From source file:ics.uci.edu.amazons3.S3Sample.java
License:Open Source License
public static void main(String[] args) throws IOException { /*/* w w w . j a v a2 s .c om*/ * This credentials provider implementation loads your AWS credentials * from a properties file at the root of your classpath. * * Important: Be sure to fill in your AWS access credentials in the * AwsCredentials.properties file before you try to run this * sample. * http://aws.amazon.com/security-credentials */ final AmazonS3 s3 = new AmazonS3Client( new BasicAWSCredentials("AKIAJTW5BOY6EXOGV2YQ", "PDcnFYIf9Hdo9GsKTEjLXretZ3yEg4mRCDQKjxu6")); String bucketName = "my-first-s3-bucket-" + UUID.randomUUID(); String key = "MyObjectKey"; System.out.println("==========================================="); System.out.println("Getting Started with Amazon S3"); System.out.println("===========================================\n"); try { /* * Create a new S3 bucket - Amazon S3 bucket names are globally unique, * so once a bucket name has been taken by any user, you can't create * another bucket with that same name. * * You can optionally specify a location for your bucket if you want to * keep your data closer to your applications or users. */ System.out.println("Creating bucket " + bucketName + "\n"); s3.createBucket(bucketName); /* * List the buckets in your account */ System.out.println("Listing buckets"); for (Bucket bucket : s3.listBuckets()) { System.out.println(" - " + bucket.getName()); } System.out.println(); /* * Upload an object to your bucket - You can easily upload a file to * S3, or upload directly an InputStream if you know the length of * the data in the stream. You can also specify your own metadata * when uploading to S3, which allows you set a variety of options * like content-type and content-encoding, plus additional metadata * specific to your applications. */ System.out.println("Uploading a new object to S3 from a file\n"); s3.putObject(new PutObjectRequest(bucketName, key, createSampleFile())); /* * Download an object - When you download an object, you get all of * the object's metadata and a stream from which to read the contents. * It's important to read the contents of the stream as quickly as * possibly since the data is streamed directly from Amazon S3 and your * network connection will remain open until you read all the data or * close the input stream. * * GetObjectRequest also supports several other options, including * conditional downloading of objects based on modification times, * ETags, and selectively downloading a range of an object. */ System.out.println("Downloading an object"); S3Object object = s3.getObject(new GetObjectRequest(bucketName, key)); System.out.println("Content-Type: " + object.getObjectMetadata().getContentType()); displayTextInputStream(object.getObjectContent()); /* * List objects in your bucket by prefix - There are many options for * listing the objects in your bucket. Keep in mind that buckets with * many objects might truncate their results when listing their objects, * so be sure to check if the returned object listing is truncated, and * use the AmazonS3.listNextBatchOfObjects(...) operation to retrieve * additional results. */ System.out.println("Listing objects"); ObjectListing objectListing = s3 .listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix("My")); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { System.out.println( " - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")"); } System.out.println(); /* * Delete an object - Unless versioning has been turned on for your bucket, * there is no way to undelete an object, so use caution when deleting objects. */ System.out.println("Deleting an object\n"); s3.deleteObject(bucketName, key); /* * Delete a bucket - A bucket must be completely empty before it can be * deleted, so remember to delete any objects from your buckets before * you try to delete them. */ System.out.println("Deleting bucket " + bucketName + "\n"); s3.deleteBucket(bucketName); } catch (AmazonServiceException ase) { System.out.println("Caught an AmazonServiceException, which means your request made it " + "to Amazon S3, but was rejected with an error response for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { System.out.println("Caught an AmazonClientException, which means the client encountered " + "a serious internal problem while trying to communicate with S3, " + "such as not being able to access the network."); System.out.println("Error Message: " + ace.getMessage()); } }
From source file:ie.peternagy.jcrypto.module.config.S3ConfigHandler.java
License:Open Source License
@Override public void validateConfig(Map config) { AWSCredentials credentials = new BasicAWSCredentials((String) config.get("access-key"), (String) config.get("secret-key")); AmazonS3 s3client = new AmazonS3Client(credentials); try {/* ww w. ja v a 2 s . c o m*/ System.out.println("S3 config validation...\n"); s3client.doesBucketExist((String) config.get("bucket-name")); System.out.println("Valid configuration!\n"); } catch (AmazonServiceException ase) { System.out.println("Caught an AmazonServiceException, which " + "means your request made it " + "to Amazon S3, but was rejected with an error response" + " for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { System.out.println("Caught an AmazonClientException, which " + "means the client encountered " + "an internal error while trying to " + "communicate with S3, " + "such as not being able to access the network."); System.out.println("Error Message: " + ace.getMessage()); } }
From source file:iit.edu.supadyay.s3.S3upload.java
public static boolean upload(String bucketName, String uploadFileName, String keyName) throws IOException, InterruptedException { //access = "AKIAJ2YSLRUZR5B3F5HQ"; //secret = "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"; //AWSCredentials credentials = new BasicAWSCredentials(access, secret); //AmazonS3 s3client = new AmazonS3Client(getCredentials()); AmazonS3 s3client = new AmazonS3Client(new InstanceProfileCredentialsProvider()); try {//from ww w . ja v a2s. c o m System.out.println("Uploading a new object to S3 from a file\n"); File file = new File(uploadFileName); System.out.println("I am before here\n"); s3client.createBucket(bucketName); System.out.println("I am here\n"); s3client.putObject(new PutObjectRequest(bucketName, keyName, file)); s3client.setObjectAcl(bucketName, keyName, CannedAccessControlList.PublicReadWrite); } catch (AmazonServiceException ase) { System.out.println("Caught an AmazonServiceException, which " + "means your request made it " + "to Amazon S3, but was rejected with an error response" + " for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); return false; } catch (AmazonClientException ace) { System.out.println("Caught an AmazonClientException, which " + "means the client encountered " + "an internal error while trying to " + "communicate with S3, " + "such as not being able to access the network."); System.out.println("Error Message: " + ace.getMessage()); return false; } return true; }
From source file:io.druid.storage.s3.S3TaskLogs.java
License:Apache License
@Override public Optional<ByteSource> streamTaskLog(final String taskid, final long offset) throws IOException { final String taskKey = getTaskLogKey(taskid); try {// w ww . j a va 2 s .c om final ObjectMetadata objectMetadata = service.getObjectMetadata(config.getS3Bucket(), taskKey); return Optional.of(new ByteSource() { @Override public InputStream openStream() throws IOException { try { final long start; final long end = objectMetadata.getContentLength() - 1; if (offset > 0 && offset < objectMetadata.getContentLength()) { start = offset; } else if (offset < 0 && (-1 * offset) < objectMetadata.getContentLength()) { start = objectMetadata.getContentLength() + offset; } else { start = 0; } final GetObjectRequest request = new GetObjectRequest(config.getS3Bucket(), taskKey) .withMatchingETagConstraint(objectMetadata.getETag()).withRange(start, end); return service.getObject(request).getObjectContent(); } catch (AmazonServiceException e) { throw new IOException(e); } } }); } catch (AmazonS3Exception e) { if (404 == e.getStatusCode() || "NoSuchKey".equals(e.getErrorCode()) || "NoSuchBucket".equals(e.getErrorCode())) { return Optional.absent(); } else { throw new IOE(e, "Failed to stream logs from: %s", taskKey); } } }
From source file:io.konig.camel.aws.s3.DeleteObjectEndpoint.java
License:Apache License
@Override public void doStart() throws Exception { super.doStart(); String envName = ""; if (System.getProperty("environmentName") != null) { envName = System.getProperty("environmentName"); }/*from w ww .j a v a 2 s . com*/ String bucketName = getConfiguration().getBucketName(); bucketName = bucketName.replace("${environmentName}", envName); configuration.setBucketName(bucketName); configuration.getAmazonS3Client().setRegion(Region.getRegion(Regions.fromName(configuration.getRegion()))); s3Client = configuration.getAmazonS3Client() != null ? configuration.getAmazonS3Client() : S3ClientFactory.getAWSS3Client(configuration, getMaxConnections()).getS3Client(); String fileName = getConfiguration().getFileName(); if (fileName != null) { LOG.trace("File name [{}] requested, so skipping bucket check...", fileName); return; } LOG.trace("Querying whether bucket [{}] already exists...", bucketName); String prefix = getConfiguration().getPrefix(); try { s3Client.listObjects(new ListObjectsRequest(bucketName, prefix, null, null, 0)); LOG.trace("Bucket [{}] already exists", bucketName); return; } catch (AmazonServiceException ase) { /* 404 means the bucket doesn't exist */ if (ase.getStatusCode() != 404) { throw ase; } } LOG.trace("Bucket [{}] doesn't exist yet", bucketName); // creates the new bucket because it doesn't exist yet CreateBucketRequest createBucketRequest = new CreateBucketRequest(getConfiguration().getBucketName()); //LOG.trace("Creating bucket [{}] in region [{}] with request [{}]...", configuration.getBucketName(), configuration.getRegion(), createBucketRequest); s3Client.createBucket(createBucketRequest); LOG.trace("Bucket created"); if (configuration.getPolicy() != null) { LOG.trace("Updating bucket [{}] with policy [{}]", bucketName, configuration.getPolicy()); s3Client.setBucketPolicy(bucketName, configuration.getPolicy()); LOG.trace("Bucket policy updated"); } }