List of usage examples for com.amazonaws.services.s3.model S3ObjectSummary getSize
public long getSize()
From source file:com.cloudhub.aws.extractor.AWSCSVExtractor.java
License:Apache License
/** * Requests billing information from Amazon S3. * This method may spawn multiple threads as needed to complete the task. * *//* ww w . jav a 2 s . com*/ @Override public String getTotalCost() { String totalCost = null; try { log.debug("Listing objects ..."); final ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName); ObjectListing objectListing; do { objectListing = s3client.listObjects(listObjectsRequest); for (final S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { log.debug(" - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")"); if (objectSummary.getKey().contains(Constants.MATCHER_BILLING_CSV.getKeyPattern())) { totalCost = persist(Constants.MATCHER_BILLING_CSV, objectSummary); } else if (objectSummary.getKey().contains(Constants.MATCHER_COST_ALLOCATION.getKeyPattern())) { totalCost = persist(Constants.MATCHER_COST_ALLOCATION, objectSummary); } } listObjectsRequest.setMarker(objectListing.getNextMarker()); } while (objectListing.isTruncated()); } catch (AmazonServiceException ase) { log.error("Caught an AmazonServiceException, " + "which means your request made it " + "to Amazon S3, but was rejected with an error response " + "for some reason."); log.error("Error Message: " + ase.getMessage()); log.error("HTTP Status Code: " + ase.getStatusCode()); log.error("AWS Error Code: " + ase.getErrorCode()); log.error("Error Type: " + ase.getErrorType()); log.error("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { log.error("Caught an AmazonClientException, " + "which means the client encountered " + "an internal error while trying to communicate" + " with S3, " + "such as not being able to access the network."); log.error("Error Message: " + ace.getMessage()); } catch (IOException ioe) { log.error("Caught an IOException while writing to disk."); log.error("Error Message: " + ioe.getMessage()); } return totalCost; }
From source file:com.cloudhub.aws.extractor.AWSCSVExtractor.java
License:Apache License
/** * Persists the Amazon S3 object to disk * * @param objectSummary - the S3 object to be persisted to disk. * @throws IOException - if an I/O error occurs. *//*from w w w .j av a 2 s . co m*/ private String persist(final CSVMatcher matcher, final S3ObjectSummary objectSummary) throws IOException { log.debug("Downloading the body of " + objectSummary.getKey() + " from Amazon S3."); final S3Object object = s3client.getObject(bucketName, objectSummary.getKey()); log.debug("Downloaded " + objectSummary.getSize() + " bytes."); log.debug("Writing the body of " + objectSummary.getKey() + " to disk path: " + dataFolder + File.separator + bucketName); final File objectCSVFile = writeOutObjectToFile(objectSummary, object.getObjectContent()); return getTotal(matcher, objectCSVFile); }
From source file:com.cloudhub.aws.extractor.AWSCSVExtractor.java
License:Apache License
/** * Writes the specified Amazon S3 object to the file system. * * @param object - the object to write out * @param inputStream - the body of the object * @throws IOException - if any I/O error occurs. *//*from w ww.ja va 2s .c o m*/ @SuppressWarnings("resource") private File writeOutObjectToFile(final S3ObjectSummary object, final InputStream inputStream) throws IOException { final File parent = new File(dataFolder, object.getBucketName()); if (!parent.exists()) { parent.mkdirs(); } final File objectFile = new File(parent, object.getKey()); final ReadableByteChannel src = Channels.newChannel(inputStream); final FileChannel dest = new FileOutputStream(objectFile).getChannel(); dest.transferFrom(src, 0, object.getSize()); dest.close(); src.close(); return objectFile; }
From source file:com.conductor.s3.S3InputFormatUtils.java
License:Apache License
/** * Efficiently gets the Hadoop {@link org.apache.hadoop.fs.FileStatus} for all S3 files under the provided * {@code dirs}//from w w w . j av a2s .c o m * * @param s3Client * s3 client * @param blockSize * the block size * @param dirs * the dirs to search through * @return the {@link org.apache.hadoop.fs.FileStatus} version of all S3 files under {@code dirs} */ static List<FileStatus> getFileStatuses(final AmazonS3 s3Client, final long blockSize, final Path... dirs) { final List<FileStatus> result = Lists.newArrayList(); for (final Path dir : dirs) { // get bucket and prefix from path final String bucket = S3HadoopUtils.getBucketFromPath(dir.toString()); final String prefix = S3HadoopUtils.getKeyFromPath(dir.toString()); // list request final ListObjectsRequest req = new ListObjectsRequest().withMaxKeys(Integer.MAX_VALUE) .withBucketName(bucket).withPrefix(prefix); // recursively page through all objects under the path for (ObjectListing listing = s3Client.listObjects(req); listing.getObjectSummaries() .size() > 0; listing = s3Client.listNextBatchOfObjects(listing)) { for (final S3ObjectSummary summary : listing.getObjectSummaries()) { final Path path = new Path( String.format("s3n://%s/%s", summary.getBucketName(), summary.getKey())); if (S3_PATH_FILTER.accept(path)) { result.add(new FileStatus(summary.getSize(), false, 1, blockSize, summary.getLastModified().getTime(), path)); } } // don't need to check the next listing if this one is not truncated if (!listing.isTruncated()) { break; } } } return result; }
From source file:com.crickdata.upload.s3.UploadLiveData.java
License:Open Source License
public Map<String, Date> uploadToS3(String fileName, boolean type) throws IOException { Statistics statistics = new Statistics(); Map<String, Date> perfMap = new HashMap<String, Date>(); AWSCredentials credentials = null;/*from ww w . j a v a 2s . c o m*/ try { credentials = new BasicAWSCredentials("AKIAI6QKTRAQE7MXQOIQ", "wIG6u1yI5ZaseeJbvYSUmD98qelIJNSCVBzt5k2q"); } catch (Exception e) { throw new AmazonClientException("Cannot load the credentials from the credential profiles file. " + "Please make sure that your credentials file is at the correct " + "location (C:\\Users\\bssan_000\\.aws\\credentials), and is in valid format.", e); } AmazonS3 s3 = new AmazonS3Client(credentials); Region usWest2 = Region.getRegion(Regions.US_WEST_2); s3.setRegion(usWest2); String bucketName; if (!type) bucketName = "cricmatchinfo"; else bucketName = "cricmatchinfoseries"; String key = fileName.replace(".json", "").trim(); try { perfMap.put("S3INSERTREQ", new Date()); statistics.setS3Req(new Date()); File f = readMatchFile(fileName); double bytes = f.length(); double kilobytes = (bytes / 1024); System.out.println("Details :" + kilobytes); s3.putObject(new PutObjectRequest(bucketName, key, f)); statistics.setSize(String.valueOf(kilobytes)); S3Object object = s3.getObject(new GetObjectRequest(bucketName, key)); perfMap.put("S3SAVERES", object.getObjectMetadata().getLastModified()); statistics.setKey(key); statistics.setS3Res(object.getObjectMetadata().getLastModified()); MyUI.stats.add(statistics); displayTextInputStream(object.getObjectContent()); ObjectListing objectListing = s3 .listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix("My")); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { System.out.println( " - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")"); } } catch (AmazonServiceException ase) { System.out.println("Caught an AmazonServiceException, which means your request made it " + "to Amazon S3, but was rejected with an error response for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { System.out.println("Caught an AmazonClientException, which means the client encountered " + "a serious internal problem while trying to communicate with S3, " + "such as not being able to access the network."); System.out.println("Error Message: " + ace.getMessage()); } return perfMap; }
From source file:com.digitaslbi.helios.mock.utils.ConnectionHelper.java
private static Map convertResultToFile(ListObjectsRequest listObjectsRequest, String path, String parentPath) { Map<String, File> filesMap = new HashMap<String, File>(); File aux;//w w w .j a v a2s. c o m ObjectListing objListing = s3Client.listObjects(listObjectsRequest); // creates the parent elemment aux = new File(); aux.setIsParent(true); aux.setFullPath(parentPath); aux.setPath(parentPath); // adapts the path in order to use as map key if (aux.getPath() != null) { aux.setPath(clearPathDelimiter(aux.getPath())); if (aux.getPath().lastIndexOf(MocksConstants.AWS_PARENT_DELIMITER.getValue()) > 0) { aux.setPath(aux.getPath() .substring(aux.getPath().lastIndexOf(MocksConstants.AWS_PARENT_DELIMITER.getValue()) + 1)); } } filesMap.put(aux.getPath(), aux); for (S3ObjectSummary objSummary : objListing.getObjectSummaries()) { aux = new File(); aux.setFullPath(objSummary.getKey()); aux.setParent(objListing.getPrefix()); // if size is 0 is considered a folder aux.setIsFile((objSummary.getSize() == 0) ? false : true); if (aux.getParent() != null) { if (!aux.getFullPath().equals(aux.getParent())) { aux.setPath(aux.getFullPath().replace(aux.getParent(), "")); } } else { aux.setPath(aux.getFullPath()); } if (aux.getPath() != null) { filesMap.put(clearPathDelimiter(aux.getPath()), aux); } } for (String folderNames : objListing.getCommonPrefixes()) { aux = new File(); aux.setFullPath(folderNames); aux.setParent(objListing.getPrefix()); aux.setIsFile(false); if (aux.getParent() != null) { aux.setPath(aux.getFullPath().replace(aux.getParent(), "")); } else { aux.setPath(aux.getFullPath()); } if (aux.getPath() != null) { filesMap.put(clearPathDelimiter(aux.getPath()), aux); } } return filesMap; }
From source file:com.digitaslbi.helios.utils.S3Helper.java
public static Folder getFolder(String path) { ListObjectsRequest listObjectsRequest; Folder folder;/*from www. ja v a2 s .c om*/ File aux; if (path == null || path.equals("/")) { listObjectsRequest = getRootFolders(); } else { listObjectsRequest = getContentByPreffix(path); } ObjectListing objListing = s3Client.listObjects(listObjectsRequest); folder = new Folder(); folder.setFiles(new ArrayList<File>()); folder.setPath(path); folder.setParent(obtainParentPath(path)); for (S3ObjectSummary objSummary : objListing.getObjectSummaries()) { aux = new File(); aux.setPath(objSummary.getKey()); aux.setParent(objListing.getPrefix()); // if size is 0 is considered a folder aux.setIsFile((objSummary.getSize() == 0) ? false : true); if (!aux.getPath().equals(path)) { folder.getFiles().add(aux); } } for (String folderNames : objListing.getCommonPrefixes()) { aux = new File(); aux.setPath(folderNames); aux.setParent(objListing.getPrefix()); aux.setIsFile(false); folder.getFiles().add(aux); } log.info("[S3Helper][getFolder] Path: " + path + " items found: " + folder.getFiles().size()); return folder; }
From source file:com.eucalyptus.objectstorage.providers.s3.S3ProviderClient.java
License:Open Source License
@Override public ListBucketResponseType listBucket(ListBucketType request) throws S3Exception { ListBucketResponseType reply = request.getReply(); User requestUser = getRequestUser(request); OsgInternalS3Client internalS3Client = null; try {/*from w ww .j a va 2s .co m*/ internalS3Client = getS3Client(requestUser); AmazonS3Client s3Client = internalS3Client.getS3Client(); ListObjectsRequest listRequest = new ListObjectsRequest(); listRequest.setBucketName(request.getBucket()); listRequest.setDelimiter(Strings.isNullOrEmpty(request.getDelimiter()) ? null : request.getDelimiter()); listRequest.setMarker(Strings.isNullOrEmpty(request.getMarker()) ? null : request.getMarker()); listRequest.setMaxKeys((request.getMaxKeys() == null ? null : Integer.parseInt(request.getMaxKeys()))); listRequest.setPrefix(Strings.isNullOrEmpty(request.getPrefix()) ? null : request.getPrefix()); ObjectListing response = s3Client.listObjects(listRequest); /* Non-optional, must have non-null values */ reply.setName(request.getBucket()); reply.setMaxKeys(response.getMaxKeys()); reply.setMarker(response.getMarker() == null ? "" : response.getMarker()); reply.setPrefix(response.getPrefix() == null ? "" : response.getPrefix()); reply.setIsTruncated(response.isTruncated()); /* Optional */ reply.setNextMarker(response.getNextMarker()); reply.setDelimiter(response.getDelimiter()); if (reply.getContents() == null) { reply.setContents(new ArrayList<ListEntry>()); } if (reply.getCommonPrefixesList() == null) { reply.setCommonPrefixesList(new ArrayList<CommonPrefixesEntry>()); } for (S3ObjectSummary obj : response.getObjectSummaries()) { //Add entry, note that the canonical user is set based on requesting user, not returned user reply.getContents() .add(new ListEntry(obj.getKey(), DateFormatter.dateToHeaderFormattedString(obj.getLastModified()), obj.getETag(), obj.getSize(), getCanonicalUser(requestUser), obj.getStorageClass())); } if (response.getCommonPrefixes() != null && response.getCommonPrefixes().size() > 0) { reply.setCommonPrefixesList(new ArrayList<CommonPrefixesEntry>()); for (String s : response.getCommonPrefixes()) { reply.getCommonPrefixesList().add(new CommonPrefixesEntry(s)); } } return reply; } catch (AmazonServiceException e) { LOG.debug("Error from backend", e); throw S3ExceptionMapper.fromAWSJavaSDK(e); } }
From source file:com.example.S3Sample02.java
License:Open Source License
public static void main(String[] args) throws IOException { /*/* w ww.j a v a 2 s .c o m*/ * The ProfileCredentialsProvider will return your [default] * credential profile by reading from the credentials file located at * (~/.aws/credentials). */ AWSCredentials credentials = null; try { credentials = new ProfileCredentialsProvider().getCredentials(); } catch (Exception e) { throw new AmazonClientException("Cannot load the credentials from the credential profiles file. " + "Please make sure that your credentials file is at the correct " + "location (~/.aws/credentials), and is in valid format.", e); } AmazonS3 s3 = new AmazonS3Client(credentials); // AP_SOUTHEAST_2 // Region usWest2 = Region.getRegion(Regions.AP_SOUTHEAST_2 ); // s3.setRegion(usWest2); // String bucketName = "my-first-s3-bucket-" + UUID.randomUUID(); String bucketName = "imos-test-data-1"; String key = "MyObjectKey" + UUID.randomUUID(); System.out.println("==========================================="); System.out.println("Getting Started with Amazon S3"); System.out.println("===========================================\n"); try { /* * Create a new S3 bucket - Amazon S3 bucket names are globally unique, * so once a bucket name has been taken by any user, you can't create * another bucket with that same name. * * You can optionally specify a location for your bucket if you want to * keep your data closer to your applications or users. */ // System.out.println("Creating bucket " + bucketName + "\n"); // s3.createBucket(bucketName); /* * List the buckets in your account */ /* System.out.println("Listing buckets"); for (Bucket bucket : s3.listBuckets()) { System.out.println(" - " + bucket.getName()); } System.out.println(); */ /* * Upload an object to your bucket - You can easily upload a file to * S3, or upload directly an InputStream if you know the length of * the data in the stream. You can also specify your own metadata * when uploading to S3, which allows you set a variety of options * like content-type and content-encoding, plus additional metadata * specific to your applications. */ System.out.println("Uploading a new object to S3 from a file\n"); s3.putObject(new PutObjectRequest(bucketName, key, createSampleFile())); System.out.println("done\n"); /* * Download an object - When you download an object, you get all of * the object's metadata and a stream from which to read the contents. * It's important to read the contents of the stream as quickly as * possibly since the data is streamed directly from Amazon S3 and your * network connection will remain open until you read all the data or * close the input stream. * * GetObjectRequest also supports several other options, including * conditional downloading of objects based on modification times, * ETags, and selectively downloading a range of an object. */ System.out.println("Downloading an object"); S3Object object = s3.getObject(new GetObjectRequest(bucketName, key)); System.out.println("Content-Type: " + object.getObjectMetadata().getContentType()); displayTextInputStream(object.getObjectContent()); System.out.println("done\n"); /* * List objects in your bucket by prefix - There are many options for * listing the objects in your bucket. Keep in mind that buckets with * many objects might truncate their results when listing their objects, * so be sure to check if the returned object listing is truncated, and * use the AmazonS3.listNextBatchOfObjects(...) operation to retrieve * additional results. */ System.out.println("Listing objects"); ObjectListing objectListing = s3 .listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix("My")); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { System.out.println( " - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")"); } System.out.println(); System.out.println("done\n"); /* * Delete an object - Unless versioning has been turned on for your bucket, * there is no way to undelete an object, so use caution when deleting objects. */ // System.out.println("Deleting an object\n"); // s3.deleteObject(bucketName, key); /* * Delete a bucket - A bucket must be completely empty before it can be * deleted, so remember to delete any objects from your buckets before * you try to delete them. */ /* System.out.println("Deleting bucket " + bucketName + "\n"); s3.deleteBucket(bucketName); */ } catch (AmazonServiceException ase) { System.out.println("Caught an AmazonServiceException, which means your request made it " + "to Amazon S3, but was rejected with an error response for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { System.out.println("Caught an AmazonClientException, which means the client encountered " + "a serious internal problem while trying to communicate with S3, " + "such as not being able to access the network."); System.out.println("Error Message: " + ace.getMessage()); } }
From source file:com.exedosoft.plat.storage.s3.S3Sample.java
License:Open Source License
public static void main(String[] args) throws IOException { /*/*from w w w . ja va 2s .c o m*/ * This credentials provider implementation loads your AWS credentials * from a properties file at the root of your classpath. * * Important: Be sure to fill in your AWS access credentials in the * AwsCredentials.properties file before you try to run this * sample. * http://aws.amazon.com/security-credentials */ AmazonS3 s3 = new AmazonS3Client(new ClasspathPropertiesFileCredentialsProvider()); Region usWest2 = Region.getRegion(Regions.US_WEST_2); s3.setRegion(usWest2); String bucketName = "my-first-s3-bucket-" + UUID.randomUUID(); String key = "MyObjectKey"; System.out.println("==========================================="); System.out.println("Getting Started with Amazon S3"); System.out.println("===========================================\n"); try { /* * Create a new S3 bucket - Amazon S3 bucket names are globally unique, * so once a bucket name has been taken by any user, you can't create * another bucket with that same name. * * You can optionally specify a location for your bucket if you want to * keep your data closer to your applications or users. */ System.out.println("Creating bucket " + bucketName + "\n"); s3.createBucket(bucketName); /* * List the buckets in your account */ System.out.println("Listing buckets"); for (Bucket bucket : s3.listBuckets()) { System.out.println(" - " + bucket.getName()); } System.out.println(); /* * Upload an object to your bucket - You can easily upload a file to * S3, or upload directly an InputStream if you know the length of * the data in the stream. You can also specify your own metadata * when uploading to S3, which allows you set a variety of options * like content-type and content-encoding, plus additional metadata * specific to your applications. */ System.out.println("Uploading a new object to S3 from a file\n"); s3.putObject(new PutObjectRequest(bucketName, key, createSampleFile())); /* * Download an object - When you download an object, you get all of * the object's metadata and a stream from which to read the contents. * It's important to read the contents of the stream as quickly as * possibly since the data is streamed directly from Amazon S3 and your * network connection will remain open until you read all the data or * close the input stream. * * GetObjectRequest also supports several other options, including * conditional downloading of objects based on modification times, * ETags, and selectively downloading a range of an object. */ System.out.println("Downloading an object"); S3Object object = s3.getObject(new GetObjectRequest(bucketName, key)); System.out.println("Content-Type: " + object.getObjectMetadata().getContentType()); displayTextInputStream(object.getObjectContent()); /* * List objects in your bucket by prefix - There are many options for * listing the objects in your bucket. Keep in mind that buckets with * many objects might truncate their results when listing their objects, * so be sure to check if the returned object listing is truncated, and * use the AmazonS3.listNextBatchOfObjects(...) operation to retrieve * additional results. */ System.out.println("Listing objects"); ObjectListing objectListing = s3 .listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix("My")); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { System.out.println( " - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")"); } System.out.println(); /* * Delete an object - Unless versioning has been turned on for your bucket, * there is no way to undelete an object, so use caution when deleting objects. */ // System.out.println("Deleting an object\n"); // s3.deleteObject(bucketName, key); /* * Delete a bucket - A bucket must be completely empty before it can be * deleted, so remember to delete any objects from your buckets before * you try to delete them. */ // System.out.println("Deleting bucket " + bucketName + "\n"); // s3.deleteBucket(bucketName); } catch (AmazonServiceException ase) { System.out.println("Caught an AmazonServiceException, which means your request made it " + "to Amazon S3, but was rejected with an error response for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { System.out.println("Caught an AmazonClientException, which means the client encountered " + "a serious internal problem while trying to communicate with S3, " + "such as not being able to access the network."); System.out.println("Error Message: " + ace.getMessage()); } }