List of usage examples for com.amazonaws.services.s3.model S3Object getObjectContent
public S3ObjectInputStream getObjectContent()
From source file:com.uiintl.backup.agent.samples.S3Sample.java
License:Open Source License
public static void main2(String[] args) throws IOException { /*/* w w w. j a va 2s . c o m*/ * This credentials provider implementation loads your AWS credentials * from a properties file at the root of your classpath. * * Important: Be sure to fill in your AWS access credentials in the * AwsCredentials.properties file before you try to run this * sample. * http://aws.amazon.com/security-credentials */ AmazonS3 s3 = new AmazonS3Client(new ClasspathPropertiesFileCredentialsProvider()); Region usWest2 = Region.getRegion(Regions.AP_SOUTHEAST_2); s3.setRegion(usWest2); String bucketName = "my-first-s3-bucket-" + UUID.randomUUID(); String key = "MyObjectKey"; System.out.println("==========================================="); System.out.println("Getting Started with Amazon S3"); System.out.println("===========================================\n"); try { /* * Create a new S3 bucket - Amazon S3 bucket names are globally unique, * so once a bucket name has been taken by any user, you can't create * another bucket with that same name. * * You can optionally specify a location for your bucket if you want to * keep your data closer to your applications or users. */ System.out.println("Creating bucket " + bucketName + "\n"); s3.createBucket(bucketName); /* * List the buckets in your account */ System.out.println("Listing buckets"); for (Bucket bucket : s3.listBuckets()) { System.out.println(" - " + bucket.getName()); } System.out.println(); /* * Upload an object to your bucket - You can easily upload a file to * S3, or upload directly an InputStream if you know the length of * the data in the stream. You can also specify your own metadata * when uploading to S3, which allows you set a variety of options * like content-type and content-encoding, plus additional metadata * specific to your applications. */ System.out.println("Uploading a new object to S3 from a file\n"); s3.putObject(new PutObjectRequest(bucketName, key, createSampleFile())); /* * Download an object - When you download an object, you get all of * the object's metadata and a stream from which to read the contents. * It's important to read the contents of the stream as quickly as * possibly since the data is streamed directly from Amazon S3 and your * network connection will remain open until you read all the data or * close the input stream. * * GetObjectRequest also supports several other options, including * conditional downloading of objects based on modification times, * ETags, and selectively downloading a range of an object. */ System.out.println("Downloading an object"); S3Object object = s3.getObject(new GetObjectRequest(bucketName, key)); System.out.println("Content-Type: " + object.getObjectMetadata().getContentType()); displayTextInputStream(object.getObjectContent()); /* * List objects in your bucket by prefix - There are many options for * listing the objects in your bucket. Keep in mind that buckets with * many objects might truncate their results when listing their objects, * so be sure to check if the returned object listing is truncated, and * use the AmazonS3.listNextBatchOfObjects(...) operation to retrieve * additional results. */ System.out.println("Listing objects"); ObjectListing objectListing = s3 .listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix("My")); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { System.out.println( " - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")"); } System.out.println(); /* * Delete an object - Unless versioning has been turned on for your bucket, * there is no way to undelete an object, so use caution when deleting objects. */ System.out.println("Deleting an object\n"); s3.deleteObject(bucketName, key); /* * Delete a bucket - A bucket must be completely empty before it can be * deleted, so remember to delete any objects from your buckets before * you try to delete them. */ System.out.println("Deleting bucket " + bucketName + "\n"); s3.deleteBucket(bucketName); } catch (AmazonServiceException ase) { System.out.println("Caught an AmazonServiceException, which means your request made it " + "to Amazon S3, but was rejected with an error response for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { System.out.println("Caught an AmazonClientException, which means the client encountered " + "a serious internal problem while trying to communicate with S3, " + "such as not being able to access the network."); System.out.println("Error Message: " + ace.getMessage()); } }
From source file:com.universal.storage.UniversalS3Storage.java
License:Open Source License
/** * This method retrieves a file from the storage. * The method will retrieve the file according to the passed path. * A file will be stored within the settings' tmp folder. * //from ww w . j av a 2s . c o m * @param path in context. * @returns a file pointing to the retrieved file. */ public File retrieveFile(String path) throws UniversalIOException { PathValidator.validatePath(path); if ("".equals(path.trim())) { return null; } if (path.trim().endsWith("/")) { UniversalIOException error = new UniversalIOException( "Invalid path. Looks like you're trying to retrieve a folder."); this.triggerOnErrorListeners(error); throw error; } File dest = null; InputStream objectData = null; try { S3Object object = s3client.getObject(new GetObjectRequest(this.settings.getRoot(), path)); objectData = object.getObjectContent(); String name = object.getKey(); int index = name.lastIndexOf("/"); if (index != -1) { name = name.substring(index); } dest = new File(FileUtil.completeFileSeparator(this.settings.getTmp()) + name); FileUtils.copyInputStreamToFile(objectData, dest); } catch (Exception e) { UniversalIOException error = new UniversalIOException(e.getMessage()); this.triggerOnErrorListeners(error); throw error; } finally { if (objectData != null) { try { objectData.close(); } catch (Exception ignore) { } } } return dest; }
From source file:com.universal.storage.UniversalS3Storage.java
License:Open Source License
/** * This method retrieves a file from the storage as InputStream. * The method will retrieve the file according to the passed path. * A file will be stored within the settings' tmp folder. * /* ww w. ja v a 2 s. co m*/ * @param path in context. * @returns an InputStream pointing to the retrieved file. */ public InputStream retrieveFileAsStream(String path) throws UniversalIOException { PathValidator.validatePath(path); if ("".equals(path.trim())) { return null; } if (path.trim().endsWith("/")) { UniversalIOException error = new UniversalIOException( "Invalid path. Looks like you're trying to retrieve a folder."); this.triggerOnErrorListeners(error); throw error; } try { S3Object object = s3client.getObject(new GetObjectRequest(this.settings.getRoot(), path)); return object.getObjectContent(); } catch (Exception e) { UniversalIOException error = new UniversalIOException(e.getMessage()); this.triggerOnErrorListeners(error); throw error; } }
From source file:com.upplication.s3fs.util.AmazonS3ClientMock.java
License:Open Source License
@Override public CopyObjectResult copyObject(String sourceBucketName, String sourceKey, String destinationBucketName, String destinationKey) throws AmazonClientException { S3Element element = find(sourceBucketName, sourceKey); if (element != null) { S3Object objectSource = element.getS3Object(); // copy object with S3Object resObj = new S3Object(); resObj.setBucketName(destinationBucketName); resObj.setKey(destinationKey);//ww w .java 2s . co m resObj.setObjectContent(objectSource.getObjectContent()); resObj.setObjectMetadata(objectSource.getObjectMetadata()); resObj.setRedirectLocation(objectSource.getRedirectLocation()); // copy permission AccessControlList permission = new AccessControlList(); permission.setOwner(element.getPermission().getOwner()); permission.grantAllPermissions(element.getPermission().getGrants().toArray(new Grant[0])); S3Element elementResult = new S3Element(resObj, permission, sourceKey.endsWith("/")); // TODO: add should replace existing objects.get(find(destinationBucketName)).remove(elementResult); objects.get(find(destinationBucketName)).add(elementResult); return new CopyObjectResult(); } throw new AmazonServiceException("object source not found"); }
From source file:com.upplication.s3fs.util.S3ObjectSummaryLookup.java
License:Open Source License
/** * get s3Object with S3Object#getObjectContent closed * @param bucket String bucket/*from www. j a va2s . co m*/ * @param key String key * @param client AmazonS3Client client * @return S3Object */ private S3Object getS3Object(String bucket, String key, AmazonS3Client client) { try { S3Object object = client.getObject(bucket, key); if (object.getObjectContent() != null) { try { object.getObjectContent().close(); } catch (IOException e) { log.debug("Error while closing S3Object for bucket: `{}` and key: `{}` -- Cause: {}", bucket, key, e.getMessage()); } } return object; } catch (AmazonS3Exception e) { if (e.getStatusCode() != 404) { throw e; } return null; } }
From source file:com.yahoo.athenz.auth.impl.aws.AwsPrivateKeyStore.java
License:Apache License
String getDecryptedData(final String bucketName, final String keyName) { String keyValue = ""; S3Object s3Object = s3.getObject(bucketName, keyName); if (LOG.isDebugEnabled()) { LOG.debug("retrieving appName {}, key {}", bucketName, keyName); }//from w ww.j a v a 2 s .c om if (null == s3Object) { LOG.error("error retrieving key {}, from bucket {}", keyName, bucketName); return keyValue; } try (S3ObjectInputStream s3InputStream = s3Object.getObjectContent(); ByteArrayOutputStream result = new ByteArrayOutputStream();) { byte[] buffer = new byte[1024]; int length; while ((length = s3InputStream.read(buffer)) != -1) { result.write(buffer, 0, length); } // if key should be decrypted, do so with KMS if (kmsDecrypt) { DecryptRequest req = new DecryptRequest().withCiphertextBlob(ByteBuffer.wrap(result.toByteArray())); ByteBuffer plainText = kms.decrypt(req).getPlaintext(); keyValue = new String(plainText.array()); } else { keyValue = result.toString(); } } catch (IOException e) { LOG.error("error getting application secret.", e); } return keyValue.trim(); }
From source file:com.yahoo.athenz.zts.store.impl.S3ChangeLogStore.java
License:Apache License
SignedDomain getSignedDomain(AmazonS3 s3, String domainName) {
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("AWSS3ChangeLog: getting signed domain {}", domainName);
}// w w w . j a va 2 s. c o m
SignedDomain signedDomain = null;
try {
S3Object object = s3.getObject(s3BucketName, domainName);
try (S3ObjectInputStream s3is = object.getObjectContent()) {
byte[] data = ByteStreams.toByteArray(s3is);
signedDomain = JSON.fromBytes(data, SignedDomain.class);
}
} catch (Exception ex) {
LOGGER.error("AWSS3ChangeLog: getSignedDomain - unable to get domain {} error: {}", domainName,
ex.getMessage());
}
return signedDomain;
}
From source file:com.yahoo.athenz.zts.store.s3.S3ChangeLogStore.java
License:Apache License
SignedDomain getSignedDomain(AmazonS3 s3, String domainName) {
SignedDomain signedDomain = null;// ww w. j a va2 s .c o m
try {
S3Object object = s3.getObject(new GetObjectRequest(s3BucketName, domainName));
if (object == null) {
LOGGER.error("AWSS3ChangeLog: getSignedDomain - domain not found " + domainName);
return null;
}
BufferedReader reader = new BufferedReader(new InputStreamReader(object.getObjectContent()));
StringBuilder data = new StringBuilder();
String line;
while ((line = reader.readLine()) != null) {
data.append(line);
}
reader.close();
signedDomain = JSON.fromString(data.toString(), SignedDomain.class);
} catch (Exception ex) {
LOGGER.error("AWSS3ChangeLog: getSignedDomain - unable to get domain " + domainName + " error: "
+ ex.getMessage());
}
return signedDomain;
}
From source file:com.yahoo.ycsb.utils.connection.S3Connection.java
License:Open Source License
public byte[] read(String key) throws InterruptedException { //long starttime = System.currentTimeMillis(); byte[] bytes = null; try {/* w ww . jav a2s. c om*/ GetObjectRequest getObjectRequest = null; GetObjectMetadataRequest getObjectMetadataRequest = null; if (ssecKey != null) { getObjectRequest = new GetObjectRequest(bucket, key).withSSECustomerKey(ssecKey); getObjectMetadataRequest = new GetObjectMetadataRequest(bucket, key).withSSECustomerKey(ssecKey); } else { getObjectRequest = new GetObjectRequest(bucket, key); getObjectMetadataRequest = new GetObjectMetadataRequest(bucket, key); } //System.out.println("Get object " + key + " from " + bucket); S3Object object = awsClient.getObject(getObjectRequest); ObjectMetadata objectMetadata = awsClient.getObjectMetadata(getObjectMetadataRequest); //System.out.println("Get object " + key + " from " + bucket + " OK"); InputStream objectData = object.getObjectContent(); //consuming the stream // writing the stream to bytes and to results int sizeOfFile = (int) objectMetadata.getContentLength(); bytes = new byte[sizeOfFile]; int offset = 0; while (offset < sizeOfFile) { int chunk_size; //read in 4k chunks chunk_size = sizeOfFile - offset > 4096 ? 4096 : sizeOfFile - offset; int nr_bytes_read = objectData.read(bytes, offset, sizeOfFile - offset); offset = offset + nr_bytes_read; if (Thread.interrupted()) { //System.out.println("interrupt " + key); objectData.close(); throw new InterruptedException(); } } //int nr_bytes_read = objectData.read(bytes, 0, sizeOfFile); objectData.close(); } catch (IOException e) { logger.warn("Not possible to get the object " + key); } //long endtime = System.currentTimeMillis(); //System.out.println("ReadS3: " + key + " " + (endtime - starttime) + " " + region); return bytes; }
From source file:com.zero_x_baadf00d.play.module.aws.s3.ebean.BaseS3FileModel.java
License:Open Source License
/** * Get the file content. In case of error (network error, file not * found, ...), this method will return null. * * @return The file content, otherwise, null * @see InputStream// www. ja va 2 s . co m * @since 16.03.13 */ public InputStream getFileContent() { if (!PlayS3.isReady()) { Logger.error("Could not get PlayS3 file content because amazonS3 variable is null"); throw new RuntimeException("Could not get file content"); } final S3Object obj = PlayS3.getAmazonS3().getObject(this.bucket, getActualFileName()); if (obj != null) { return obj.getObjectContent(); } return null; }