List of usage examples for com.amazonaws.services.s3.model S3ObjectSummary getKey
public String getKey()
From source file:com.treasure_data.td_import.source.S3Source.java
License:Apache License
public static List<Source> createSources(SourceDesc desc) { String rawPath = desc.getPath(); String[] elm = rawPath.split("/"); String bucket = elm[0];//from w w w . j a va2 s .c o m String basePath = rawPath.substring(bucket.length() + 1, rawPath.length()); System.setProperty("com.amazonaws.sdk.disableCertChecking", "1"); AmazonS3Client client = createAmazonS3Client(desc); List<S3ObjectSummary> s3objects = getSources(client, bucket, basePath); List<Source> srcs = new ArrayList<Source>(); for (S3ObjectSummary s3object : s3objects) { LOG.info(String.format("create s3-src s3object=%s, rawPath=%s", s3object.getKey(), rawPath)); srcs.add(new S3Source(createAmazonS3Client(desc), rawPath, s3object)); } return srcs; }
From source file:com.treasure_data.td_import.source.S3Source.java
License:Apache License
static List<S3ObjectSummary> filterSources(List<S3ObjectSummary> s3objects, String basePath) { String regex = basePath.replace("*", "([^\\s]*)"); Pattern pattern = Pattern.compile(regex); LOG.info(String.format("regex matching: regex=%s", regex)); List<S3ObjectSummary> matched = new ArrayList<S3ObjectSummary>(); for (S3ObjectSummary s3object : s3objects) { Matcher m = pattern.matcher(s3object.getKey()); if (m.matches()) { matched.add(s3object);/* w w w . j av a 2s .c om*/ } } return matched; }
From source file:com.treasure_data.td_import.source.S3Source.java
License:Apache License
S3Source(AmazonS3Client client, String rawPath, S3ObjectSummary s3object) { super("s3://" + s3object.getBucketName() + "/" + s3object.getKey()); this.client = client; this.bucket = s3object.getBucketName(); this.key = s3object.getKey(); this.size = s3object.getSize(); this.rawPath = rawPath; }
From source file:com.ub.ml.S3Sample.java
License:Open Source License
public static void main(String[] args) throws IOException { /*//from w w w . j av a2 s. c om * Create your credentials file at ~/.aws/credentials (C:\Users\USER_NAME\.aws\credentials for Windows users) * and save the following lines after replacing the underlined values with your own. * * [default] * aws_access_key_id = YOUR_ACCESS_KEY_ID * aws_secret_access_key = YOUR_SECRET_ACCESS_KEY */ AmazonS3 s3 = new AmazonS3Client(); Region usWest2 = Region.getRegion(Regions.US_EAST_1); s3.setRegion(usWest2); String bucketName = "my-first-s3-bucket-" + UUID.randomUUID(); String key = "MyObjectKey"; System.out.println("==========================================="); System.out.println("Getting Started with Amazon S3"); System.out.println("===========================================\n"); try { /* * Create a new S3 bucket - Amazon S3 bucket names are globally unique, * so once a bucket name has been taken by any user, you can't create * another bucket with that same name. * * You can optionally specify a location for your bucket if you want to * keep your data closer to your applications or users. */ System.out.println("Creating bucket " + bucketName + "\n"); s3.createBucket(bucketName); /* * List the buckets in your account */ System.out.println("Listing buckets"); for (Bucket bucket : s3.listBuckets()) { System.out.println(" - " + bucket.getName()); } System.out.println(); /* * Upload an object to your bucket - You can easily upload a file to * S3, or upload directly an InputStream if you know the length of * the data in the stream. You can also specify your own metadata * when uploading to S3, which allows you set a variety of options * like content-type and content-encoding, plus additional metadata * specific to your applications. */ System.out.println("Uploading a new object to S3 from a file\n"); s3.putObject(new PutObjectRequest(bucketName, key, createSampleFile())); /* * Download an object - When you download an object, you get all of * the object's metadata and a stream from which to read the contents. * It's important to read the contents of the stream as quickly as * possibly since the data is streamed directly from Amazon S3 and your * network connection will remain open until you read all the data or * close the input stream. * * GetObjectRequest also supports several other options, including * conditional downloading of objects based on modification times, * ETags, and selectively downloading a range of an object. */ System.out.println("Downloading an object"); S3Object object = s3.getObject(new GetObjectRequest(bucketName, key)); System.out.println("Content-Type: " + object.getObjectMetadata().getContentType()); displayTextInputStream(object.getObjectContent()); /* * List objects in your bucket by prefix - There are many options for * listing the objects in your bucket. Keep in mind that buckets with * many objects might truncate their results when listing their objects, * so be sure to check if the returned object listing is truncated, and * use the AmazonS3.listNextBatchOfObjects(...) operation to retrieve * additional results. */ System.out.println("Listing objects"); ObjectListing objectListing = s3 .listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix("My")); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { System.out.println( " - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")"); } System.out.println(); /* * Delete an object - Unless versioning has been turned on for your bucket, * there is no way to undelete an object, so use caution when deleting objects. */ //System.out.println("Deleting an object\n"); //s3.deleteObject(bucketName, key); /* * Delete a bucket - A bucket must be completely empty before it can be * deleted, so remember to delete any objects from your buckets before * you try to delete them. */ //System.out.println("Deleting bucket " + bucketName + "\n"); //s3.deleteBucket(bucketName); } catch (AmazonServiceException ase) { System.out.println("Caught an AmazonServiceException, which means your request made it " + "to Amazon S3, but was rejected with an error response for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { System.out.println("Caught an AmazonClientException, which means the client encountered " + "a serious internal problem while trying to communicate with S3, " + "such as not being able to access the network."); System.out.println("Error Message: " + ace.getMessage()); } }
From source file:com.uiintl.backup.agent.samples.S3Sample.java
License:Open Source License
public static void main2(String[] args) throws IOException { /*// ww w. jav a 2s .c o m * This credentials provider implementation loads your AWS credentials * from a properties file at the root of your classpath. * * Important: Be sure to fill in your AWS access credentials in the * AwsCredentials.properties file before you try to run this * sample. * http://aws.amazon.com/security-credentials */ AmazonS3 s3 = new AmazonS3Client(new ClasspathPropertiesFileCredentialsProvider()); Region usWest2 = Region.getRegion(Regions.AP_SOUTHEAST_2); s3.setRegion(usWest2); String bucketName = "my-first-s3-bucket-" + UUID.randomUUID(); String key = "MyObjectKey"; System.out.println("==========================================="); System.out.println("Getting Started with Amazon S3"); System.out.println("===========================================\n"); try { /* * Create a new S3 bucket - Amazon S3 bucket names are globally unique, * so once a bucket name has been taken by any user, you can't create * another bucket with that same name. * * You can optionally specify a location for your bucket if you want to * keep your data closer to your applications or users. */ System.out.println("Creating bucket " + bucketName + "\n"); s3.createBucket(bucketName); /* * List the buckets in your account */ System.out.println("Listing buckets"); for (Bucket bucket : s3.listBuckets()) { System.out.println(" - " + bucket.getName()); } System.out.println(); /* * Upload an object to your bucket - You can easily upload a file to * S3, or upload directly an InputStream if you know the length of * the data in the stream. You can also specify your own metadata * when uploading to S3, which allows you set a variety of options * like content-type and content-encoding, plus additional metadata * specific to your applications. */ System.out.println("Uploading a new object to S3 from a file\n"); s3.putObject(new PutObjectRequest(bucketName, key, createSampleFile())); /* * Download an object - When you download an object, you get all of * the object's metadata and a stream from which to read the contents. * It's important to read the contents of the stream as quickly as * possibly since the data is streamed directly from Amazon S3 and your * network connection will remain open until you read all the data or * close the input stream. * * GetObjectRequest also supports several other options, including * conditional downloading of objects based on modification times, * ETags, and selectively downloading a range of an object. */ System.out.println("Downloading an object"); S3Object object = s3.getObject(new GetObjectRequest(bucketName, key)); System.out.println("Content-Type: " + object.getObjectMetadata().getContentType()); displayTextInputStream(object.getObjectContent()); /* * List objects in your bucket by prefix - There are many options for * listing the objects in your bucket. Keep in mind that buckets with * many objects might truncate their results when listing their objects, * so be sure to check if the returned object listing is truncated, and * use the AmazonS3.listNextBatchOfObjects(...) operation to retrieve * additional results. */ System.out.println("Listing objects"); ObjectListing objectListing = s3 .listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix("My")); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { System.out.println( " - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")"); } System.out.println(); /* * Delete an object - Unless versioning has been turned on for your bucket, * there is no way to undelete an object, so use caution when deleting objects. */ System.out.println("Deleting an object\n"); s3.deleteObject(bucketName, key); /* * Delete a bucket - A bucket must be completely empty before it can be * deleted, so remember to delete any objects from your buckets before * you try to delete them. */ System.out.println("Deleting bucket " + bucketName + "\n"); s3.deleteBucket(bucketName); } catch (AmazonServiceException ase) { System.out.println("Caught an AmazonServiceException, which means your request made it " + "to Amazon S3, but was rejected with an error response for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { System.out.println("Caught an AmazonClientException, which means the client encountered " + "a serious internal problem while trying to communicate with S3, " + "such as not being able to access the network."); System.out.println("Error Message: " + ace.getMessage()); } }
From source file:com.universal.storage.UniversalS3Storage.java
License:Open Source License
/** * This method wipes the root folder of a storage, basically, will remove all files and folder in it. * Be careful with this method because in too many cases this action won't provide a rollback action. * /*from www . j a va2s.co m*/ * This method loops over the versions for deletion, the bucket will be empty and without any version of its objects. */ public void wipe() throws UniversalIOException { ObjectListing object_listing = this.s3client.listObjects(this.settings.getRoot()); while (true) { for (Iterator<?> iterator = object_listing.getObjectSummaries().iterator(); iterator.hasNext();) { S3ObjectSummary summary = (S3ObjectSummary) iterator.next(); this.s3client.deleteObject(this.settings.getRoot(), summary.getKey()); } if (object_listing.isTruncated()) { object_listing = this.s3client.listNextBatchOfObjects(object_listing); } else { break; } } ; VersionListing version_listing = this.s3client .listVersions(new ListVersionsRequest().withBucketName(this.settings.getRoot())); while (true) { for (Iterator<?> iterator = version_listing.getVersionSummaries().iterator(); iterator.hasNext();) { S3VersionSummary vs = (S3VersionSummary) iterator.next(); this.s3client.deleteVersion( this.settings.getRoot(), vs.getKey(), vs.getVersionId()); } if (version_listing.isTruncated()) { version_listing = this.s3client.listNextBatchOfVersions(version_listing); } else { break; } } }
From source file:com.upplication.s3fs.S3FileSystemProvider.java
License:Open Source License
@Override public <A extends BasicFileAttributes> A readAttributes(Path path, Class<A> type, LinkOption... options) throws IOException { Preconditions.checkArgument(path instanceof S3Path, "path must be an instance of %s", S3Path.class.getName()); S3Path s3Path = (S3Path) path; if (type == BasicFileAttributes.class) { S3ObjectSummary objectSummary = s3ObjectSummaryLookup.lookup(s3Path); // parse the data to BasicFileAttributes. FileTime lastModifiedTime = null; if (objectSummary.getLastModified() != null) { lastModifiedTime = FileTime.from(objectSummary.getLastModified().getTime(), TimeUnit.MILLISECONDS); }//from w w w. jav a 2 s . c om long size = objectSummary.getSize(); boolean directory = false; boolean regularFile = false; String key = objectSummary.getKey(); // check if is a directory and exists the key of this directory at amazon s3 if (objectSummary.getKey().equals(s3Path.getKey() + "/") && objectSummary.getKey().endsWith("/")) { directory = true; } // is a directory but not exists at amazon s3 else if ((!objectSummary.getKey().equals(s3Path.getKey()) || "".equals(s3Path.getKey())) && objectSummary.getKey().startsWith(s3Path.getKey())) { directory = true; // no metadata, we fake one size = 0; // delete extra part key = s3Path.getKey() + "/"; } // is a file: else { regularFile = true; } return type.cast(new S3FileAttributes(key, lastModifiedTime, size, directory, regularFile)); } // not support attribute class throw new UnsupportedOperationException(format("only %s supported", BasicFileAttributes.class)); }
From source file:com.upplication.s3fs.S3FileSystemProvider.java
License:Open Source License
/** * Get the Control List, if the path not exists * (because the path is a directory and this key isnt created at amazon s3) * then return the ACL of the first child. */*www . j a va2s. c o m*/ * @param path {@link S3Path} * @return AccessControlList * @throws NoSuchFileException if not found the path and any child */ private AccessControlList getAccessControl(S3Path path) throws NoSuchFileException { S3ObjectSummary obj = s3ObjectSummaryLookup.lookup(path); // check first for file: return path.getFileSystem().getClient().getObjectAcl(obj.getBucketName(), obj.getKey()); }
From source file:com.upplication.s3fs.S3Iterator.java
License:Open Source License
/** * add to the listPath the elements at the same level that s3Path * @param listPath List not null list to add * @param current ObjectListing to walk/*from w ww. jav a 2 s. co m*/ */ private void parseObjectListing(List<S3Path> listPath, ObjectListing current) { // add all the objects i.e. the files for (final S3ObjectSummary objectSummary : current.getObjectSummaries()) { final String key = objectSummary.getKey(); final S3Path path = new S3Path(s3FileSystem, "/" + bucket, key.split("/")); path.setObjectSummary(objectSummary); listPath.add(path); } // add all the common prefixes i.e. the directories for (final String dir : current.getCommonPrefixes()) { if (dir.equals("/")) continue; listPath.add(new S3Path(s3FileSystem, "/" + bucket, dir)); } }
From source file:com.upplication.s3fs.util.S3ObjectSummaryLookup.java
License:Open Source License
private boolean matchName(String fileName, S3ObjectSummary summary) { String foundKey = summary.getKey(); // they are different names return false if (!foundKey.startsWith(fileName)) { return false; }//from w ww .j a v a 2 s. c o m // when they are the same length, they are identical if (foundKey.length() == fileName.length()) return true; return foundKey.charAt(fileName.length()) == '/'; }