List of usage examples for com.amazonaws.services.s3.model ListObjectsRequest ListObjectsRequest
public ListObjectsRequest(String bucketName, String prefix, String marker, String delimiter, Integer maxKeys)
From source file:io.konig.camel.aws.s3.DeleteObjectEndpoint.java
License:Apache License
@Override public void doStart() throws Exception { super.doStart(); String envName = ""; if (System.getProperty("environmentName") != null) { envName = System.getProperty("environmentName"); }// w ww . j a va2s.c o m String bucketName = getConfiguration().getBucketName(); bucketName = bucketName.replace("${environmentName}", envName); configuration.setBucketName(bucketName); configuration.getAmazonS3Client().setRegion(Region.getRegion(Regions.fromName(configuration.getRegion()))); s3Client = configuration.getAmazonS3Client() != null ? configuration.getAmazonS3Client() : S3ClientFactory.getAWSS3Client(configuration, getMaxConnections()).getS3Client(); String fileName = getConfiguration().getFileName(); if (fileName != null) { LOG.trace("File name [{}] requested, so skipping bucket check...", fileName); return; } LOG.trace("Querying whether bucket [{}] already exists...", bucketName); String prefix = getConfiguration().getPrefix(); try { s3Client.listObjects(new ListObjectsRequest(bucketName, prefix, null, null, 0)); LOG.trace("Bucket [{}] already exists", bucketName); return; } catch (AmazonServiceException ase) { /* 404 means the bucket doesn't exist */ if (ase.getStatusCode() != 404) { throw ase; } } LOG.trace("Bucket [{}] doesn't exist yet", bucketName); // creates the new bucket because it doesn't exist yet CreateBucketRequest createBucketRequest = new CreateBucketRequest(getConfiguration().getBucketName()); //LOG.trace("Creating bucket [{}] in region [{}] with request [{}]...", configuration.getBucketName(), configuration.getRegion(), createBucketRequest); s3Client.createBucket(createBucketRequest); LOG.trace("Bucket created"); if (configuration.getPolicy() != null) { LOG.trace("Updating bucket [{}] with policy [{}]", bucketName, configuration.getPolicy()); s3Client.setBucketPolicy(bucketName, configuration.getPolicy()); LOG.trace("Bucket policy updated"); } }
From source file:jenkins.plugins.itemstorage.s3.S3Profile.java
License:Open Source License
public boolean exists(String bucketName, String path) { ObjectListing objectListing = helper.client() .listObjects(new ListObjectsRequest(bucketName, path, null, null, 1)); return !objectListing.getObjectSummaries().isEmpty(); }
From source file:n3phele.storage.s3.CloudStorageImpl.java
License:Open Source License
public boolean createBucket(Repository repo) throws ForbiddenException { Credential credential = repo.getCredential().decrypt(); AmazonS3Client s3 = new AmazonS3Client( new BasicAWSCredentials(credential.getAccount(), credential.getSecret())); s3.setEndpoint(repo.getTarget().toString()); try {//from w ww .java 2 s . c om try { s3.listObjects(new ListObjectsRequest(repo.getRoot(), null, null, null, 0)); // it exists and the current account owns it return false; } catch (AmazonServiceException ase) { switch (ase.getStatusCode()) { case 403: /* * A permissions error means the bucket exists, but is owned by * another account. */ throw new ForbiddenException( "Bucket " + repo.getRoot() + " has already been created by another user."); case 404: Bucket bucket = s3.createBucket(repo.getRoot()); log.info("Bucket created " + bucket.getName()); return true; default: throw ase; } } } catch (AmazonServiceException e) { log.log(Level.WARNING, "Service Error processing " + repo, e); } catch (AmazonClientException e) { log.log(Level.SEVERE, "Client Error processing " + repo, e); } return false; }
From source file:n3phele.storage.s3.CloudStorageImpl.java
License:Open Source License
public List<FileNode> getFileList(Repository repo, String prefix, int max) { List<FileNode> result = new ArrayList<FileNode>(); Credential credential = repo.getCredential().decrypt(); AmazonS3Client s3 = new AmazonS3Client( new BasicAWSCredentials(credential.getAccount(), credential.getSecret())); s3.setEndpoint(repo.getTarget().toString()); Policy p = null;/*from ww w.j av a 2 s . co m*/ try { BucketPolicy bp = s3.getBucketPolicy(repo.getRoot()); log.info("Policy text " + bp.getPolicyText()); if (bp != null && bp.getPolicyText() != null) { p = PolicyHelper.parse(bp.getPolicyText()); log.info("Policy object is " + (p == null ? null : p.toJson())); } } catch (Exception e) { log.log(Level.WARNING, "Policy not supported", e); } try { ObjectListing s3Objects = s3 .listObjects(new ListObjectsRequest(repo.getRoot(), prefix, null, "/", max)); if (s3Objects.getCommonPrefixes() != null) { for (String dirName : s3Objects.getCommonPrefixes()) { String name = dirName; if (dirName.endsWith("/")) { name = dirName.substring(0, dirName.length() - 1); } boolean isPublic = isPublicFolder(p, repo.getRoot(), name); name = name.substring(name.lastIndexOf("/") + 1); FileNode folder = FileNode.newFolder(name, prefix, repo, isPublic); log.info("Folder:" + folder); result.add(folder); } } if (s3Objects.getObjectSummaries() != null) { for (S3ObjectSummary fileSummary : s3Objects.getObjectSummaries()) { String key = fileSummary.getKey(); if (key != null && !key.equals(prefix)) { String name = key.substring(key.lastIndexOf("/") + 1); UriBuilder builder = UriBuilder.fromUri(repo.getTarget()).path(repo.getRoot()); if (prefix != null && !prefix.isEmpty()) { builder = builder.path(prefix); } FileNode file = FileNode.newFile(name, prefix, repo, fileSummary.getLastModified(), fileSummary.getSize(), builder.path(name).toString()); log.info("File:" + file); result.add(file); } } } } catch (AmazonServiceException e) { log.log(Level.WARNING, "Service Error processing " + repo, e); } catch (AmazonClientException e) { log.log(Level.SEVERE, "Client Error processing " + repo, e); } return result; }
From source file:org.apache.camel.component.aws.s3.S3Endpoint.java
License:Apache License
@Override public void doStart() throws Exception { super.doStart(); String fileName = getConfiguration().getFileName(); if (fileName != null) { LOG.trace("File name [{}] requested, so skipping bucket check...", fileName); return;/*from ww w . ja v a2s . c om*/ } String bucketName = getConfiguration().getBucketName(); LOG.trace("Quering whether bucket [{}] already exists...", bucketName); try { getS3Client().listObjects(new ListObjectsRequest(bucketName, null, null, null, 0)); LOG.trace("Bucket [{}] already exists", bucketName); return; } catch (AmazonServiceException ase) { /* 404 means the bucket doesn't exist */ if (ase.getStatusCode() != 404) { throw ase; } } LOG.trace("Bucket [{}] doesn't exist yet", bucketName); // creates the new bucket because it doesn't exist yet CreateBucketRequest createBucketRequest = new CreateBucketRequest(getConfiguration().getBucketName()); if (getConfiguration().getRegion() != null) { createBucketRequest.setRegion(getConfiguration().getRegion()); } LOG.trace("Creating bucket [{}] in region [{}] with request [{}]...", new Object[] { configuration.getBucketName(), configuration.getRegion(), createBucketRequest }); getS3Client().createBucket(createBucketRequest); LOG.trace("Bucket created"); if (configuration.getPolicy() != null) { LOG.trace("Updating bucket [{}] with policy [{}]", bucketName, configuration.getPolicy()); getS3Client().setBucketPolicy(bucketName, configuration.getPolicy()); LOG.trace("Bucket policy updated"); } }
From source file:org.duracloud.s3storage.S3StorageProvider.java
License:Apache License
private List<S3ObjectSummary> listObjects(String bucketName, String prefix, long maxResults, String marker) { int numResults = new Long(maxResults).intValue(); ListObjectsRequest request = new ListObjectsRequest(bucketName, prefix, marker, null, numResults); try {/*w w w. j a v a 2 s. c o m*/ ObjectListing objectListing = s3Client.listObjects(request); return objectListing.getObjectSummaries(); } catch (AmazonClientException e) { String err = "Could not get contents of S3 bucket " + bucketName + " due to error: " + e.getMessage(); throw new StorageException(err, e, RETRY); } }
From source file:org.mule.module.s3.S3Connector.java
License:Open Source License
/** * Lazily lists all objects for a given prefix. As S3 does not limit in any * way the number of objects, such listing can retrieve an arbitrary amount * of objects, and may need to perform extra calls to the api while it is * iterated./*from w ww .j a v a2 s .com*/ * * {@sample.xml ../../../doc/mule-module-s3.xml.sample s3:list-objects} * * @param bucketName * the target bucket's name * @param prefix * the prefix of the objects to be listed. If unspecified, all * objects are listed * @param marker * where in the bucket to begin listing. The list will only * include keys that occur lexicographically after the marker * @param delimiter * causes keys that contain the same string between a prefix and * the first occurrence of the delimiter to be rolled up into a * single result element. These rolled-up keys are not returned * elsewhere in the response. The most commonly used delimiter is * "/", which simulates a hierarchical organization similar to a * file system directory structure. * @param maxKeys * The maximum number of keys to include in the response. If * maxKeys is not specified, Amazon S3 will limit the number of * results in the response. * @param encodingType * The encoding method to be applied on the response. An object * key can contain any Unicode character; however, XML 1.0 parser * cannot parse some characters, such as characters with an ASCII * value from 0 to 10. For characters that are not supported in * XML 1.0, you can add this parameter to request that Amazon S3 * encode the keys in the response. * @return An iterable */ @Processor public Iterable<S3ObjectSummary> listObjects(String bucketName, @Optional String prefix, @Optional String marker, @Optional String delimiter, @Optional Integer maxKeys, @Default("NOT_ENCODED") EncodingType encodingType) { String encodingTypeString = null; if (encodingType != null) { encodingTypeString = encodingType.sdkValue(); } ListObjectsRequest request = new ListObjectsRequest(bucketName, prefix, marker, delimiter, maxKeys) .withEncodingType(encodingTypeString); return client.listObjects(request); }
From source file:org.onebusaway.admin.service.impl.S3FileServiceImpl.java
License:Apache License
@Override /**//from w w w . j ava 2 s. c o m * check to see if the given bundle directory exists in the configured bucket. * Do not include leading slashes in the filename(key). */ public boolean bundleDirectoryExists(String filename) { ListObjectsRequest request = new ListObjectsRequest(_bucketName, filename, null, null, 1); ObjectListing listing = _s3.listObjects(request); return listing.getObjectSummaries().size() > 0; }
From source file:org.onebusaway.admin.service.impl.S3FileServiceImpl.java
License:Apache License
@Override /**/*from w ww. j a v a 2s .c o m*/ * Return tabular data (filename, flag, modified date) about bundle directories. */ public List<String[]> listBundleDirectories(int maxResults) { List<String[]> rows = new ArrayList<String[]>(); HashMap<String, String> map = new HashMap<String, String>(); ListObjectsRequest request = new ListObjectsRequest(_bucketName, null, null, "/", maxResults); ObjectListing listing = null; do { if (listing == null) { listing = _s3.listObjects(request); if (listing.getCommonPrefixes() != null) { // short circuit if common prefixes works List<String> commonPrefixes = listing.getCommonPrefixes(); for (String key : commonPrefixes) { Date lastModified = getLastModifiedTimeForKey(key); String lastModifiedStr = "n/a"; if (lastModified != null) { lastModifiedStr = "" + lastModified.toString(); } String[] columns = { parseKey(key), getStatus(key), lastModifiedStr }; rows.add(columns); } return rows; } _log.error("prefixes=" + listing.getCommonPrefixes()); } else { listing = _s3.listNextBatchOfObjects(listing); } for (S3ObjectSummary summary : listing.getObjectSummaries()) { String key = parseKey(summary.getKey()); if (!map.containsKey(key)) { String[] columns = { key, " ", "" + summary.getLastModified().getTime() }; rows.add(columns); map.put(key, key); } } } while (listing.isTruncated()); return rows; }
From source file:org.onebusaway.admin.service.impl.S3FileServiceImpl.java
License:Apache License
private Date getLastModifiedTimeForKey(String key) { ListObjectsRequest request = new ListObjectsRequest(_bucketName, key, null, "/", 1); ObjectListing listing = _s3.listObjects(request); if (!listing.getObjectSummaries().isEmpty()) return listing.getObjectSummaries().get(0).getLastModified(); return null;// w ww.ja va 2 s .com }