List of usage examples for com.amazonaws.services.s3.model ObjectListing getObjectSummaries
public List<S3ObjectSummary> getObjectSummaries()
From source file:com.upplication.s3fs.util.S3ObjectSummaryLookup.java
License:Open Source License
/** * Get the {@link com.amazonaws.services.s3.model.S3ObjectSummary} that represent this Path or her first child if this path not exists * @param s3Path {@link com.upplication.s3fs.S3Path} * @return {@link com.amazonaws.services.s3.model.S3ObjectSummary} * @throws java.nio.file.NoSuchFileException if not found the path and any child *//*from w w w . j a v a 2 s.co m*/ public S3ObjectSummary lookup(S3Path s3Path) throws NoSuchFileException { /* * check is object summary has been cached */ S3ObjectSummary summary = s3Path.fetchObjectSummary(); if (summary != null) { return summary; } final AmazonS3Client client = s3Path.getFileSystem().getClient(); /* * when `key` is an empty string retrieve the object meta-data of the bucket */ if ("".equals(s3Path.getKey())) { ObjectMetadata meta = client.getObjectMetadata(s3Path.getBucket(), ""); if (meta == null) throw new NoSuchFileException("s3://" + s3Path.getBucket()); summary = new S3ObjectSummary(); summary.setBucketName(s3Path.getBucket()); summary.setETag(meta.getETag()); summary.setKey(s3Path.getKey()); summary.setLastModified(meta.getLastModified()); summary.setSize(meta.getContentLength()); // TODO summary.setOwner(?); // TODO summary.setStorageClass(?); return summary; } /* * Lookup for the object summary for the specified object key * by using a `listObjects` request */ String marker = null; while (true) { ListObjectsRequest request = new ListObjectsRequest(); request.setBucketName(s3Path.getBucket()); request.setPrefix(s3Path.getKey()); request.setMaxKeys(250); if (marker != null) request.setMarker(marker); ObjectListing listing = client.listObjects(request); List<S3ObjectSummary> results = listing.getObjectSummaries(); if (results.isEmpty()) { break; } for (S3ObjectSummary item : results) { if (matchName(s3Path.getKey(), item)) { return item; } } if (listing.isTruncated()) marker = listing.getNextMarker(); else break; } throw new NoSuchFileException("s3://" + s3Path.getBucket() + "/" + s3Path.toString()); }
From source file:com.yahoo.athenz.zts.store.impl.S3ChangeLogStore.java
License:Apache License
/** * list the objects in the zts bucket. If the mod time is specified as 0 * then we want to list all objects otherwise, we only list objects * that are newer than the specified timestamp * @param s3 AWS S3 client object// w w w.ja v a2s . co m * @param domains collection to be updated to include domain names * @param modTime only include domains newer than this timestamp */ void listObjects(AmazonS3 s3, Collection<String> domains, long modTime) { if (LOGGER.isDebugEnabled()) { LOGGER.debug("listObjects: Retrieving domains from {} with mod time > {}", s3BucketName, modTime); } ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(s3BucketName)); String objectName; while (objectListing != null) { // process each entry in our result set and add the domain // name to our return list final List<S3ObjectSummary> objectSummaries = objectListing.getObjectSummaries(); boolean listTruncated = objectListing.isTruncated(); if (LOGGER.isDebugEnabled()) { LOGGER.debug("listObjects: retrieved {} objects, more objects available - {}", objectSummaries.size(), listTruncated); } for (S3ObjectSummary objectSummary : objectSummaries) { // if mod time is specified then make sure we automatically skip // any domains older than the specified value if (modTime > 0 && objectSummary.getLastModified().getTime() <= modTime) { continue; } // for now skip any folders/objects that start with '.' objectName = objectSummary.getKey(); if (objectName.charAt(0) == '.') { continue; } domains.add(objectName); } // check if the object listing is truncated or not (break out in this case) // technically we can skip this call and just call listNextBatchOfResults // since that returns null if the object listing is not truncated but // this direct check here makes the logic easier to follow if (!listTruncated) { break; } objectListing = s3.listNextBatchOfObjects(objectListing); } }
From source file:com.yahoo.athenz.zts.store.s3.S3ChangeLogStore.java
License:Apache License
/** * list the objects in the zts bucket. If te mod time is specified as 0 * then we want to list all objects otherwise, we only list objects * that are newer than the specified timestamp * @param s3 AWS S3 client object/*w w w . ja v a2 s. c o m*/ * @param domains collection to be updated to include domain names * @param modTime only include domains newer than this timestamp */ void listObjects(AmazonS3 s3, Collection<String> domains, long modTime) { ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(s3BucketName)); String objectName = null; while (objectListing != null) { // process each entry in our result set and add the domain // name to our return list for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { // if mod time is specified then make sure we automatically skip // any domains older than the specified value if (modTime > 0 && objectSummary.getLastModified().getTime() <= modTime) { continue; } // for now skip any folders/objects that start with '.' objectName = objectSummary.getKey(); if (objectName.charAt(0) == '.') { continue; } domains.add(objectName); } // check if the object listing is truncated or not (break out in this case) // technically we can skip this call and just call listNextBatchOfResults // since that returns null if the object listing is not truncated but // this direct check here makes the logic easier to follow if (!objectListing.isTruncated()) { break; } objectListing = s3.listNextBatchOfObjects(objectListing); } }
From source file:com.yahoo.ycsb.db.S3Client.java
License:Open Source License
/** * Perform an emulation of a database scan operation on a S3 bucket. * * @param bucket//from w w w. ja v a 2 s . co m * The name of the bucket * @param startkey * The file key of the first file to read. * @param recordcount * The number of files to read * @param fields * The list of fields to read, or null for all of them * @param result * A Vector of HashMaps, where each HashMap is a set field/value * pairs for one file * */ protected Status scanFromStorage(String bucket, String startkey, int recordcount, Vector<HashMap<String, ByteIterator>> result, SSECustomerKey ssecLocal) { int counter = 0; ObjectListing listing = s3Client.listObjects(bucket); List<S3ObjectSummary> summaries = listing.getObjectSummaries(); List<String> keyList = new ArrayList(); int startkeyNumber = 0; int numberOfIteration = 0; // getting the list of files in the bucket while (listing.isTruncated()) { listing = s3Client.listNextBatchOfObjects(listing); summaries.addAll(listing.getObjectSummaries()); } for (S3ObjectSummary summary : summaries) { String summaryKey = summary.getKey(); keyList.add(summaryKey); } // Sorting the list of files in Alphabetical order Collections.sort(keyList); // sorting the list // Getting the position of the startingfile for the scan for (String key : keyList) { if (key.equals(startkey)) { startkeyNumber = counter; } else { counter = counter + 1; } } // Checking if the total number of file is bigger than the file to read, // if not using the total number of Files if (recordcount < keyList.size()) { numberOfIteration = recordcount; } else { numberOfIteration = keyList.size(); } // Reading the Files starting from the startkey File till the end // of the Files or Till the recordcount number for (int i = startkeyNumber; i < numberOfIteration; i++) { HashMap<String, ByteIterator> resultTemp = new HashMap<String, ByteIterator>(); readFromStorage(bucket, keyList.get(i), resultTemp, ssecLocal); result.add(resultTemp); } return Status.OK; }
From source file:com.zhang.aws.s3.S3Sample.java
License:Open Source License
public static void main(String[] args) throws IOException { /*/*from w w w . j a va2s . c o m*/ * The ProfileCredentialsProvider will return your [default] * credential profile by reading from the credentials file located at * (~/.aws/credentials). */ ResourceBundle bundle = ResourceBundle.getBundle("credentials"); AWSCredentials credentials = null; try { // credentials = new ProfileCredentialsProvider().getCredentials(); credentials = new BasicAWSCredentials(bundle.getString("aws_access_key_id"), bundle.getString("aws_secret_access_key")); } catch (Exception e) { throw new AmazonClientException("Cannot load the credentials from the credential profiles file. " + "Please make sure that your credentials file is at the correct " + "location (~/.aws/credentials), and is in valid format.", e); } AmazonS3 s3 = new AmazonS3Client(credentials); Region usWest2 = Region.getRegion(Regions.US_WEST_2); s3.setRegion(usWest2); String bucketName = "elasticbeanstalk-us-west-2-948206320069"; String key = "MyObjectKey2"; System.out.println("==========================================="); System.out.println("Getting Started with Amazon S3"); System.out.println("===========================================\n"); try { /* * Create a new S3 bucket - Amazon S3 bucket names are globally unique, * so once a bucket name has been taken by any user, you can't create * another bucket with that same name. * * You can optionally specify a location for your bucket if you want to * keep your data closer to your applications or users. */ System.out.println("Creating bucket " + bucketName + "\n"); // s3.createBucket(bucketName); /* * List the buckets in your account */ System.out.println("Listing buckets"); for (Bucket bucket : s3.listBuckets()) { System.out.println(" - " + bucket.getName()); } System.out.println(); /* * Upload an object to your bucket - You can easily upload a file to * S3, or upload directly an InputStream if you know the length of * the data in the stream. You can also specify your own metadata * when uploading to S3, which allows you set a variety of options * like content-type and content-encoding, plus additional metadata * specific to your applications. */ System.out.println("Uploading a new object to S3 from a file\n"); // s3.putObject(new PutObjectRequest(bucketName, key, createSampleFile())); s3.putObject(new PutObjectRequest(bucketName, key, getFileFromDisk())); /*** * * ?url * */ GeneratePresignedUrlRequest urlRequest = new GeneratePresignedUrlRequest(bucketName, key); URL generatePresignedUrl = s3.generatePresignedUrl(urlRequest); System.out.println("public url:" + generatePresignedUrl.toString()); /* * Download an object - When you download an object, you get all of * the object's metadata and a stream from which to read the contents. * It's important to read the contents of the stream as quickly as * possibly since the data is streamed directly from Amazon S3 and your * network connection will remain open until you read all the data or * close the input stream. * * GetObjectRequest also supports several other options, including * conditional downloading of objects based on modification times, * ETags, and selectively downloading a range of an object. */ // System.out.println("Downloading an object"); // S3Object object = s3.getObject(new GetObjectRequest(bucketName, key)); // System.out.println("Content-Type: " + object.getObjectMetadata().getContentType()); // displayTextInputStream(object.getObjectContent()); /* * List objects in your bucket by prefix - There are many options for * listing the objects in your bucket. Keep in mind that buckets with * many objects might truncate their results when listing their objects, * so be sure to check if the returned object listing is truncated, and * use the AmazonS3.listNextBatchOfObjects(...) operation to retrieve * additional results. */ System.out.println("Listing objects"); ObjectListing objectListing = s3 .listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix("My")); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { System.out.println( " - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")"); } System.out.println(); /* * Delete an object - Unless versioning has been turned on for your bucket, * there is no way to undelete an object, so use caution when deleting objects. */ // System.out.println("Deleting an object\n"); // s3.deleteObject(bucketName, key); /* * Delete a bucket - A bucket must be completely empty before it can be * deleted, so remember to delete any objects from your buckets before * you try to delete them. */ // System.out.println("Deleting bucket " + bucketName + "\n"); // s3.deleteBucket(bucketName); System.out.println("------------------------------------------"); } catch (AmazonServiceException ase) { System.out.println("Caught an AmazonServiceException, which means your request made it " + "to Amazon S3, but was rejected with an error response for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { System.out.println("Caught an AmazonClientException, which means the client encountered " + "a serious internal problem while trying to communicate with S3, " + "such as not being able to access the network."); System.out.println("Error Message: " + ace.getMessage()); } }
From source file:common.S3Processor.java
License:Open Source License
public static List<String> getObjectList() { List<String> restaurantObjecName = new ArrayList<String>(); ListObjectsRequest request = new ListObjectsRequest(); request.setBucketName(BUCKET_NAME);/*from ww w .jav a 2 s . co m*/ ObjectListing objectListing = s3.listObjects(request); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { restaurantObjecName.add(objectSummary.getKey()); } return restaurantObjecName; }
From source file:controllers.s3locationmodify.java
License:Open Source License
private static long calculateregionload(String regionname) { AmazonS3 s3 = new AmazonS3Client(credentials); long size = 0; AmazonS3 s3client = new AmazonS3Client(new ProfileCredentialsProvider()); //try {/*from w w w. java 2 s.c om*/ ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(regionname); ObjectListing objectListing; do { objectListing = s3client.listObjects(listObjectsRequest); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { size = objectSummary.getSize(); } listObjectsRequest.setMarker(objectListing.getNextMarker()); } while (objectListing.isTruncated()); return size; }
From source file:core.connector.s3.sync.S3Connector.java
License:GNU General Public License
@Override public List<FileInfo> listDirectory(String path) throws ConnectorException { try {/*from www .j a v a2s .c om*/ ObjectListing bucketListing = s3 .listObjects(new ListObjectsRequest().withBucketName(info.getBucketName()).withPrefix(path)); List<FileInfo> listing = new ArrayList<FileInfo>(bucketListing.getObjectSummaries().size()); boolean finished = false; while (!finished) { for (S3ObjectSummary s3s : bucketListing.getObjectSummaries()) { String key = s3s.getKey(); long size = s3s.getSize(); Date date = s3s.getLastModified(); listing.add(new FileInfo(key, key.substring(path.length() + 1), size, date, s3s.getETag())); } if (bucketListing.isTruncated()) bucketListing = s3.listNextBatchOfObjects(bucketListing); else finished = true; } return listing; } catch (Exception e) { throw new ConnectorException(e); } }
From source file:cz.pichlik.goodsentiment.server.repository.S3RepositoryBase.java
License:Apache License
public List<String> list(final String bucket, final String key) { ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket).withPrefix(key); ObjectListing objectListing = s3Client.listObjects(listObjectsRequest); LinkedList<String> result = new LinkedList<>(); for (;;) {//from ww w .j a v a 2 s . c om for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { result.add(objectSummary.getKey().toString()); } if (objectListing.isTruncated()) { objectListing = s3Client.listNextBatchOfObjects(objectListing); } else { return result; } } }
From source file:dashboard.AmazonLogs.java
License:Open Source License
public int readAmazonLogs(int n, String AWS_USER, String AWS_PASS, String IPfile, String ERRfile, String bucketName, String DELETE_PROCESSED_LOGS, String API_KEY, String TOKEN, String apiuser, String apipass) throws Exception { if (n < 1) return 0; int eventsNumber = 0; String line = null;//from w w w. j a v a 2s . c o m int begin = 0; int zips = 0; int deletedZips = 0; int mixpanelStatus = 0; String registrant = ""; String ip = ""; String prevIP = ""; Mixpanel mix = new Mixpanel(); Whois w = new Whois(apiuser, apipass); int index = -1; Registrant r; ArrayList<Registrant> rList = new ArrayList<Registrant>(); ArrayList<Registrant> eList = new ArrayList<Registrant>(); IPList ipl = new IPList(); IPList errl = new IPList(); // Log files Bucket AWSCredentials credentials = new BasicAWSCredentials(AWS_USER, AWS_PASS); AmazonS3Client s3Client = new AmazonS3Client(credentials); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName); BufferedReader br = null; ObjectListing objectListing = s3Client.listObjects(listObjectsRequest); ObjectListing nextObjectListing = objectListing; zips = 0; Boolean more = true; if (objectListing == null) more = false; else { ipl.loadList(rList, IPfile); ipl.printList(rList, 30); } while (more) { // Reads 1000 files for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { // Handle ZIP files try { // Open and send to mixpanel events of one ZIP file String key = objectSummary.getKey(); S3Object object = s3Client.getObject(new GetObjectRequest(bucketName, key)); // Extract ZIP and read Object to reader br = new BufferedReader(new InputStreamReader(new GZIPInputStream(object.getObjectContent()))); zips++; // Read the lines from the unzipped file, break it and send to Mixpanel while ((line = br.readLine()) != null) { if (line.startsWith("#")) continue; if (line.trim().equals("")) continue; String[] values = line.split("\\s"); String eventTime = values[0] + " " + values[1]; ip = values[4]; if (ip != prevIP) { prevIP = ip; index = ipl.ipInList(ip, rList); if (index >= 0) { r = rList.get(index); registrant = r.name; // Update counter for this IP r.counter = r.counter + 1; rList.set(index, r); } else { // WHOIS - Check registrant of this IP address registrant = w.whoisIP(ip); // if there was an error, try again if (registrant.equals("ERROR")) registrant = w.whoisIP(ip); // if there was a second error, add it to errors list if (registrant.equals("ERROR")) { eList.add(new Registrant(ip, registrant, 1)); } else { // If name includes a comma, exclude the comma registrant = registrant.replace(",", ""); rList.add(new Registrant(ip, registrant, 1)); } } } String method = values[5]; String fileName = values[7]; String statusCode = values[8]; String userAgent = values[10]; String fName = fileName; if (fileName.contains("gigaspaces-")) { begin = fileName.lastIndexOf("gigaspaces-") + 11; fName = fileName.substring(begin, fileName.length()); } eventsNumber++; System.out.println(eventsNumber + ": " + eventTime + " " + ip + " " + registrant); // ==================================================== // Track the event in Mixpanel (using the POST import) // ==================================================== mixpanelStatus = mix.postCDNEventToMixpanel(API_KEY, TOKEN, ip, "Cloudfront CDN", eventTime, method, fileName, fName, userAgent, statusCode, registrant); } // while on ZIP file lines if (mixpanelStatus == 1 & DELETE_PROCESSED_LOGS.equals("YES")) { // Delete the CDN log ZIP file s3Client.deleteObject(bucketName, key); System.out.println("========= Deleted Zip " + zips + " ===== List Size " + rList.size() + " =========="); deletedZips++; } } catch (IOException e) { e.printStackTrace(); return eventsNumber; } finally { if (br != null) { br.close(); } if (eventsNumber >= n) { System.out.println("\n>>> " + eventsNumber + " events in " + zips + " Zip files. Deleted " + deletedZips + " Zip files.\n"); ipl.printList(rList, 100); ipl.saveList(rList, IPfile); if (!eList.isEmpty()) { SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd-HH-mm"); String fName = ERRfile + sdf.format(new Date()) + ".txt"; System.out.println("\n>>> " + eList.size() + " DomainTools errors:"); errl.saveList(eList, fName); } else System.out.println("\n>>> No DomainTools errors"); return eventsNumber; } } } // for (continue to next ZIP file // If there are more ZIP files, read next batch of 1000 if (objectListing.isTruncated()) { nextObjectListing = s3Client.listNextBatchOfObjects(objectListing); objectListing = nextObjectListing; } else more = false; // no more files } // while next objectListing System.out.println("\n>>> " + eventsNumber + " events in " + zips + " Zip files. Deleted " + deletedZips + " Zip files.\n"); ipl.printList(rList, 50); ipl.saveList(rList, IPfile); if (!eList.isEmpty()) { SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd-HH-mm"); String fName = ERRfile + sdf.format(new Date()) + ".txt"; System.out.println("\n>>> " + eList.size() + " DomainTools errors:"); errl.saveList(eList, fName); } else System.out.println("\n>>> No DomainTools errors"); return eventsNumber; }