List of usage examples for com.amazonaws.services.s3.model S3ObjectSummary getKey
public String getKey()
From source file:com.yahoo.athenz.zts.store.impl.S3ChangeLogStore.java
License:Apache License
/** * list the objects in the zts bucket. If the mod time is specified as 0 * then we want to list all objects otherwise, we only list objects * that are newer than the specified timestamp * @param s3 AWS S3 client object/*from www .java 2 s. co m*/ * @param domains collection to be updated to include domain names * @param modTime only include domains newer than this timestamp */ void listObjects(AmazonS3 s3, Collection<String> domains, long modTime) { if (LOGGER.isDebugEnabled()) { LOGGER.debug("listObjects: Retrieving domains from {} with mod time > {}", s3BucketName, modTime); } ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(s3BucketName)); String objectName; while (objectListing != null) { // process each entry in our result set and add the domain // name to our return list final List<S3ObjectSummary> objectSummaries = objectListing.getObjectSummaries(); boolean listTruncated = objectListing.isTruncated(); if (LOGGER.isDebugEnabled()) { LOGGER.debug("listObjects: retrieved {} objects, more objects available - {}", objectSummaries.size(), listTruncated); } for (S3ObjectSummary objectSummary : objectSummaries) { // if mod time is specified then make sure we automatically skip // any domains older than the specified value if (modTime > 0 && objectSummary.getLastModified().getTime() <= modTime) { continue; } // for now skip any folders/objects that start with '.' objectName = objectSummary.getKey(); if (objectName.charAt(0) == '.') { continue; } domains.add(objectName); } // check if the object listing is truncated or not (break out in this case) // technically we can skip this call and just call listNextBatchOfResults // since that returns null if the object listing is not truncated but // this direct check here makes the logic easier to follow if (!listTruncated) { break; } objectListing = s3.listNextBatchOfObjects(objectListing); } }
From source file:com.yahoo.athenz.zts.store.s3.S3ChangeLogStore.java
License:Apache License
/** * list the objects in the zts bucket. If te mod time is specified as 0 * then we want to list all objects otherwise, we only list objects * that are newer than the specified timestamp * @param s3 AWS S3 client object/* www . j a v a 2 s .com*/ * @param domains collection to be updated to include domain names * @param modTime only include domains newer than this timestamp */ void listObjects(AmazonS3 s3, Collection<String> domains, long modTime) { ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(s3BucketName)); String objectName = null; while (objectListing != null) { // process each entry in our result set and add the domain // name to our return list for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { // if mod time is specified then make sure we automatically skip // any domains older than the specified value if (modTime > 0 && objectSummary.getLastModified().getTime() <= modTime) { continue; } // for now skip any folders/objects that start with '.' objectName = objectSummary.getKey(); if (objectName.charAt(0) == '.') { continue; } domains.add(objectName); } // check if the object listing is truncated or not (break out in this case) // technically we can skip this call and just call listNextBatchOfResults // since that returns null if the object listing is not truncated but // this direct check here makes the logic easier to follow if (!objectListing.isTruncated()) { break; } objectListing = s3.listNextBatchOfObjects(objectListing); } }
From source file:com.yahoo.ycsb.db.S3Client.java
License:Open Source License
/** * Perform an emulation of a database scan operation on a S3 bucket. * * @param bucket// w ww.j av a 2 s.c o m * The name of the bucket * @param startkey * The file key of the first file to read. * @param recordcount * The number of files to read * @param fields * The list of fields to read, or null for all of them * @param result * A Vector of HashMaps, where each HashMap is a set field/value * pairs for one file * */ protected Status scanFromStorage(String bucket, String startkey, int recordcount, Vector<HashMap<String, ByteIterator>> result, SSECustomerKey ssecLocal) { int counter = 0; ObjectListing listing = s3Client.listObjects(bucket); List<S3ObjectSummary> summaries = listing.getObjectSummaries(); List<String> keyList = new ArrayList(); int startkeyNumber = 0; int numberOfIteration = 0; // getting the list of files in the bucket while (listing.isTruncated()) { listing = s3Client.listNextBatchOfObjects(listing); summaries.addAll(listing.getObjectSummaries()); } for (S3ObjectSummary summary : summaries) { String summaryKey = summary.getKey(); keyList.add(summaryKey); } // Sorting the list of files in Alphabetical order Collections.sort(keyList); // sorting the list // Getting the position of the startingfile for the scan for (String key : keyList) { if (key.equals(startkey)) { startkeyNumber = counter; } else { counter = counter + 1; } } // Checking if the total number of file is bigger than the file to read, // if not using the total number of Files if (recordcount < keyList.size()) { numberOfIteration = recordcount; } else { numberOfIteration = keyList.size(); } // Reading the Files starting from the startkey File till the end // of the Files or Till the recordcount number for (int i = startkeyNumber; i < numberOfIteration; i++) { HashMap<String, ByteIterator> resultTemp = new HashMap<String, ByteIterator>(); readFromStorage(bucket, keyList.get(i), resultTemp, ssecLocal); result.add(resultTemp); } return Status.OK; }
From source file:com.zhang.aws.s3.S3Sample.java
License:Open Source License
public static void main(String[] args) throws IOException { /*/*from w ww.j a v a 2 s. co m*/ * The ProfileCredentialsProvider will return your [default] * credential profile by reading from the credentials file located at * (~/.aws/credentials). */ ResourceBundle bundle = ResourceBundle.getBundle("credentials"); AWSCredentials credentials = null; try { // credentials = new ProfileCredentialsProvider().getCredentials(); credentials = new BasicAWSCredentials(bundle.getString("aws_access_key_id"), bundle.getString("aws_secret_access_key")); } catch (Exception e) { throw new AmazonClientException("Cannot load the credentials from the credential profiles file. " + "Please make sure that your credentials file is at the correct " + "location (~/.aws/credentials), and is in valid format.", e); } AmazonS3 s3 = new AmazonS3Client(credentials); Region usWest2 = Region.getRegion(Regions.US_WEST_2); s3.setRegion(usWest2); String bucketName = "elasticbeanstalk-us-west-2-948206320069"; String key = "MyObjectKey2"; System.out.println("==========================================="); System.out.println("Getting Started with Amazon S3"); System.out.println("===========================================\n"); try { /* * Create a new S3 bucket - Amazon S3 bucket names are globally unique, * so once a bucket name has been taken by any user, you can't create * another bucket with that same name. * * You can optionally specify a location for your bucket if you want to * keep your data closer to your applications or users. */ System.out.println("Creating bucket " + bucketName + "\n"); // s3.createBucket(bucketName); /* * List the buckets in your account */ System.out.println("Listing buckets"); for (Bucket bucket : s3.listBuckets()) { System.out.println(" - " + bucket.getName()); } System.out.println(); /* * Upload an object to your bucket - You can easily upload a file to * S3, or upload directly an InputStream if you know the length of * the data in the stream. You can also specify your own metadata * when uploading to S3, which allows you set a variety of options * like content-type and content-encoding, plus additional metadata * specific to your applications. */ System.out.println("Uploading a new object to S3 from a file\n"); // s3.putObject(new PutObjectRequest(bucketName, key, createSampleFile())); s3.putObject(new PutObjectRequest(bucketName, key, getFileFromDisk())); /*** * * ?url * */ GeneratePresignedUrlRequest urlRequest = new GeneratePresignedUrlRequest(bucketName, key); URL generatePresignedUrl = s3.generatePresignedUrl(urlRequest); System.out.println("public url:" + generatePresignedUrl.toString()); /* * Download an object - When you download an object, you get all of * the object's metadata and a stream from which to read the contents. * It's important to read the contents of the stream as quickly as * possibly since the data is streamed directly from Amazon S3 and your * network connection will remain open until you read all the data or * close the input stream. * * GetObjectRequest also supports several other options, including * conditional downloading of objects based on modification times, * ETags, and selectively downloading a range of an object. */ // System.out.println("Downloading an object"); // S3Object object = s3.getObject(new GetObjectRequest(bucketName, key)); // System.out.println("Content-Type: " + object.getObjectMetadata().getContentType()); // displayTextInputStream(object.getObjectContent()); /* * List objects in your bucket by prefix - There are many options for * listing the objects in your bucket. Keep in mind that buckets with * many objects might truncate their results when listing their objects, * so be sure to check if the returned object listing is truncated, and * use the AmazonS3.listNextBatchOfObjects(...) operation to retrieve * additional results. */ System.out.println("Listing objects"); ObjectListing objectListing = s3 .listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix("My")); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { System.out.println( " - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")"); } System.out.println(); /* * Delete an object - Unless versioning has been turned on for your bucket, * there is no way to undelete an object, so use caution when deleting objects. */ // System.out.println("Deleting an object\n"); // s3.deleteObject(bucketName, key); /* * Delete a bucket - A bucket must be completely empty before it can be * deleted, so remember to delete any objects from your buckets before * you try to delete them. */ // System.out.println("Deleting bucket " + bucketName + "\n"); // s3.deleteBucket(bucketName); System.out.println("------------------------------------------"); } catch (AmazonServiceException ase) { System.out.println("Caught an AmazonServiceException, which means your request made it " + "to Amazon S3, but was rejected with an error response for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { System.out.println("Caught an AmazonClientException, which means the client encountered " + "a serious internal problem while trying to communicate with S3, " + "such as not being able to access the network."); System.out.println("Error Message: " + ace.getMessage()); } }
From source file:common.S3Processor.java
License:Open Source License
public static List<String> getObjectList() { List<String> restaurantObjecName = new ArrayList<String>(); ListObjectsRequest request = new ListObjectsRequest(); request.setBucketName(BUCKET_NAME);/*from w ww . j a v a 2 s.c o m*/ ObjectListing objectListing = s3.listObjects(request); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { restaurantObjecName.add(objectSummary.getKey()); } return restaurantObjecName; }
From source file:core.connector.s3.sync.S3Connector.java
License:GNU General Public License
@Override public List<FileInfo> listDirectory(String path) throws ConnectorException { try {//w ww. j a va2 s . c o m ObjectListing bucketListing = s3 .listObjects(new ListObjectsRequest().withBucketName(info.getBucketName()).withPrefix(path)); List<FileInfo> listing = new ArrayList<FileInfo>(bucketListing.getObjectSummaries().size()); boolean finished = false; while (!finished) { for (S3ObjectSummary s3s : bucketListing.getObjectSummaries()) { String key = s3s.getKey(); long size = s3s.getSize(); Date date = s3s.getLastModified(); listing.add(new FileInfo(key, key.substring(path.length() + 1), size, date, s3s.getETag())); } if (bucketListing.isTruncated()) bucketListing = s3.listNextBatchOfObjects(bucketListing); else finished = true; } return listing; } catch (Exception e) { throw new ConnectorException(e); } }
From source file:cz.pichlik.goodsentiment.server.repository.S3RepositoryBase.java
License:Apache License
public List<String> list(final String bucket, final String key) { ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket).withPrefix(key); ObjectListing objectListing = s3Client.listObjects(listObjectsRequest); LinkedList<String> result = new LinkedList<>(); for (;;) {/*from w w w . j a va 2 s .com*/ for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { result.add(objectSummary.getKey().toString()); } if (objectListing.isTruncated()) { objectListing = s3Client.listNextBatchOfObjects(objectListing); } else { return result; } } }
From source file:dashboard.AmazonLogs.java
License:Open Source License
public int readAmazonLogs(int n, String AWS_USER, String AWS_PASS, String IPfile, String ERRfile, String bucketName, String DELETE_PROCESSED_LOGS, String API_KEY, String TOKEN, String apiuser, String apipass) throws Exception { if (n < 1) return 0; int eventsNumber = 0; String line = null;/*from w w w. ja va 2 s . co m*/ int begin = 0; int zips = 0; int deletedZips = 0; int mixpanelStatus = 0; String registrant = ""; String ip = ""; String prevIP = ""; Mixpanel mix = new Mixpanel(); Whois w = new Whois(apiuser, apipass); int index = -1; Registrant r; ArrayList<Registrant> rList = new ArrayList<Registrant>(); ArrayList<Registrant> eList = new ArrayList<Registrant>(); IPList ipl = new IPList(); IPList errl = new IPList(); // Log files Bucket AWSCredentials credentials = new BasicAWSCredentials(AWS_USER, AWS_PASS); AmazonS3Client s3Client = new AmazonS3Client(credentials); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName); BufferedReader br = null; ObjectListing objectListing = s3Client.listObjects(listObjectsRequest); ObjectListing nextObjectListing = objectListing; zips = 0; Boolean more = true; if (objectListing == null) more = false; else { ipl.loadList(rList, IPfile); ipl.printList(rList, 30); } while (more) { // Reads 1000 files for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { // Handle ZIP files try { // Open and send to mixpanel events of one ZIP file String key = objectSummary.getKey(); S3Object object = s3Client.getObject(new GetObjectRequest(bucketName, key)); // Extract ZIP and read Object to reader br = new BufferedReader(new InputStreamReader(new GZIPInputStream(object.getObjectContent()))); zips++; // Read the lines from the unzipped file, break it and send to Mixpanel while ((line = br.readLine()) != null) { if (line.startsWith("#")) continue; if (line.trim().equals("")) continue; String[] values = line.split("\\s"); String eventTime = values[0] + " " + values[1]; ip = values[4]; if (ip != prevIP) { prevIP = ip; index = ipl.ipInList(ip, rList); if (index >= 0) { r = rList.get(index); registrant = r.name; // Update counter for this IP r.counter = r.counter + 1; rList.set(index, r); } else { // WHOIS - Check registrant of this IP address registrant = w.whoisIP(ip); // if there was an error, try again if (registrant.equals("ERROR")) registrant = w.whoisIP(ip); // if there was a second error, add it to errors list if (registrant.equals("ERROR")) { eList.add(new Registrant(ip, registrant, 1)); } else { // If name includes a comma, exclude the comma registrant = registrant.replace(",", ""); rList.add(new Registrant(ip, registrant, 1)); } } } String method = values[5]; String fileName = values[7]; String statusCode = values[8]; String userAgent = values[10]; String fName = fileName; if (fileName.contains("gigaspaces-")) { begin = fileName.lastIndexOf("gigaspaces-") + 11; fName = fileName.substring(begin, fileName.length()); } eventsNumber++; System.out.println(eventsNumber + ": " + eventTime + " " + ip + " " + registrant); // ==================================================== // Track the event in Mixpanel (using the POST import) // ==================================================== mixpanelStatus = mix.postCDNEventToMixpanel(API_KEY, TOKEN, ip, "Cloudfront CDN", eventTime, method, fileName, fName, userAgent, statusCode, registrant); } // while on ZIP file lines if (mixpanelStatus == 1 & DELETE_PROCESSED_LOGS.equals("YES")) { // Delete the CDN log ZIP file s3Client.deleteObject(bucketName, key); System.out.println("========= Deleted Zip " + zips + " ===== List Size " + rList.size() + " =========="); deletedZips++; } } catch (IOException e) { e.printStackTrace(); return eventsNumber; } finally { if (br != null) { br.close(); } if (eventsNumber >= n) { System.out.println("\n>>> " + eventsNumber + " events in " + zips + " Zip files. Deleted " + deletedZips + " Zip files.\n"); ipl.printList(rList, 100); ipl.saveList(rList, IPfile); if (!eList.isEmpty()) { SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd-HH-mm"); String fName = ERRfile + sdf.format(new Date()) + ".txt"; System.out.println("\n>>> " + eList.size() + " DomainTools errors:"); errl.saveList(eList, fName); } else System.out.println("\n>>> No DomainTools errors"); return eventsNumber; } } } // for (continue to next ZIP file // If there are more ZIP files, read next batch of 1000 if (objectListing.isTruncated()) { nextObjectListing = s3Client.listNextBatchOfObjects(objectListing); objectListing = nextObjectListing; } else more = false; // no more files } // while next objectListing System.out.println("\n>>> " + eventsNumber + " events in " + zips + " Zip files. Deleted " + deletedZips + " Zip files.\n"); ipl.printList(rList, 50); ipl.saveList(rList, IPfile); if (!eList.isEmpty()) { SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd-HH-mm"); String fName = ERRfile + sdf.format(new Date()) + ".txt"; System.out.println("\n>>> " + eList.size() + " DomainTools errors:"); errl.saveList(eList, fName); } else System.out.println("\n>>> No DomainTools errors"); return eventsNumber; }
From source file:dashboard.ImportCDN.java
License:Open Source License
public static int readAmazonLogs(int n) throws Exception { if (n < 1) return 0; int eventsNumber = 0; int begin = 0; int zips = 0; int postFailures = 0; int deletedZips = 0; int mixpanelStatus = 0; String line = null;//w ww . j a va2 s .c om // Log files Bucket AWSCredentials credentials = new BasicAWSCredentials(AWS_USER, AWS_PASS); AmazonS3Client s3Client = new AmazonS3Client(credentials); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName); // Set MARKER - from which Log File to start reading // listObjectsRequest.setMarker("E2DXXJR0N8BXOK.2013-03-18-10.ICK6IvaY.gz"); BufferedReader br = null; ObjectListing objectListing = s3Client.listObjects(listObjectsRequest); ObjectListing nextObjectListing = objectListing; zips = 0; Boolean more = true; if (objectListing == null) more = false; while (more) { // Reads 1000 files for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { // Handle ZIP files try { // Open and send to mixpanel events of one ZIP file String key = objectSummary.getKey(); S3Object object = s3Client.getObject(new GetObjectRequest(bucketName, key)); // Extract ZIP and read Object to reader br = new BufferedReader(new InputStreamReader(new GZIPInputStream(object.getObjectContent()))); zips++; // Read the lines from the unzipped file, break it and send to Mixpanel while ((line = br.readLine()) != null) { if (line.startsWith("#")) continue; if (line.trim().equals("")) continue; String[] values = line.split("\\s"); String eventTime = values[0] + " " + values[1]; String ip = values[4]; String method = values[5]; String fileName = values[7]; String statusCode = values[8]; String userAgent = values[10]; String fName = fileName; if (fileName.contains("gigaspaces-")) { begin = fileName.lastIndexOf("gigaspaces-") + 11; fName = fileName.substring(begin, fileName.length()); } eventsNumber++; System.out.println(eventsNumber + ": " + eventTime + " " + ip); // Track the event in Mixpanel (using the POST import) mixpanelStatus = postCDNEventToMixpanel(ip, "Cloudfront CDN", eventTime, method, fileName, fName, userAgent, statusCode); // If failed if (mixpanelStatus != 1) { postFailures++; System.out.println(" >>> POST event to Mixpanel Failed!! " + postFailures); } } // while on ZIP file lines if (mixpanelStatus == 1 & DELETE_PROCESSED_LOGS.equals("YES")) { // Delete the CDN log ZIP file s3Client.deleteObject(bucketName, key); System.out.println("============ Deleted Zip " + zips + " ============"); deletedZips++; } else System.out.println("============ Zip " + zips + " (not deleted) ============"); } catch (IOException e) { e.printStackTrace(); return eventsNumber; } finally { if (br != null) { br.close(); } if (eventsNumber >= n) { System.out.println("\n>>> " + eventsNumber + " events in " + zips + " Zip files."); System.out.println("\n>>> " + deletedZips + " Zip files deleted."); System.out.println("\n>>> " + postFailures + " post Failures\n"); return eventsNumber; } } } // for (continue to next ZIP file // If there are more ZIP files, read next batch of 1000 if (objectListing.isTruncated()) { nextObjectListing = s3Client.listNextBatchOfObjects(objectListing); objectListing = nextObjectListing; } else more = false; // no more files } // while next objectListing System.out.println("\n>>> " + eventsNumber + " events in " + zips + " Zip files."); System.out.println("\n>>> " + deletedZips + " Zip files deleted."); System.out.println("\n>>> " + postFailures + " post Failures\n"); return eventsNumber; }
From source file:datameer.awstasks.ant.s3.model.CreateBucketCommand.java
License:Apache License
@Override public void execute(Project project, AmazonS3 s3Service) { String name = getNormalizedName(); boolean doesBucketExist = s3Service.doesBucketExist(name); if (isEmptyIfExistent() && doesBucketExist) { List<S3ObjectSummary> s3Objects = s3Service.listObjects(name).getObjectSummaries(); for (S3ObjectSummary s3Object : s3Objects) { s3Service.deleteObject(name, s3Object.getKey()); }/*from w w w . j a va2s . c o m*/ doesBucketExist = false; } if (!doesBucketExist) { try { s3Service.createBucket(name, _location); System.out.println("created bucket '" + name + "'"); } catch (Exception e) { throw new RuntimeException("failed to create bucket '" + name + "'", e); } } }