List of usage examples for com.amazonaws.services.s3.model ObjectListing getObjectSummaries
public List<S3ObjectSummary> getObjectSummaries()
From source file:dashboard.ImportCDN.java
License:Open Source License
public static int readAmazonLogs(int n) throws Exception { if (n < 1) return 0; int eventsNumber = 0; int begin = 0; int zips = 0; int postFailures = 0; int deletedZips = 0; int mixpanelStatus = 0; String line = null;/*www .ja v a 2s .c om*/ // Log files Bucket AWSCredentials credentials = new BasicAWSCredentials(AWS_USER, AWS_PASS); AmazonS3Client s3Client = new AmazonS3Client(credentials); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName); // Set MARKER - from which Log File to start reading // listObjectsRequest.setMarker("E2DXXJR0N8BXOK.2013-03-18-10.ICK6IvaY.gz"); BufferedReader br = null; ObjectListing objectListing = s3Client.listObjects(listObjectsRequest); ObjectListing nextObjectListing = objectListing; zips = 0; Boolean more = true; if (objectListing == null) more = false; while (more) { // Reads 1000 files for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { // Handle ZIP files try { // Open and send to mixpanel events of one ZIP file String key = objectSummary.getKey(); S3Object object = s3Client.getObject(new GetObjectRequest(bucketName, key)); // Extract ZIP and read Object to reader br = new BufferedReader(new InputStreamReader(new GZIPInputStream(object.getObjectContent()))); zips++; // Read the lines from the unzipped file, break it and send to Mixpanel while ((line = br.readLine()) != null) { if (line.startsWith("#")) continue; if (line.trim().equals("")) continue; String[] values = line.split("\\s"); String eventTime = values[0] + " " + values[1]; String ip = values[4]; String method = values[5]; String fileName = values[7]; String statusCode = values[8]; String userAgent = values[10]; String fName = fileName; if (fileName.contains("gigaspaces-")) { begin = fileName.lastIndexOf("gigaspaces-") + 11; fName = fileName.substring(begin, fileName.length()); } eventsNumber++; System.out.println(eventsNumber + ": " + eventTime + " " + ip); // Track the event in Mixpanel (using the POST import) mixpanelStatus = postCDNEventToMixpanel(ip, "Cloudfront CDN", eventTime, method, fileName, fName, userAgent, statusCode); // If failed if (mixpanelStatus != 1) { postFailures++; System.out.println(" >>> POST event to Mixpanel Failed!! " + postFailures); } } // while on ZIP file lines if (mixpanelStatus == 1 & DELETE_PROCESSED_LOGS.equals("YES")) { // Delete the CDN log ZIP file s3Client.deleteObject(bucketName, key); System.out.println("============ Deleted Zip " + zips + " ============"); deletedZips++; } else System.out.println("============ Zip " + zips + " (not deleted) ============"); } catch (IOException e) { e.printStackTrace(); return eventsNumber; } finally { if (br != null) { br.close(); } if (eventsNumber >= n) { System.out.println("\n>>> " + eventsNumber + " events in " + zips + " Zip files."); System.out.println("\n>>> " + deletedZips + " Zip files deleted."); System.out.println("\n>>> " + postFailures + " post Failures\n"); return eventsNumber; } } } // for (continue to next ZIP file // If there are more ZIP files, read next batch of 1000 if (objectListing.isTruncated()) { nextObjectListing = s3Client.listNextBatchOfObjects(objectListing); objectListing = nextObjectListing; } else more = false; // no more files } // while next objectListing System.out.println("\n>>> " + eventsNumber + " events in " + zips + " Zip files."); System.out.println("\n>>> " + deletedZips + " Zip files deleted."); System.out.println("\n>>> " + postFailures + " post Failures\n"); return eventsNumber; }
From source file:dk.dma.nogoservice.service.S3DataLoader.java
License:Apache License
public List<String> getFiles() { ObjectListing listing = amazonS3.listObjects(DATA_BUCKET); return listing.getObjectSummaries().stream().map(S3ObjectSummary::getKey).collect(Collectors.toList()); }
From source file:ecplugins.s3.S3Util.java
License:Apache License
/** * This procedure deletes the bucket along with its contents * @param bucketName/*w ww . j a va 2 s.c om*/ * @return * @throws Exception */ public static boolean DeleteBucket(String bucketName) throws Exception { Properties props = TestUtils.getProperties(); BasicAWSCredentials credentials = new BasicAWSCredentials(props.getProperty(StringConstants.ACCESS_ID), props.getProperty(StringConstants.SECRET_ACCESS_ID)); // Create TransferManager TransferManager tx = new TransferManager(credentials); // Get S3 Client AmazonS3 s3 = tx.getAmazonS3Client(); if (s3.doesBucketExist(bucketName)) { // Multi-object delete by specifying only keys (no version ID). DeleteObjectsRequest multiObjectDeleteRequest = new DeleteObjectsRequest(bucketName).withQuiet(false); //get keys List<String> keys = new ArrayList<String>(); ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName)); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { keys.add(objectSummary.getKey()); } // Create request that include only object key names. List<DeleteObjectsRequest.KeyVersion> justKeys = new ArrayList<DeleteObjectsRequest.KeyVersion>(); for (String key : keys) { justKeys.add(new DeleteObjectsRequest.KeyVersion(key)); } if (justKeys.size() == 0) { return false; } multiObjectDeleteRequest.setKeys(justKeys); // Execute DeleteObjects - Amazon S3 add delete marker for each object // deletion. The objects no disappear from your bucket (verify). DeleteObjectsResult delObjRes = null; delObjRes = s3.deleteObjects(multiObjectDeleteRequest); s3.deleteBucket(bucketName); return true; } else { System.out.println("Error: Bucket with name " + bucketName + " does not exists."); return false; } }
From source file:edu.iit.s3bucket.S3Bucket.java
/** * * @return//from w ww .j a v a 2 s .co m */ public boolean emptyBucket() { DeleteObjectsRequest multiObjectDeleteRequest = new DeleteObjectsRequest(this.bucketname); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(this.bucketname); List<KeyVersion> keys = new ArrayList<KeyVersion>(); ObjectListing objectListing = s3client.listObjects(listObjectsRequest); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { keys.add(new KeyVersion(objectSummary.getKey())); } multiObjectDeleteRequest.setKeys(keys); try { DeleteObjectsResult delObjRes = s3client.deleteObjects(multiObjectDeleteRequest); return true; } catch (MultiObjectDeleteException e) { return false; } }
From source file:edu.iu.finch.core.NexRad.java
License:Apache License
public List<S3ObjectSummary> listBucket(String key) { ObjectListing listing = s3Client.listObjects(BUCKET_NAME, key); List<S3ObjectSummary> summaries = listing.getObjectSummaries(); while (listing.isTruncated()) { listing = s3Client.listNextBatchOfObjects(listing); summaries.addAll(listing.getObjectSummaries()); }//ww w . jav a 2 s . com return summaries; }
From source file:edu.si.services.beans.cameratrap.AmazonS3ClientMock.java
License:Apache License
@Override public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException { if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) { AmazonServiceException ex = new AmazonServiceException("Unknown bucket"); ex.setStatusCode(404);//from www. ja v a2 s . c om throw ex; } ObjectListing objectListing = new ObjectListing(); for (int index = 0; index < objects.size(); index++) { if (objects.get(index).getBucketName().equals(listObjectsRequest.getBucketName())) { S3ObjectSummary s3ObjectSummary = new S3ObjectSummary(); s3ObjectSummary.setBucketName(objects.get(index).getBucketName()); s3ObjectSummary.setKey(objects.get(index).getKey()); objectListing.getObjectSummaries().add(s3ObjectSummary); } } return objectListing; }
From source file:edu.umass.cs.aws.support.examples.AWSStatusCheck.java
License:Apache License
/** * * @param args//from ww w. j a v a 2 s.c om * @throws Exception */ public static void main(String[] args) throws Exception { init(); /* * Amazon EC2 */ for (String endpoint : endpoints) { try { ec2.setEndpoint(endpoint); System.out.println("**** Endpoint: " + endpoint); DescribeAvailabilityZonesResult availabilityZonesResult = ec2.describeAvailabilityZones(); System.out.println("You have access to " + availabilityZonesResult.getAvailabilityZones().size() + " Availability Zones."); for (AvailabilityZone zone : availabilityZonesResult.getAvailabilityZones()) { System.out.println(zone.getZoneName()); } DescribeInstancesResult describeInstancesRequest = ec2.describeInstances(); List<Reservation> reservations = describeInstancesRequest.getReservations(); Set<Instance> instances = new HashSet<Instance>(); System.out.println("Instances: "); for (Reservation reservation : reservations) { for (Instance instance : reservation.getInstances()) { instances.add(instance); System.out.println(instance.getPublicDnsName() + " is " + instance.getState().getName()); } } System.out.println("Security groups: "); DescribeSecurityGroupsResult describeSecurityGroupsResult = ec2.describeSecurityGroups(); for (SecurityGroup securityGroup : describeSecurityGroupsResult.getSecurityGroups()) { System.out.println(securityGroup.getGroupName()); } //System.out.println("You have " + instances.size() + " Amazon EC2 instance(s) running."); } catch (AmazonServiceException ase) { System.out.println("Caught Exception: " + ase.getMessage()); System.out.println("Reponse Status Code: " + ase.getStatusCode()); System.out.println("Error Code: " + ase.getErrorCode()); System.out.println("Request ID: " + ase.getRequestId()); } /* * Amazon SimpleDB * */ try { ListDomainsRequest sdbRequest = new ListDomainsRequest().withMaxNumberOfDomains(100); ListDomainsResult sdbResult = sdb.listDomains(sdbRequest); int totalItems = 0; for (String domainName : sdbResult.getDomainNames()) { DomainMetadataRequest metadataRequest = new DomainMetadataRequest().withDomainName(domainName); DomainMetadataResult domainMetadata = sdb.domainMetadata(metadataRequest); totalItems += domainMetadata.getItemCount(); } System.out.println("You have " + sdbResult.getDomainNames().size() + " Amazon SimpleDB domain(s)" + "containing a total of " + totalItems + " items."); } catch (AmazonServiceException ase) { System.out.println("Caught Exception: " + ase.getMessage()); System.out.println("Reponse Status Code: " + ase.getStatusCode()); System.out.println("Error Code: " + ase.getErrorCode()); System.out.println("Request ID: " + ase.getRequestId()); } /* * Amazon S3 *. */ try { List<Bucket> buckets = s3.listBuckets(); long totalSize = 0; int totalItems = 0; for (Bucket bucket : buckets) { /* * In order to save bandwidth, an S3 object listing does not * contain every object in the bucket; after a certain point the * S3ObjectListing is truncated, and further pages must be * obtained with the AmazonS3Client.listNextBatchOfObjects() * method. */ ObjectListing objects = s3.listObjects(bucket.getName()); do { for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) { totalSize += objectSummary.getSize(); totalItems++; } objects = s3.listNextBatchOfObjects(objects); } while (objects.isTruncated()); } System.out.println("You have " + buckets.size() + " Amazon S3 bucket(s), " + "containing " + totalItems + " objects with a total size of " + totalSize + " bytes."); } catch (AmazonServiceException ase) { /* * AmazonServiceExceptions represent an error response from an AWS * services, i.e. your request made it to AWS, but the AWS service * either found it invalid or encountered an error trying to execute * it. */ System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { /* * AmazonClientExceptions represent an error that occurred inside * the client on the local host, either while trying to send the * request to AWS or interpret the response. For example, if no * network connection is available, the client won't be able to * connect to AWS to execute a request and will throw an * AmazonClientException. */ System.out.println("Error Message: " + ace.getMessage()); } } }
From source file:eu.stratosphere.nephele.fs.s3.S3FileSystem.java
License:Apache License
private S3FileStatus[] listBucketContent(final Path f, final S3BucketObjectPair bop) throws IOException { ObjectListing listing = null; final List<S3FileStatus> resultList = new ArrayList<S3FileStatus>(); final int depth = (bop.hasObject() ? getDepth(bop.getObject()) + 1 : 0); while (true) { if (listing == null) { if (bop.hasObject()) { listing = this.s3Client.listObjects(bop.getBucket(), bop.getObject()); } else { listing = this.s3Client.listObjects(bop.getBucket()); }/*from w ww . j a va 2 s . c o m*/ } else { listing = this.s3Client.listNextBatchOfObjects(listing); } final List<S3ObjectSummary> list = listing.getObjectSummaries(); final Iterator<S3ObjectSummary> it = list.iterator(); while (it.hasNext()) { final S3ObjectSummary os = it.next(); String key = os.getKey(); final int childDepth = getDepth(os.getKey()); if (childDepth != depth) { continue; } // Remove the prefix if (bop.hasObject()) { if (key.startsWith(bop.getObject())) { key = key.substring(bop.getObject().length()); } // This has been the prefix itself if (key.isEmpty()) { continue; } } final long modificationDate = dateToLong(os.getLastModified()); S3FileStatus fileStatus; if (objectRepresentsDirectory(os)) { fileStatus = new S3FileStatus(extendPath(f, key), 0, true, modificationDate, 0L); } else { fileStatus = new S3FileStatus(extendPath(f, key), os.getSize(), false, modificationDate, 0L); } resultList.add(fileStatus); } if (!listing.isTruncated()) { break; } } /* * System.out.println("---- RETURN CONTENT ----"); * for (final FileStatus entry : resultList) { * System.out.println(entry.getPath()); * } * System.out.println("------------------------"); */ return resultList.toArray(new S3FileStatus[0]); }
From source file:exemplos.S3Sample.java
License:Open Source License
public static void main(String[] args) throws IOException { /*//from w w w . j av a 2 s .c o m * This credentials provider implementation loads your AWS credentials * from a properties file at the root of your classpath. * * Important: Be sure to fill in your AWS access credentials in the * AwsCredentials.properties file before you try to run this * sample. * http://aws.amazon.com/security-credentials */ AmazonS3 s3 = new AmazonS3Client(new ClasspathPropertiesFileCredentialsProvider()); Region usWest2 = Region.getRegion(Regions.US_WEST_2); s3.setRegion(usWest2); String bucketName = "my-first-s3-bucket-" + UUID.randomUUID(); String key = "MyObjectKey"; System.out.println("==========================================="); System.out.println("Getting Started with Amazon S3"); System.out.println("===========================================\n"); try { /* * Create a new S3 bucket - Amazon S3 bucket names are globally unique, * so once a bucket name has been taken by any user, you can't create * another bucket with that same name. * * You can optionally specify a location for your bucket if you want to * keep your data closer to your applications or users. */ System.out.println("Creating bucket " + bucketName + "\n"); s3.createBucket(bucketName); /* * List the buckets in your account */ System.out.println("Listing buckets"); for (Bucket bucket : s3.listBuckets()) { System.out.println(" - " + bucket.getName()); } System.out.println(); /* * Upload an object to your bucket - You can easily upload a file to * S3, or upload directly an InputStream if you know the length of * the data in the stream. You can also specify your own metadata * when uploading to S3, which allows you set a variety of options * like content-type and content-encoding, plus additional metadata * specific to your applications. */ System.out.println("Uploading a new object to S3 from a file\n"); s3.putObject(new PutObjectRequest(bucketName, key, createSampleFile())); /* * Download an object - When you download an object, you get all of * the object's metadata and a stream from which to read the contents. * It's important to read the contents of the stream as quickly as * possibly since the data is streamed directly from Amazon S3 and your * network connection will remain open until you read all the data or * close the input stream. * * GetObjectRequest also supports several other options, including * conditional downloading of objects based on modification times, * ETags, and selectively downloading a range of an object. */ System.out.println("Downloading an object"); S3Object object = s3.getObject(new GetObjectRequest(bucketName, key)); System.out.println("Content-Type: " + object.getObjectMetadata().getContentType()); displayTextInputStream(object.getObjectContent()); /* * List objects in your bucket by prefix - There are many options for * listing the objects in your bucket. Keep in mind that buckets with * many objects might truncate their results when listing their objects, * so be sure to check if the returned object listing is truncated, and * use the AmazonS3.listNextBatchOfObjects(...) operation to retrieve * additional results. */ System.out.println("Listing objects"); ObjectListing objectListing = s3 .listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix("My")); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { System.out.println( " - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")"); } System.out.println(); /* * Delete an object - Unless versioning has been turned on for your bucket, * there is no way to undelete an object, so use caution when deleting objects. */ System.out.println("Deleting an object\n"); s3.deleteObject(bucketName, key); /* * Delete a bucket - A bucket must be completely empty before it can be * deleted, so remember to delete any objects from your buckets before * you try to delete them. */ System.out.println("Deleting bucket " + bucketName + "\n"); s3.deleteBucket(bucketName); } catch (AmazonServiceException ase) { System.out.println("Caught an AmazonServiceException, which means your request made it " + "to Amazon S3, but was rejected with an error response for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { System.out.println("Caught an AmazonClientException, which means the client encountered " + "a serious internal problem while trying to communicate with S3, " + "such as not being able to access the network."); System.out.println("Error Message: " + ace.getMessage()); } }
From source file:fi.yle.tools.aws.maven.SimpleStorageServiceWagon.java
License:Apache License
private List<String> getResourceNames(ObjectListing objectListing, Pattern pattern) { List<String> resourceNames = new ArrayList<String>(); for (String commonPrefix : objectListing.getCommonPrefixes()) { resourceNames.add(getResourceName(commonPrefix, pattern)); }// ww w. ja v a2s.c om for (S3ObjectSummary s3ObjectSummary : objectListing.getObjectSummaries()) { resourceNames.add(getResourceName(s3ObjectSummary.getKey(), pattern)); } return resourceNames; }