Example usage for com.amazonaws.services.s3.model ObjectListing getObjectSummaries

List of usage examples for com.amazonaws.services.s3.model ObjectListing getObjectSummaries

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.services.s3.model.*
  5. ObjectListing
  6. getObjectSummaries

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.model ObjectListing getObjectSummaries.

Prototype

public List<S3ObjectSummary> getObjectSummaries()

Source Link

Document

Gets the list of object summaries describing the objects stored in the S3 bucket.

Usage

From source file:com.climate.oada.dao.impl.S3ResourceDAO.java

License:Open Source License

@Override
public List<FileResource> getFileUrls(Long userId, String type) {
    List<FileResource> retval = new ArrayList<FileResource>();
    long validfor = new Long(validHours).longValue() * HOURS_TO_MILLISECONDS;
    try {//from   w w  w.  ja v  a2 s . c o  m
        AmazonS3 s3client = new AmazonS3Client(new ProfileCredentialsProvider());
        String prefix = userId.toString() + S3_SEPARATOR + type;

        LOG.debug("Listing objects from bucket " + bucketName + " with prefix " + prefix);

        ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName)
                .withPrefix(prefix);
        ObjectListing objectListing;
        do {
            objectListing = s3client.listObjects(listObjectsRequest);
            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                LOG.debug(" - " + objectSummary.getKey() + "  " + "(size = " + objectSummary.getSize() + ")");

                Date expiration = new Date();
                long milliSeconds = expiration.getTime();
                milliSeconds += validfor;
                expiration.setTime(milliSeconds);

                GeneratePresignedUrlRequest generatePresignedUrlRequest = new GeneratePresignedUrlRequest(
                        bucketName, objectSummary.getKey());
                generatePresignedUrlRequest.setMethod(HttpMethod.GET);
                generatePresignedUrlRequest.setExpiration(expiration);

                FileResource res = new FileResource();
                res.setFileURL(s3client.generatePresignedUrl(generatePresignedUrlRequest));
                retval.add(res);
            }
            listObjectsRequest.setMarker(objectListing.getNextMarker());
        } while (objectListing.isTruncated());
    } catch (AmazonServiceException ase) {
        logAWSServiceException(ase);
    } catch (AmazonClientException ace) {
        logAWSClientException(ace);
    } catch (Exception e) {
        LOG.error("Unable to retrieve S3 file URLs " + e.getMessage());
    }
    return retval;
}

From source file:com.clouddrive.parth.AmazonOperations.java

public List<String> getFiles(String userName) {
    List<String> list = null;
    if (isBucketPresent(userName)) {
        ObjectListing bucketList = s3.listObjects(new ListObjectsRequest().withBucketName(userName));
        if (bucketList != null) {
            list = new ArrayList<String>();
        }//from ww w. j av a2s  . com
        for (S3ObjectSummary objectSummary : bucketList.getObjectSummaries()) {
            list.add(objectSummary.getKey());
        }
    }
    return list;
}

From source file:com.cloudhub.aws.extractor.AWSCSVExtractor.java

License:Apache License

/**
 * Requests billing information from Amazon S3.
 * This method may spawn multiple threads as needed to complete the task.
 *
 *///from   w  w w. j  av  a2 s. c o  m
@Override
public String getTotalCost() {
    String totalCost = null;

    try {
        log.debug("Listing objects ...");

        final ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName);

        ObjectListing objectListing;
        do {
            objectListing = s3client.listObjects(listObjectsRequest);
            for (final S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                log.debug(" - " + objectSummary.getKey() + "  " + "(size = " + objectSummary.getSize() + ")");

                if (objectSummary.getKey().contains(Constants.MATCHER_BILLING_CSV.getKeyPattern())) {
                    totalCost = persist(Constants.MATCHER_BILLING_CSV, objectSummary);

                } else if (objectSummary.getKey().contains(Constants.MATCHER_COST_ALLOCATION.getKeyPattern())) {
                    totalCost = persist(Constants.MATCHER_COST_ALLOCATION, objectSummary);
                }
            }
            listObjectsRequest.setMarker(objectListing.getNextMarker());
        } while (objectListing.isTruncated());

    } catch (AmazonServiceException ase) {
        log.error("Caught an AmazonServiceException, " + "which means your request made it "
                + "to Amazon S3, but was rejected with an error response " + "for some reason.");
        log.error("Error Message:    " + ase.getMessage());
        log.error("HTTP Status Code: " + ase.getStatusCode());
        log.error("AWS Error Code:   " + ase.getErrorCode());
        log.error("Error Type:       " + ase.getErrorType());
        log.error("Request ID:       " + ase.getRequestId());

    } catch (AmazonClientException ace) {
        log.error("Caught an AmazonClientException, " + "which means the client encountered "
                + "an internal error while trying to communicate" + " with S3, "
                + "such as not being able to access the network.");
        log.error("Error Message: " + ace.getMessage());

    } catch (IOException ioe) {
        log.error("Caught an IOException while writing to disk.");
        log.error("Error Message: " + ioe.getMessage());

    }

    return totalCost;
}

From source file:com.crickdata.upload.s3.UploadLiveData.java

License:Open Source License

public Map<String, Date> uploadToS3(String fileName, boolean type) throws IOException {

    Statistics statistics = new Statistics();
    Map<String, Date> perfMap = new HashMap<String, Date>();
    AWSCredentials credentials = null;/*from  w  ww .  j  a  va  2s . co  m*/
    try {
        credentials = new BasicAWSCredentials("AKIAI6QKTRAQE7MXQOIQ",
                "wIG6u1yI5ZaseeJbvYSUmD98qelIJNSCVBzt5k2q");
    } catch (Exception e) {
        throw new AmazonClientException("Cannot load the credentials from the credential profiles file. "
                + "Please make sure that your credentials file is at the correct "
                + "location (C:\\Users\\bssan_000\\.aws\\credentials), and is in valid format.", e);
    }

    AmazonS3 s3 = new AmazonS3Client(credentials);
    Region usWest2 = Region.getRegion(Regions.US_WEST_2);
    s3.setRegion(usWest2);
    String bucketName;
    if (!type)
        bucketName = "cricmatchinfo";
    else
        bucketName = "cricmatchinfoseries";
    String key = fileName.replace(".json", "").trim();
    try {
        perfMap.put("S3INSERTREQ", new Date());
        statistics.setS3Req(new Date());
        File f = readMatchFile(fileName);

        double bytes = f.length();
        double kilobytes = (bytes / 1024);
        System.out.println("Details :" + kilobytes);
        s3.putObject(new PutObjectRequest(bucketName, key, f));
        statistics.setSize(String.valueOf(kilobytes));

        S3Object object = s3.getObject(new GetObjectRequest(bucketName, key));
        perfMap.put("S3SAVERES", object.getObjectMetadata().getLastModified());
        statistics.setKey(key);
        statistics.setS3Res(object.getObjectMetadata().getLastModified());
        MyUI.stats.add(statistics);

        displayTextInputStream(object.getObjectContent());

        ObjectListing objectListing = s3
                .listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix("My"));
        for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
            System.out.println(
                    " - " + objectSummary.getKey() + "  " + "(size = " + objectSummary.getSize() + ")");
        }
    } catch (AmazonServiceException ase) {
        System.out.println("Caught an AmazonServiceException, which means your request made it "
                + "to Amazon S3, but was rejected with an error response for some reason.");
        System.out.println("Error Message:    " + ase.getMessage());
        System.out.println("HTTP Status Code: " + ase.getStatusCode());
        System.out.println("AWS Error Code:   " + ase.getErrorCode());
        System.out.println("Error Type:       " + ase.getErrorType());
        System.out.println("Request ID:       " + ase.getRequestId());
    } catch (AmazonClientException ace) {
        System.out.println("Caught an AmazonClientException, which means the client encountered "
                + "a serious internal problem while trying to communicate with S3, "
                + "such as not being able to access the network.");
        System.out.println("Error Message: " + ace.getMessage());
    }
    return perfMap;
}

From source file:com.davidsoergel.s3napback.S3ops.java

License:Apache License

public static void delete(TransferManager tx, String bucket, String fileprefix) throws InterruptedException {
    logger.info("Deleting " + fileprefix);

    List<DeleteObjectsRequest.KeyVersion> keys = new ArrayList<DeleteObjectsRequest.KeyVersion>();

    ObjectListing objectListing = tx.getAmazonS3Client()
            .listObjects(new ListObjectsRequest().withBucketName(bucket).withPrefix(fileprefix));
    for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
        keys.add(new DeleteObjectsRequest.KeyVersion(objectSummary.getKey()));
    }//  w ww .j a  va  2  s  . c  o  m

    DeleteObjectsRequest req = new DeleteObjectsRequest(bucket);
    req.setKeys(keys);
    DeleteObjectsResult result = tx.getAmazonS3Client().deleteObjects(req);
}

From source file:com.davidsoergel.s3napback.S3ops.java

License:Apache License

public static void list(StreamingTransferManager tx, String bucket) throws InterruptedException {

    //** sort by date
    SortedMap<String, SortedMap<String, S3ObjectSummary>> blocks = new TreeMap<String, SortedMap<String, S3ObjectSummary>>();

    ObjectListing current = tx.getAmazonS3Client().listObjects(new ListObjectsRequest().withBucketName(bucket));

    List<S3ObjectSummary> keyList = current.getObjectSummaries();
    ObjectListing next = tx.getAmazonS3Client().listNextBatchOfObjects(current);
    keyList.addAll(next.getObjectSummaries());

    while (next.isTruncated()) {
        current = tx.getAmazonS3Client().listNextBatchOfObjects(next);
        keyList.addAll(current.getObjectSummaries());
        next = tx.getAmazonS3Client().listNextBatchOfObjects(current);
    }//from  w  w w. ja  va  2  s .  c  o m
    keyList.addAll(next.getObjectSummaries());

    for (S3ObjectSummary objectSummary : keyList) {
        String[] c = objectSummary.getKey().split(":");
        if (c.length != 2) {
            logger.warn("ignoring malformed filename " + objectSummary.getKey());
        } else {
            String filename = c[0];
            String chunknum = c[1];

            SortedMap<String, S3ObjectSummary> chunks = blocks.get(filename);
            if (chunks == null) {
                chunks = new TreeMap<String, S3ObjectSummary>();
                blocks.put(filename, chunks);
            }

            chunks.put(chunknum, objectSummary);
        }
    }

    // now the files and chunks are in the maps in order
    for (Map.Entry<String, SortedMap<String, S3ObjectSummary>> blockEntry : blocks.entrySet()) {
        String filename = blockEntry.getKey();
        SortedMap<String, S3ObjectSummary> chunks = blockEntry.getValue();

        long totalsize = 0;
        Date lastModified = null;
        for (Map.Entry<String, S3ObjectSummary> entry : chunks.entrySet()) {
            totalsize += entry.getValue().getSize();
            lastModified = entry.getValue().getLastModified();
        }
        String[] line = { bucket, filename, "" + chunks.keySet().size(), "" + totalsize,
                lastModified.toString() };

        System.err.println(StringUtils.join("\t", line));

        // 2008-04-10 04:07:50 - dev.davidsoergel.com.backup1:MySQL/all-0 - 153.38k in 1 data blocks
    }
}

From source file:com.davidsoergel.s3napback.S3ops.java

License:Apache License

public static void download(StreamingTransferManager tx, String bucket, String fileprefix)
        throws InterruptedException, IOException {
    // first list the files

    SortedMap<String, S3ObjectSummary> chunks = new TreeMap<String, S3ObjectSummary>();

    ObjectListing objectListing = tx.getAmazonS3Client()
            .listObjects(new ListObjectsRequest().withBucketName(bucket).withPrefix(fileprefix));
    for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
        chunks.put(objectSummary.getKey(), objectSummary);
    }//from  w  w  w . jav a2  s  .  c  o  m

    logger.info("Downloading " + fileprefix);
    Date start = new Date();
    // now the chunks are in the map in order
    long totalBytes = 0;
    BufferedOutputStream out = new BufferedOutputStream(System.out);
    for (Map.Entry<String, S3ObjectSummary> entry : chunks.entrySet()) {
        String key = entry.getKey();
        logger.info("Downloading " + key);

        Download myDownload = tx.download(bucket, key, out);
        while (myDownload.isDone() == false) {
            long bytes = totalBytes + myDownload.getProgress().getBytesTransfered();
            Double mb = (double) bytes / 1024. / 1024.;
            Double sec = (new Date().getTime() - start.getTime()) / 1000.;
            Double rate = mb / sec;

            logger.info(String.format("%.2f MB, %.2fMB/s", mb, rate));
            // Do work while we wait for our upload to complete...
            Thread.sleep(500);
        }
        totalBytes += myDownload.getProgress().getBytesTransfered();
    }
    out.close();

    Long bytes = totalBytes;
    Double mb = (double) bytes / 1024. / 1024.;
    Double sec = (new Date().getTime() - start.getTime()) / 1000.;
    Double rate = mb / sec;

    logger.info(String.format("Downloaded %s to stdout, %d bytes, %.2f sec, %.2fMB/s", fileprefix, totalBytes,
            sec, rate));
    //logger.info("Downloaded " + fileprefix + " to stdout, " + totalBytes + " bytes, " + sec +" sec, " + rate + " MB/sec");
}

From source file:com.digitaslbi.helios.mock.utils.ConnectionHelper.java

private static Map convertResultToFile(ListObjectsRequest listObjectsRequest, String path, String parentPath) {
    Map<String, File> filesMap = new HashMap<String, File>();
    File aux;//from ww w  .jav  a2 s  .  c  om

    ObjectListing objListing = s3Client.listObjects(listObjectsRequest);

    // creates the parent elemment
    aux = new File();
    aux.setIsParent(true);
    aux.setFullPath(parentPath);
    aux.setPath(parentPath);

    // adapts the path in order to use as map key
    if (aux.getPath() != null) {
        aux.setPath(clearPathDelimiter(aux.getPath()));

        if (aux.getPath().lastIndexOf(MocksConstants.AWS_PARENT_DELIMITER.getValue()) > 0) {
            aux.setPath(aux.getPath()
                    .substring(aux.getPath().lastIndexOf(MocksConstants.AWS_PARENT_DELIMITER.getValue()) + 1));
        }
    }
    filesMap.put(aux.getPath(), aux);

    for (S3ObjectSummary objSummary : objListing.getObjectSummaries()) {
        aux = new File();
        aux.setFullPath(objSummary.getKey());
        aux.setParent(objListing.getPrefix());

        // if size is 0 is considered a folder
        aux.setIsFile((objSummary.getSize() == 0) ? false : true);

        if (aux.getParent() != null) {
            if (!aux.getFullPath().equals(aux.getParent())) {
                aux.setPath(aux.getFullPath().replace(aux.getParent(), ""));
            }
        } else {
            aux.setPath(aux.getFullPath());
        }

        if (aux.getPath() != null) {
            filesMap.put(clearPathDelimiter(aux.getPath()), aux);
        }
    }

    for (String folderNames : objListing.getCommonPrefixes()) {
        aux = new File();
        aux.setFullPath(folderNames);
        aux.setParent(objListing.getPrefix());
        aux.setIsFile(false);

        if (aux.getParent() != null) {
            aux.setPath(aux.getFullPath().replace(aux.getParent(), ""));
        } else {
            aux.setPath(aux.getFullPath());
        }

        if (aux.getPath() != null) {
            filesMap.put(clearPathDelimiter(aux.getPath()), aux);
        }
    }

    return filesMap;
}

From source file:com.digitaslbi.helios.utils.S3Helper.java

public static Folder getFolder(String path) {
    ListObjectsRequest listObjectsRequest;
    Folder folder;//from   w w  w .  j  a va2s  . com
    File aux;

    if (path == null || path.equals("/")) {
        listObjectsRequest = getRootFolders();
    } else {
        listObjectsRequest = getContentByPreffix(path);
    }

    ObjectListing objListing = s3Client.listObjects(listObjectsRequest);

    folder = new Folder();
    folder.setFiles(new ArrayList<File>());

    folder.setPath(path);
    folder.setParent(obtainParentPath(path));

    for (S3ObjectSummary objSummary : objListing.getObjectSummaries()) {
        aux = new File();
        aux.setPath(objSummary.getKey());
        aux.setParent(objListing.getPrefix());

        // if size is 0 is considered a folder
        aux.setIsFile((objSummary.getSize() == 0) ? false : true);

        if (!aux.getPath().equals(path)) {
            folder.getFiles().add(aux);
        }
    }

    for (String folderNames : objListing.getCommonPrefixes()) {
        aux = new File();
        aux.setPath(folderNames);
        aux.setParent(objListing.getPrefix());
        aux.setIsFile(false);

        folder.getFiles().add(aux);
    }

    log.info("[S3Helper][getFolder] Path: " + path + " items found: " + folder.getFiles().size());

    return folder;
}

From source file:com.dongli.model.MyJSONData.java

License:Open Source License

public static JSONArray getAllObjects() throws MyRESTException {

    // return value
    JSONArray jsonArray = new JSONArray();

    try {/*from ww  w.  j  a v  a2s. c o  m*/
        ObjectListing objects = MyAWSStorage.getInstance().s3client
                .listObjects(MyConfiguration.getInstance().bucket);

        do {
            for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
                JSONObject jb = new JSONObject();
                jb.put("url", MyConfiguration.getInstance().urlPrefix + MyConfiguration.getInstance().rootPrefix
                        + "/" + objectSummary.getKey());
                jsonArray.put(jb);
                // System.out.println(objectSummary.getKey());
            }

            objects = MyAWSStorage.getInstance().s3client.listNextBatchOfObjects(objects);
        } while (objects.isTruncated());
    } catch (JSONException e) {
        throw new MyRESTException("Failed to get all JSON object.");
    } catch (AmazonServiceException ase) {
        throw new MyRESTException("Failed to get all JSON object.");
    } catch (AmazonClientException ace) {
        throw new MyRESTException("Failed to get all JSON object.");
    }

    return jsonArray;
}