Example usage for com.amazonaws.services.s3.model S3ObjectSummary getKey

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.model S3ObjectSummary getKey.

Prototype

public String getKey() 

Source Link

Document

Gets the key under which this object is stored in Amazon S3.

Usage

From source file:com.netflix.ice.processor.BillingFileProcessor.java

License:Apache License

@Override
protected void poll() throws Exception {

    TreeMap<DateTime, List<BillingFile>> filesToProcess = Maps.newTreeMap();
    Map<DateTime, List<BillingFile>> monitorFilesToProcess = Maps.newTreeMap();

    // list the tar.gz file in billing file folder
    for (int i = 0; i < config.billingS3BucketNames.length; i++) {
        String billingS3BucketName = config.billingS3BucketNames[i];
        String billingS3BucketPrefix = config.billingS3BucketPrefixes.length > i
                ? config.billingS3BucketPrefixes[i]
                : "";
        String accountId = config.billingAccountIds.length > i ? config.billingAccountIds[i] : "";
        String billingAccessRoleName = config.billingAccessRoleNames.length > i
                ? config.billingAccessRoleNames[i]
                : "";
        String billingAccessExternalId = config.billingAccessExternalIds.length > i
                ? config.billingAccessExternalIds[i]
                : "";

        logger.info("trying to list objects in billing bucket " + billingS3BucketName
                + " using assume role, and external id " + billingAccessRoleName + " "
                + billingAccessExternalId);
        List<S3ObjectSummary> objectSummaries = AwsUtils.listAllObjects(billingS3BucketName,
                billingS3BucketPrefix, accountId, billingAccessRoleName, billingAccessExternalId);
        logger.info("found " + objectSummaries.size() + " in billing bucket " + billingS3BucketName);
        TreeMap<DateTime, S3ObjectSummary> filesToProcessInOneBucket = Maps.newTreeMap();
        Map<DateTime, S3ObjectSummary> monitorFilesToProcessInOneBucket = Maps.newTreeMap();

        // for each file, download&process if not needed
        for (S3ObjectSummary objectSummary : objectSummaries) {

            String fileKey = objectSummary.getKey();
            DateTime dataTime = AwsUtils.getDateTimeFromFileNameWithTags(fileKey);
            boolean withTags = true;
            if (dataTime == null) {
                dataTime = AwsUtils.getDateTimeFromFileName(fileKey);
                withTags = false;//w w w  .ja va2 s . c  o  m
            }

            if (dataTime != null && !dataTime.isBefore(config.startDate)) {
                if (!filesToProcessInOneBucket.containsKey(dataTime)
                        || withTags && config.resourceService != null
                        || !withTags && config.resourceService == null)
                    filesToProcessInOneBucket.put(dataTime, objectSummary);
                else
                    logger.info("ignoring file " + objectSummary.getKey());
            } else {
                logger.info("ignoring file " + objectSummary.getKey());
            }
        }

        for (S3ObjectSummary objectSummary : objectSummaries) {
            String fileKey = objectSummary.getKey();
            DateTime dataTime = AwsUtils.getDateTimeFromFileNameWithMonitoring(fileKey);

            if (dataTime != null && !dataTime.isBefore(config.startDate)) {
                monitorFilesToProcessInOneBucket.put(dataTime, objectSummary);
            }
        }

        for (DateTime key : filesToProcessInOneBucket.keySet()) {
            List<BillingFile> list = filesToProcess.get(key);
            if (list == null) {
                list = Lists.newArrayList();
                filesToProcess.put(key, list);
            }
            list.add(new BillingFile(filesToProcessInOneBucket.get(key), accountId, billingAccessRoleName,
                    billingAccessExternalId, billingS3BucketPrefix));
        }

        for (DateTime key : monitorFilesToProcessInOneBucket.keySet()) {
            List<BillingFile> list = monitorFilesToProcess.get(key);
            if (list == null) {
                list = Lists.newArrayList();
                monitorFilesToProcess.put(key, list);
            }
            list.add(new BillingFile(monitorFilesToProcessInOneBucket.get(key), accountId,
                    billingAccessRoleName, billingAccessExternalId, billingS3BucketPrefix));
        }
    }

    for (DateTime dataTime : filesToProcess.keySet()) {
        startMilli = endMilli = dataTime.getMillis();
        init();

        boolean hasNewFiles = false;
        boolean hasTags = false;
        long lastProcessed = lastProcessTime(AwsUtils.monthDateFormat.print(dataTime));

        for (BillingFile billingFile : filesToProcess.get(dataTime)) {
            S3ObjectSummary objectSummary = billingFile.s3ObjectSummary;
            if (objectSummary.getLastModified().getTime() < lastProcessed) {
                logger.info("data has been processed. ignoring " + objectSummary.getKey() + "...");
                continue;
            }
            hasNewFiles = true;
        }

        if (!hasNewFiles) {
            logger.info("data has been processed. ignoring all files at "
                    + AwsUtils.monthDateFormat.print(dataTime));
            continue;
        }

        long processTime = new DateTime(DateTimeZone.UTC).getMillis();
        for (BillingFile billingFile : filesToProcess.get(dataTime)) {

            S3ObjectSummary objectSummary = billingFile.s3ObjectSummary;
            String fileKey = objectSummary.getKey();

            File file = new File(config.localDir, fileKey.substring(billingFile.prefix.length()));
            logger.info("trying to download " + fileKey + "...");
            boolean downloaded = AwsUtils.downloadFileIfChangedSince(objectSummary.getBucketName(),
                    billingFile.prefix, file, lastProcessed, billingFile.accountId, billingFile.accessRoleName,
                    billingFile.externalId);
            if (downloaded)
                logger.info("downloaded " + fileKey);
            else {
                logger.info("file already downloaded " + fileKey + "...");
            }

            logger.info("processing " + fileKey + "...");
            boolean withTags = fileKey.contains("with-resources-and-tags");
            hasTags = hasTags || withTags;
            processingMonitor = false;
            processBillingZipFile(file, withTags);
            logger.info("done processing " + fileKey);
        }

        if (monitorFilesToProcess.get(dataTime) != null) {
            for (BillingFile monitorBillingFile : monitorFilesToProcess.get(dataTime)) {

                S3ObjectSummary monitorObjectSummary = monitorBillingFile.s3ObjectSummary;
                if (monitorObjectSummary != null) {
                    String monitorFileKey = monitorObjectSummary.getKey();
                    logger.info("processing " + monitorFileKey + "...");
                    File monitorFile = new File(config.localDir,
                            monitorFileKey.substring(monitorFileKey.lastIndexOf("/") + 1));
                    logger.info("trying to download " + monitorFileKey + "...");
                    boolean downloaded = AwsUtils.downloadFileIfChangedSince(
                            monitorObjectSummary.getBucketName(), monitorBillingFile.prefix, monitorFile,
                            lastProcessed, monitorBillingFile.accountId, monitorBillingFile.accessRoleName,
                            monitorBillingFile.externalId);
                    if (downloaded)
                        logger.info("downloaded " + monitorFile);
                    else
                        logger.warn(monitorFile + "already downloaded...");
                    FileInputStream in = new FileInputStream(monitorFile);
                    try {
                        processingMonitor = true;
                        processBillingFile(monitorFile.getName(), in, true);
                    } catch (Exception e) {
                        logger.error("Error processing " + monitorFile, e);
                    } finally {
                        in.close();
                    }
                }
            }
        }

        if (dataTime.equals(filesToProcess.lastKey())) {
            int hours = (int) ((endMilli - startMilli) / 3600000L);
            logger.info("cut hours to " + hours);
            cutData(hours);
        }

        // now get reservation capacity to calculate upfront and un-used cost
        for (Ec2InstanceReservationPrice.ReservationUtilization utilization : Ec2InstanceReservationPrice.ReservationUtilization
                .values())
            processReservations(utilization);

        if (hasTags && config.resourceService != null)
            config.resourceService.commit();

        logger.info("archiving results for " + dataTime + "...");
        archive();
        logger.info("done archiving " + dataTime);

        updateProcessTime(AwsUtils.monthDateFormat.print(dataTime), processTime);
        if (dataTime.equals(filesToProcess.lastKey())) {
            sendOndemandCostAlert();
        }
    }

    logger.info("AWS usage processed.");
}

---

From source file:com.netflix.spinnaker.front50.model.S3StorageService.java

License:Apache License

private boolean filterS3ObjectSummary(S3ObjectSummary s3ObjectSummary, String metadataFilename) {
    return s3ObjectSummary.getKey().endsWith(metadataFilename);
}

---

From source file:com.netflix.spinnaker.front50.model.S3Support.java

License:Apache License

private boolean filterS3ObjectSummary(S3ObjectSummary s3ObjectSummary) {
    return s3ObjectSummary.getKey().endsWith(getMetadataFilename());
}

---

From source file:com.netflix.spinnaker.front50.model.S3Support.java

License:Apache License

private String extractItemName(S3ObjectSummary s3ObjectSummary) {
    return s3ObjectSummary.getKey().replaceAll(rootFolder, "").replaceAll("/" + getMetadataFilename(), "");
}

---

From source file:com.pinterest.terrapin.TerrapinUtil.java

License:Apache License

static public List<Pair<Path, Long>> getS3FileList(AWSCredentials credentials, String s3Bucket,
        String s3KeyPrefix) {/*from  ww w  . j a va2 s .c  o m*/
    List<Pair<Path, Long>> fileSizePairList = Lists.newArrayListWithCapacity(Constants.MAX_ALLOWED_SHARDS);
    AmazonS3Client s3Client = new AmazonS3Client(credentials);
    // List files and build the path using the s3n: prefix.
    // Note that keys > marker are retrieved where the > is by lexicographic order.
    String prefix = s3KeyPrefix;
    String marker = prefix;
    while (true) {
        boolean reachedEnd = false;
        ObjectListing listing = s3Client
                .listObjects(new ListObjectsRequest().withBucketName(s3Bucket).withMarker(marker));
        List<S3ObjectSummary> summaries = listing.getObjectSummaries();

        if (summaries.isEmpty()) {
            break;
        }

        for (S3ObjectSummary summary : summaries) {
            if (summary.getKey().startsWith(prefix)) {
                fileSizePairList.add(new ImmutablePair(new Path("s3n", s3Bucket, "/" + summary.getKey()),
                        summary.getSize()));
                if (fileSizePairList.size() > Constants.MAX_ALLOWED_SHARDS) {
                    throw new RuntimeException("Too many files " + fileSizePairList.size());
                }
            } else {
                // We found a key which does not match the prefix, stop.
                reachedEnd = true;
                break;
            }
        }
        if (reachedEnd) {
            break;
        }
        marker = summaries.get(summaries.size() - 1).getKey();
    }
    return fileSizePairList;
}

---

From source file:com.proofpoint.event.collector.combiner.S3StorageSystem.java

License:Apache License

@Override
public List<StoredObject> listObjects(URI storageArea) {
    S3StorageHelper.checkValidS3Uri(storageArea);

    String s3Path = getS3ObjectKey(storageArea);
    Iterator<S3ObjectSummary> iter = new S3ObjectListing(s3Service,
            new ListObjectsRequest(getS3Bucket(storageArea), s3Path, null, "/", null)).iterator();

    ImmutableList.Builder<StoredObject> builder = ImmutableList.builder();
    while (iter.hasNext()) {
        S3ObjectSummary summary = iter.next();
        builder.add(new StoredObject(buildS3Location(storageArea, summary.getKey().substring(s3Path.length())),
                summary.getETag(), summary.getSize(), summary.getLastModified().getTime()));
    }//from w  w w.  j  a va2s . c o  m
    return builder.build();
}

---

From source file:com.proofpoint.event.monitor.s3.S3StorageSystem.java

License:Apache License

public List<URI> listObjects(URI storageArea) {
    S3StorageHelper.checkValidS3Uri(storageArea);

    String s3Path = getS3ObjectKey(storageArea);
    Iterable<S3ObjectSummary> objectListing = new S3ObjectListing(s3Service,
            new ListObjectsRequest(getS3Bucket(storageArea), s3Path, null, "/", null));

    ImmutableList.Builder<URI> builder = ImmutableList.builder();
    for (S3ObjectSummary summary : objectListing) {
        builder.add(buildS3Location(storageArea, summary.getKey().substring(s3Path.length())));
    }/*from w w  w  . j  a  va2  s. c  o m*/
    return builder.build();
}

---

From source file:com.qubole.presto.kinesis.s3config.S3TableConfigClient.java

License:Apache License

/**
 * Connect to S3 directory to look for new or updated table definitions and then
 * update the map./*from  www.j a  v a2  s  . c o m*/
 */
protected void updateTablesFromS3() {
    long now = System.currentTimeMillis();

    List<S3ObjectSummary> objectList = this.getObjectSummaries();
    AmazonS3Client s3client = this.clientManager.getS3Client();
    AmazonS3URI directoryURI = new AmazonS3URI(this.bucketUrl);

    for (S3ObjectSummary objInfo : objectList) {
        if (!this.internalMap.containsKey(objInfo.getKey())
                || objInfo.getLastModified().getTime() >= this.lastCheck) {
            // New or updated file, so we must read from AWS
            try {
                if (objInfo.getKey().endsWith("/")) {
                    continue;
                }

                log.info("Getting : %s - %s", objInfo.getBucketName(), objInfo.getKey());
                S3Object object = s3client
                        .getObject(new GetObjectRequest(objInfo.getBucketName(), objInfo.getKey()));

                StringBuilder resultStr = new StringBuilder("");
                try (BufferedReader reader = new BufferedReader(
                        new InputStreamReader(object.getObjectContent()))) {
                    boolean hasMore = true;
                    while (hasMore) {
                        String line = reader.readLine();
                        if (line != null) {
                            resultStr.append(line);
                        } else {
                            hasMore = false;
                        }
                    }

                    KinesisStreamDescription table = streamDescriptionCodec.fromJson(resultStr.toString());

                    internalMap.put(objInfo.getKey(), table);
                    log.info("Put table description into the map from %s", objInfo.getKey());
                } catch (IOException iox) {
                    log.error("Problem reading input stream from object.", iox);
                }
            } catch (AmazonServiceException ase) {
                StringBuilder sb = new StringBuilder();
                sb.append("Caught an AmazonServiceException, which means your request made it ");
                sb.append("to Amazon S3, but was rejected with an error response for some reason.\n");
                sb.append("Error Message:    " + ase.getMessage());
                sb.append("HTTP Status Code: " + ase.getStatusCode());
                sb.append("AWS Error Code:   " + ase.getErrorCode());
                sb.append("Error Type:       " + ase.getErrorType());
                sb.append("Request ID:       " + ase.getRequestId());
                log.error(sb.toString(), ase);
            } catch (AmazonClientException ace) {
                StringBuilder sb = new StringBuilder();
                sb.append("Caught an AmazonClientException, " + "which means the client encountered "
                        + "an internal error while trying to communicate" + " with S3, "
                        + "such as not being able to access the network.");
                sb.append("Error Message: " + ace.getMessage());
                log.error(sb.toString(), ace);
            }
        }
    } // end loop through object descriptions

    log.info("Completed updating table definitions from S3.");
    this.lastCheck = now;

    return;
}

---

From source file:com.rathravane.clerk.impl.s3.S3IamDb.java

License:Apache License

@Override
public void sweepExpiredTags() throws IamSvcException {
    final TreeSet<String> keys = new TreeSet<String>();
    try {/*from   ww w.j a va2 s.  c o  m*/
        final String prefix = makeByTagId("");
        final ListObjectsRequest listObjectRequest = new ListObjectsRequest().withBucketName(fBucketId)
                .withPrefix(prefix);
        ObjectListing objects = fDb.listObjects(listObjectRequest);
        do {
            for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
                final String tagId = objectSummary.getKey().substring(prefix.length());
                keys.add(tagId);
            }
            objects = fDb.listNextBatchOfObjects(objects);
        } while (objects.isTruncated());
    } catch (AmazonS3Exception x) {
        throw new IamSvcException(x);
    }

    for (String key : keys) {
        loadTagObject(key, false);
    }
}

---

From source file:com.rathravane.clerk.impl.s3.S3IamDb.java

License:Apache License

List<String> loadKeysBelow(String key) throws IamSvcException {
    final LinkedList<String> result = new LinkedList<String>();
    try {//from   w  w  w .j  a  va  2 s. co  m
        final ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(fBucketId)
                .withPrefix(key);
        ObjectListing objectListing;
        do {
            objectListing = fDb.listObjects(listObjectsRequest);
            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                result.add(objectSummary.getKey());
            }
            listObjectsRequest.setMarker(objectListing.getNextMarker());
        } while (objectListing.isTruncated());
    } catch (AmazonClientException e) {
        throw new IamSvcException(e);
    }
    return result;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• »