Example usage for com.amazonaws.services.s3.model S3ObjectSummary getBucketName

List of usage examples for com.amazonaws.services.s3.model S3ObjectSummary getBucketName

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.services.s3.model.*
  5. S3ObjectSummary
  6. getBucketName

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.model S3ObjectSummary getBucketName.

Prototype

public String getBucketName()

Source Link

Document

Gets the name of the Amazon S3 bucket in which this object is stored.

Usage

From source file:awslabs.lab41.SolutionCode.java

License:Open Source License

@Override
public void removeLabBuckets(AmazonS3Client s3Client, List<String> bucketNames) {
    for (String bucketName : bucketNames) {
        try {//from   ww w  . ja v  a2s . c  o  m
            ObjectListing objectListing = s3Client
                    .listObjects(new ListObjectsRequest().withBucketName(bucketName));
            for (S3ObjectSummary s3ObjectSummary : objectListing.getObjectSummaries()) {
                DeleteObjectRequest deleteObjectRequest = new DeleteObjectRequest(
                        s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey());

                s3Client.deleteObject(deleteObjectRequest);
            }

            s3Client.deleteBucket(new DeleteBucketRequest(bucketName));
        } catch (AmazonS3Exception s3E) {
            if (!s3E.getErrorCode().equals("NoSuchBucket")) {
                // This error wasn't expected, so rethrow.
                throw s3E;
            }
        }
    }
}

From source file:com.cloudhub.aws.extractor.AWSCSVExtractor.java

License:Apache License

/**
 * Writes the specified Amazon S3 object to the file system.
 *
 * @param object - the object to write out
 * @param inputStream - the body of the object
 * @throws IOException - if any I/O error occurs.
 *///w  ww.  ja v  a 2 s. co m
@SuppressWarnings("resource")
private File writeOutObjectToFile(final S3ObjectSummary object, final InputStream inputStream)
        throws IOException {

    final File parent = new File(dataFolder, object.getBucketName());
    if (!parent.exists()) {
        parent.mkdirs();
    }
    final File objectFile = new File(parent, object.getKey());

    final ReadableByteChannel src = Channels.newChannel(inputStream);
    final FileChannel dest = new FileOutputStream(objectFile).getChannel();

    dest.transferFrom(src, 0, object.getSize());
    dest.close();
    src.close();

    return objectFile;
}

From source file:com.conductor.s3.S3InputFormatUtils.java

License:Apache License

/**
 * Efficiently gets the Hadoop {@link org.apache.hadoop.fs.FileStatus} for all S3 files under the provided
 * {@code dirs}//w ww .  j  a v  a  2 s  . com
 * 
 * @param s3Client
 *            s3 client
 * @param blockSize
 *            the block size
 * @param dirs
 *            the dirs to search through
 * @return the {@link org.apache.hadoop.fs.FileStatus} version of all S3 files under {@code dirs}
 */
static List<FileStatus> getFileStatuses(final AmazonS3 s3Client, final long blockSize, final Path... dirs) {
    final List<FileStatus> result = Lists.newArrayList();
    for (final Path dir : dirs) {
        // get bucket and prefix from path
        final String bucket = S3HadoopUtils.getBucketFromPath(dir.toString());
        final String prefix = S3HadoopUtils.getKeyFromPath(dir.toString());
        // list request
        final ListObjectsRequest req = new ListObjectsRequest().withMaxKeys(Integer.MAX_VALUE)
                .withBucketName(bucket).withPrefix(prefix);
        // recursively page through all objects under the path
        for (ObjectListing listing = s3Client.listObjects(req); listing.getObjectSummaries()
                .size() > 0; listing = s3Client.listNextBatchOfObjects(listing)) {
            for (final S3ObjectSummary summary : listing.getObjectSummaries()) {
                final Path path = new Path(
                        String.format("s3n://%s/%s", summary.getBucketName(), summary.getKey()));
                if (S3_PATH_FILTER.accept(path)) {
                    result.add(new FileStatus(summary.getSize(), false, 1, blockSize,
                            summary.getLastModified().getTime(), path));
                }
            }
            // don't need to check the next listing if this one is not truncated
            if (!listing.isTruncated()) {
                break;
            }
        }
    }
    return result;
}

From source file:com.facebook.presto.kinesis.s3config.S3TableConfigClient.java

License:Apache License

/**
 * Connect to S3 directory to look for new or updated table definitions and then
 * update the map./*  w  w  w . ja  v  a 2s . c o  m*/
 */
protected void updateTablesFromS3() {
    long now = System.currentTimeMillis();

    List<S3ObjectSummary> objectList = this.getObjectSummaries();
    AmazonS3Client s3client = this.clientManager.getS3Client();
    AmazonS3URI directoryURI = new AmazonS3URI(this.bucketUrl);

    // Build map of "deltas" which in the end contains new definitions and deleted tables
    HashMap<String, KinesisStreamDescription> deltasMap = new HashMap<String, KinesisStreamDescription>();
    internalMapLock.readLock().lock();
    try {
        Iterator<String> keysIter = this.internalMap.keySet().iterator();
        while (keysIter.hasNext()) {
            deltasMap.put(keysIter.next(), dummyStreamDesc);
        }
    } finally {
        internalMapLock.readLock().unlock();
    }

    for (S3ObjectSummary objInfo : objectList) {
        if (!deltasMap.containsKey(objInfo.getKey()) || objInfo.getLastModified().getTime() >= this.lastCheck) {
            // New or updated file, so we must read from AWS
            try {
                if (objInfo.getKey().endsWith("/")) {
                    continue;
                }

                log.info("Getting : %s - %s", objInfo.getBucketName(), objInfo.getKey());
                S3Object object = s3client
                        .getObject(new GetObjectRequest(objInfo.getBucketName(), objInfo.getKey()));

                StringBuilder resultStr = new StringBuilder("");
                try (BufferedReader reader = new BufferedReader(
                        new InputStreamReader(object.getObjectContent()))) {
                    boolean hasMore = true;
                    while (hasMore) {
                        String line = reader.readLine();
                        if (line != null) {
                            resultStr.append(line);
                        } else {
                            hasMore = false;
                        }
                    }

                    KinesisStreamDescription table = streamDescriptionCodec.fromJson(resultStr.toString());

                    deltasMap.put(objInfo.getKey(), table);
                    log.info("Put table description into the map from %s : %s.%s", objInfo.getKey(),
                            table.getSchemaName(), table.getTableName());
                } catch (IOException iox) {
                    log.error("Problem reading input stream from object.", iox);
                } catch (IllegalArgumentException iax) {
                    // Note: this gets thrown by airlift json library when the input is malformed.
                    log.error("Invalid JSON table description.", iax);
                }
            } catch (AmazonServiceException ase) {
                StringBuilder sb = new StringBuilder();
                sb.append("Caught an AmazonServiceException, which means your request made it ");
                sb.append("to Amazon S3, but was rejected with an error response for some reason.\n");
                sb.append("Error Message:    " + ase.getMessage());
                sb.append("HTTP Status Code: " + ase.getStatusCode());
                sb.append("AWS Error Code:   " + ase.getErrorCode());
                sb.append("Error Type:       " + ase.getErrorType());
                sb.append("Request ID:       " + ase.getRequestId());
                log.error(sb.toString(), ase);
            } catch (AmazonClientException ace) {
                StringBuilder sb = new StringBuilder();
                sb.append("Caught an AmazonClientException, " + "which means the client encountered "
                        + "an internal error while trying to communicate" + " with S3, "
                        + "such as not being able to access the network.");
                sb.append("Error Message: " + ace.getMessage());
                log.error(sb.toString(), ace);
            }
        } else if (deltasMap.containsKey(objInfo.getKey())) {
            deltasMap.remove(objInfo.getKey());
        }
    } // end loop through object descriptions

    // Deltas: key pointing to dummy means delete, key pointing to other object means update.
    // This approach lets us delete and update while shortening the locked critical section.
    Iterator<Map.Entry<String, KinesisStreamDescription>> deltasIter = deltasMap.entrySet().iterator();
    internalMapLock.writeLock().lock();
    try {
        while (deltasIter.hasNext()) {
            Map.Entry<String, KinesisStreamDescription> entry = deltasIter.next();
            if (entry.getValue().getTableName().equals("__DUMMY__")) {
                this.internalMap.remove(entry.getKey());
            } else {
                this.internalMap.put(entry.getKey(), entry.getValue());
            }
        }
    } finally {
        internalMapLock.writeLock().unlock();
    }

    log.info("Completed updating table definitions from S3.");
    this.lastCheck = now;

    return;
}

From source file:com.github.lbroudoux.elasticsearch.river.s3.connector.S3Connector.java

License:Apache License

/**
 * Get the download url of this S3 object. May return null if the
 * object bucket and key cannot be converted to a URL.
 * @param summary A S3 object/*from  w  w  w  . j  a v  a2s  .co m*/
 * @param feedDefinition The holder of S3 feed definition.
 * @return The resource url if possible (access is subject to AWS credential)
 */
public String getDownloadUrl(S3ObjectSummary summary, S3RiverFeedDefinition feedDefinition) {
    String resourceUrl = s3Client.getResourceUrl(summary.getBucketName(), summary.getKey());
    // If a download host (actually a vhost such as cloudfront offers) is specified, use it to
    // recreate a vhosted resource url. This is made by substitution of the generic host name in url. 
    if (resourceUrl != null && feedDefinition.getDownloadHost() != null) {
        int hostPosEnd = resourceUrl.indexOf("s3.amazonaws.com/") + "s3.amazonaws.com".length();
        String vhostResourceUrl = feedDefinition.getDownloadHost() + resourceUrl.substring(hostPosEnd);
        return vhostResourceUrl;
    }
    return resourceUrl;
}

From source file:com.ikanow.infinit.e.harvest.extraction.document.file.AwsInfiniteFile.java

License:Open Source License

@Override
public InfiniteFile[] listFiles(Date optionalFilterDate, int maxDocs) {
    InfiniteFile[] fileList = null;/*from   www.  j a va  2 s.  c o  m*/
    ObjectListing list = null;
    _overwriteTime = 0L;
    ListObjectsRequest listRequest = new ListObjectsRequest().withBucketName(_awsBucketName);
    if (null != _awsObjectName) {
        listRequest.withPrefix(_awsObjectName);
    }
    listRequest.withDelimiter("/");
    list = ((AmazonS3Client) _awsClient).listObjects(listRequest);
    fileList = new InfiniteFile[list.getObjectSummaries().size() + list.getCommonPrefixes().size()];
    //TESTED (3.2)
    int nAdded = 0;
    // Get the sub-directories
    for (String subDir : list.getCommonPrefixes()) {
        // Create directories:
        fileList[nAdded] = new AwsInfiniteFile(_awsBucketName, subDir, null, _awsClient);
        nAdded++;
    } //TESTED (3b.3)
      // Get the files:
    for (S3ObjectSummary s3Obj : list.getObjectSummaries()) {
        if (!s3Obj.getKey().endsWith("/")) {
            fileList[nAdded] = new AwsInfiniteFile(s3Obj.getBucketName(), s3Obj.getKey(),
                    s3Obj.getLastModified(), _awsClient);
            long fileTime = fileList[nAdded].getDate();
            if (fileTime > _overwriteTime) {
                _overwriteTime = fileTime;
            } //TESTED (3.2)
            nAdded++;
        }
    }
    return fileList;
}

From source file:com.jktsoftware.amazondownloader.download.S3TypeBucket.java

License:Open Source License

public List<IObject> getObjectsInRepo() {
    String repoid = getRepoId();/*ww w . ja  v  a 2s  . co  m*/
    AWSCredentials awscredentials = new BasicAWSCredentials(this.credentials.getAccessKey(),
            this.credentials.getSecretAccessKey());

    AmazonS3 s3 = new AmazonS3Client(awscredentials);
    s3.setEndpoint(endpoint);

    System.out.println("Getting objects");
    ObjectListing objectListing = s3.listObjects(repoid);

    List<IObject> objects = new ArrayList<IObject>();

    for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
        S3TypeObject obj = new S3TypeObject(objectSummary.getKey(), objectSummary.getSize(),
                objectSummary.getBucketName(), objectSummary.getStorageClass(), s3);
        objects.add(obj);
    }
    return objects;
}

From source file:com.miovision.oss.awsbillingtools.s3.scanner.S3BillingRecordFileScanner.java

License:Open Source License

protected S3BillingRecordFile parse(S3ObjectSummary s3ObjectSummary) {
    final String bucketName = s3ObjectSummary.getBucketName();
    final String key = s3ObjectSummary.getKey();

    return S3BillingRecordFile.parseS3Key(bucketName, key, DELIMITER).orElse(null);
}

From source file:com.netflix.ice.processor.BillingFileProcessor.java

License:Apache License

@Override
protected void poll() throws Exception {

    TreeMap<DateTime, List<BillingFile>> filesToProcess = Maps.newTreeMap();
    Map<DateTime, List<BillingFile>> monitorFilesToProcess = Maps.newTreeMap();

    // list the tar.gz file in billing file folder
    for (int i = 0; i < config.billingS3BucketNames.length; i++) {
        String billingS3BucketName = config.billingS3BucketNames[i];
        String billingS3BucketPrefix = config.billingS3BucketPrefixes.length > i
                ? config.billingS3BucketPrefixes[i]
                : "";
        String accountId = config.billingAccountIds.length > i ? config.billingAccountIds[i] : "";
        String billingAccessRoleName = config.billingAccessRoleNames.length > i
                ? config.billingAccessRoleNames[i]
                : "";
        String billingAccessExternalId = config.billingAccessExternalIds.length > i
                ? config.billingAccessExternalIds[i]
                : "";

        logger.info("trying to list objects in billing bucket " + billingS3BucketName
                + " using assume role, and external id " + billingAccessRoleName + " "
                + billingAccessExternalId);
        List<S3ObjectSummary> objectSummaries = AwsUtils.listAllObjects(billingS3BucketName,
                billingS3BucketPrefix, accountId, billingAccessRoleName, billingAccessExternalId);
        logger.info("found " + objectSummaries.size() + " in billing bucket " + billingS3BucketName);
        TreeMap<DateTime, S3ObjectSummary> filesToProcessInOneBucket = Maps.newTreeMap();
        Map<DateTime, S3ObjectSummary> monitorFilesToProcessInOneBucket = Maps.newTreeMap();

        // for each file, download&process if not needed
        for (S3ObjectSummary objectSummary : objectSummaries) {

            String fileKey = objectSummary.getKey();
            DateTime dataTime = AwsUtils.getDateTimeFromFileNameWithTags(fileKey);
            boolean withTags = true;
            if (dataTime == null) {
                dataTime = AwsUtils.getDateTimeFromFileName(fileKey);
                withTags = false;/*ww w  . j a v a2 s  .com*/
            }

            if (dataTime != null && !dataTime.isBefore(config.startDate)) {
                if (!filesToProcessInOneBucket.containsKey(dataTime)
                        || withTags && config.resourceService != null
                        || !withTags && config.resourceService == null)
                    filesToProcessInOneBucket.put(dataTime, objectSummary);
                else
                    logger.info("ignoring file " + objectSummary.getKey());
            } else {
                logger.info("ignoring file " + objectSummary.getKey());
            }
        }

        for (S3ObjectSummary objectSummary : objectSummaries) {
            String fileKey = objectSummary.getKey();
            DateTime dataTime = AwsUtils.getDateTimeFromFileNameWithMonitoring(fileKey);

            if (dataTime != null && !dataTime.isBefore(config.startDate)) {
                monitorFilesToProcessInOneBucket.put(dataTime, objectSummary);
            }
        }

        for (DateTime key : filesToProcessInOneBucket.keySet()) {
            List<BillingFile> list = filesToProcess.get(key);
            if (list == null) {
                list = Lists.newArrayList();
                filesToProcess.put(key, list);
            }
            list.add(new BillingFile(filesToProcessInOneBucket.get(key), accountId, billingAccessRoleName,
                    billingAccessExternalId, billingS3BucketPrefix));
        }

        for (DateTime key : monitorFilesToProcessInOneBucket.keySet()) {
            List<BillingFile> list = monitorFilesToProcess.get(key);
            if (list == null) {
                list = Lists.newArrayList();
                monitorFilesToProcess.put(key, list);
            }
            list.add(new BillingFile(monitorFilesToProcessInOneBucket.get(key), accountId,
                    billingAccessRoleName, billingAccessExternalId, billingS3BucketPrefix));
        }
    }

    for (DateTime dataTime : filesToProcess.keySet()) {
        startMilli = endMilli = dataTime.getMillis();
        init();

        boolean hasNewFiles = false;
        boolean hasTags = false;
        long lastProcessed = lastProcessTime(AwsUtils.monthDateFormat.print(dataTime));

        for (BillingFile billingFile : filesToProcess.get(dataTime)) {
            S3ObjectSummary objectSummary = billingFile.s3ObjectSummary;
            if (objectSummary.getLastModified().getTime() < lastProcessed) {
                logger.info("data has been processed. ignoring " + objectSummary.getKey() + "...");
                continue;
            }
            hasNewFiles = true;
        }

        if (!hasNewFiles) {
            logger.info("data has been processed. ignoring all files at "
                    + AwsUtils.monthDateFormat.print(dataTime));
            continue;
        }

        long processTime = new DateTime(DateTimeZone.UTC).getMillis();
        for (BillingFile billingFile : filesToProcess.get(dataTime)) {

            S3ObjectSummary objectSummary = billingFile.s3ObjectSummary;
            String fileKey = objectSummary.getKey();

            File file = new File(config.localDir, fileKey.substring(billingFile.prefix.length()));
            logger.info("trying to download " + fileKey + "...");
            boolean downloaded = AwsUtils.downloadFileIfChangedSince(objectSummary.getBucketName(),
                    billingFile.prefix, file, lastProcessed, billingFile.accountId, billingFile.accessRoleName,
                    billingFile.externalId);
            if (downloaded)
                logger.info("downloaded " + fileKey);
            else {
                logger.info("file already downloaded " + fileKey + "...");
            }

            logger.info("processing " + fileKey + "...");
            boolean withTags = fileKey.contains("with-resources-and-tags");
            hasTags = hasTags || withTags;
            processingMonitor = false;
            processBillingZipFile(file, withTags);
            logger.info("done processing " + fileKey);
        }

        if (monitorFilesToProcess.get(dataTime) != null) {
            for (BillingFile monitorBillingFile : monitorFilesToProcess.get(dataTime)) {

                S3ObjectSummary monitorObjectSummary = monitorBillingFile.s3ObjectSummary;
                if (monitorObjectSummary != null) {
                    String monitorFileKey = monitorObjectSummary.getKey();
                    logger.info("processing " + monitorFileKey + "...");
                    File monitorFile = new File(config.localDir,
                            monitorFileKey.substring(monitorFileKey.lastIndexOf("/") + 1));
                    logger.info("trying to download " + monitorFileKey + "...");
                    boolean downloaded = AwsUtils.downloadFileIfChangedSince(
                            monitorObjectSummary.getBucketName(), monitorBillingFile.prefix, monitorFile,
                            lastProcessed, monitorBillingFile.accountId, monitorBillingFile.accessRoleName,
                            monitorBillingFile.externalId);
                    if (downloaded)
                        logger.info("downloaded " + monitorFile);
                    else
                        logger.warn(monitorFile + "already downloaded...");
                    FileInputStream in = new FileInputStream(monitorFile);
                    try {
                        processingMonitor = true;
                        processBillingFile(monitorFile.getName(), in, true);
                    } catch (Exception e) {
                        logger.error("Error processing " + monitorFile, e);
                    } finally {
                        in.close();
                    }
                }
            }
        }

        if (dataTime.equals(filesToProcess.lastKey())) {
            int hours = (int) ((endMilli - startMilli) / 3600000L);
            logger.info("cut hours to " + hours);
            cutData(hours);
        }

        // now get reservation capacity to calculate upfront and un-used cost
        for (Ec2InstanceReservationPrice.ReservationUtilization utilization : Ec2InstanceReservationPrice.ReservationUtilization
                .values())
            processReservations(utilization);

        if (hasTags && config.resourceService != null)
            config.resourceService.commit();

        logger.info("archiving results for " + dataTime + "...");
        archive();
        logger.info("done archiving " + dataTime);

        updateProcessTime(AwsUtils.monthDateFormat.print(dataTime), processTime);
        if (dataTime.equals(filesToProcess.lastKey())) {
            sendOndemandCostAlert();
        }
    }

    logger.info("AWS usage processed.");
}

From source file:com.qubole.presto.kinesis.s3config.S3TableConfigClient.java

License:Apache License

/**
 * Connect to S3 directory to look for new or updated table definitions and then
 * update the map.//from   w ww. j  ava2s  .co m
 */
protected void updateTablesFromS3() {
    long now = System.currentTimeMillis();

    List<S3ObjectSummary> objectList = this.getObjectSummaries();
    AmazonS3Client s3client = this.clientManager.getS3Client();
    AmazonS3URI directoryURI = new AmazonS3URI(this.bucketUrl);

    for (S3ObjectSummary objInfo : objectList) {
        if (!this.internalMap.containsKey(objInfo.getKey())
                || objInfo.getLastModified().getTime() >= this.lastCheck) {
            // New or updated file, so we must read from AWS
            try {
                if (objInfo.getKey().endsWith("/")) {
                    continue;
                }

                log.info("Getting : %s - %s", objInfo.getBucketName(), objInfo.getKey());
                S3Object object = s3client
                        .getObject(new GetObjectRequest(objInfo.getBucketName(), objInfo.getKey()));

                StringBuilder resultStr = new StringBuilder("");
                try (BufferedReader reader = new BufferedReader(
                        new InputStreamReader(object.getObjectContent()))) {
                    boolean hasMore = true;
                    while (hasMore) {
                        String line = reader.readLine();
                        if (line != null) {
                            resultStr.append(line);
                        } else {
                            hasMore = false;
                        }
                    }

                    KinesisStreamDescription table = streamDescriptionCodec.fromJson(resultStr.toString());

                    internalMap.put(objInfo.getKey(), table);
                    log.info("Put table description into the map from %s", objInfo.getKey());
                } catch (IOException iox) {
                    log.error("Problem reading input stream from object.", iox);
                }
            } catch (AmazonServiceException ase) {
                StringBuilder sb = new StringBuilder();
                sb.append("Caught an AmazonServiceException, which means your request made it ");
                sb.append("to Amazon S3, but was rejected with an error response for some reason.\n");
                sb.append("Error Message:    " + ase.getMessage());
                sb.append("HTTP Status Code: " + ase.getStatusCode());
                sb.append("AWS Error Code:   " + ase.getErrorCode());
                sb.append("Error Type:       " + ase.getErrorType());
                sb.append("Request ID:       " + ase.getRequestId());
                log.error(sb.toString(), ase);
            } catch (AmazonClientException ace) {
                StringBuilder sb = new StringBuilder();
                sb.append("Caught an AmazonClientException, " + "which means the client encountered "
                        + "an internal error while trying to communicate" + " with S3, "
                        + "such as not being able to access the network.");
                sb.append("Error Message: " + ace.getMessage());
                log.error(sb.toString(), ace);
            }
        }
    } // end loop through object descriptions

    log.info("Completed updating table definitions from S3.");
    this.lastCheck = now;

    return;
}