Example usage for com.amazonaws.services.s3.model S3ObjectSummary getKey

List of usage examples for com.amazonaws.services.s3.model S3ObjectSummary getKey

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.services.s3.model.*
  5. S3ObjectSummary
  6. getKey

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.model S3ObjectSummary getKey.

Prototype

public String getKey()

Source Link

Document

Gets the key under which this object is stored in Amazon S3.

Usage

From source file:org.apache.nifi.minifi.c2.cache.s3.S3WritableConfiguration.java

License:Apache License

/**
 * Creates a new S3 writable configuration.
 * @param s3 An S3 {@link AmazonS3 client}.
 * @param s3ObjectSummary The S3 object {@link S3ObjectSummary summary}.
 * @param version The version of the configuration.
 *//*w w w  .  java 2s. c  o  m*/
public S3WritableConfiguration(AmazonS3 s3, S3ObjectSummary s3ObjectSummary, String version) {

    this.s3 = s3;
    this.s3Object = s3.getObject(s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey());
    this.version = version;

}

From source file:org.apache.nifi.processors.aws.s3.AbstractS3IT.java

License:Apache License

@AfterClass
public static void oneTimeTearDown() {
    // Empty the bucket before deleting it.
    try {/*from   w ww.j av a2 s  . c  om*/
        ObjectListing objectListing = client.listObjects(BUCKET_NAME);

        while (true) {
            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                client.deleteObject(BUCKET_NAME, objectSummary.getKey());
            }

            if (objectListing.isTruncated()) {
                objectListing = client.listNextBatchOfObjects(objectListing);
            } else {
                break;
            }
        }

        DeleteBucketRequest dbr = new DeleteBucketRequest(BUCKET_NAME);
        client.deleteBucket(dbr);
    } catch (final AmazonS3Exception e) {
        System.err.println("Unable to delete bucket " + BUCKET_NAME + e.toString());
    }

    if (client.doesBucketExist(BUCKET_NAME)) {
        Assert.fail("Incomplete teardown, subsequent tests might fail");
    }

}

From source file:org.apache.streams.s3.S3PersistReader.java

License:Apache License

public void prepare(Object configurationObject) {
    // Connect to S3
    synchronized (this) {
        // Create the credentials Object
        AWSCredentials credentials = new BasicAWSCredentials(s3ReaderConfiguration.getKey(),
                s3ReaderConfiguration.getSecretKey());

        ClientConfiguration clientConfig = new ClientConfiguration();
        clientConfig.setProtocol(Protocol.valueOf(s3ReaderConfiguration.getProtocol().toString()));

        // We do not want path style access
        S3ClientOptions clientOptions = new S3ClientOptions();
        clientOptions.setPathStyleAccess(false);

        this.amazonS3Client = new AmazonS3Client(credentials, clientConfig);
        if (!Strings.isNullOrEmpty(s3ReaderConfiguration.getRegion()))
            this.amazonS3Client
                    .setRegion(Region.getRegion(Regions.fromName(s3ReaderConfiguration.getRegion())));
        this.amazonS3Client.setS3ClientOptions(clientOptions);
    }/*from  w ww  .ja v a2s. c  o  m*/

    final ListObjectsRequest request = new ListObjectsRequest()
            .withBucketName(this.s3ReaderConfiguration.getBucket())
            .withPrefix(s3ReaderConfiguration.getReaderPath()).withMaxKeys(500);

    ObjectListing listing = this.amazonS3Client.listObjects(request);

    this.files = new ArrayList<String>();

    /**
     * If you can list files that are in this path, then you must be dealing with a directory
     * if you cannot list files that are in this path, then you are most likely dealing with
     * a simple file.
     */
    boolean hasCommonPrefixes = listing.getCommonPrefixes().size() > 0 ? true : false;
    boolean hasObjectSummaries = listing.getObjectSummaries().size() > 0 ? true : false;

    if (hasCommonPrefixes || hasObjectSummaries) {
        // Handle the 'directory' use case
        do {
            if (hasCommonPrefixes) {
                for (String file : listing.getCommonPrefixes()) {
                    this.files.add(file);
                }
            } else {
                for (final S3ObjectSummary objectSummary : listing.getObjectSummaries()) {
                    this.files.add(objectSummary.getKey());
                }
            }

            // get the next batch.
            listing = this.amazonS3Client.listNextBatchOfObjects(listing);
        } while (listing.isTruncated());
    } else {
        // handle the single file use-case
        this.files.add(s3ReaderConfiguration.getReaderPath());
    }

    if (this.files.size() <= 0)
        LOGGER.error("There are no files to read");

    this.persistQueue = Queues.synchronizedQueue(new LinkedBlockingQueue<StreamsDatum>(10000));
    this.executor = Executors.newSingleThreadExecutor();
}

From source file:org.apache.tajo.storage.s3.S3TableSpace.java

License:Apache License

/**
 * Calculate the total size of all objects in the indicated bucket
 *
 * @param path to use/*w w w .  ja v  a  2s  .  com*/
 * @return calculated size
 * @throws IOException
 */
@Override
public long calculateSize(Path path) throws IOException {
    long totalBucketSize = 0L;

    if (s3Enabled) {
        String key = pathToKey(path);

        final FileStatus fileStatus = fs.getFileStatus(path);

        if (fileStatus.isDirectory()) {
            if (!key.isEmpty()) {
                key = key + "/";
            }

            ListObjectsRequest request = new ListObjectsRequest();
            request.setBucketName(uri.getHost());
            request.setPrefix(key);
            request.setMaxKeys(maxKeys);

            if (LOG.isDebugEnabled()) {
                LOG.debug("listStatus: doing listObjects for directory " + key);
            }

            ObjectListing objects = s3.listObjects(request);

            while (true) {
                for (S3ObjectSummary summary : objects.getObjectSummaries()) {
                    Path keyPath = keyToPath(summary.getKey()).makeQualified(uri, fs.getWorkingDirectory());

                    // Skip over keys that are ourselves and old S3N _$folder$ files
                    if (keyPath.equals(path) || summary.getKey().endsWith(S3N_FOLDER_SUFFIX)) {
                        if (LOG.isDebugEnabled()) {
                            LOG.debug("Ignoring: " + keyPath);
                        }
                        continue;
                    }

                    if (!objectRepresentsDirectory(summary.getKey(), summary.getSize())) {
                        totalBucketSize += summary.getSize();
                    }
                }

                if (objects.isTruncated()) {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("listStatus: list truncated - getting next batch");
                    }
                    objects = s3.listNextBatchOfObjects(objects);
                } else {
                    break;
                }
            }
        } else {
            return fileStatus.getLen();
        }
    } else {
        totalBucketSize = fs.getContentSummary(path).getLength();
    }

    return totalBucketSize;
}

From source file:org.apache.zeppelin.notebook.repo.OldS3NotebookRepo.java

License:Apache License

@Override
public List<OldNoteInfo> list(AuthenticationInfo subject) throws IOException {
    List<OldNoteInfo> infos = new LinkedList<>();
    OldNoteInfo info;//ww  w  . j a v  a  2 s  . c  o  m
    try {
        ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName)
                .withPrefix(user + "/" + "notebook");
        ObjectListing objectListing;
        do {
            objectListing = s3client.listObjects(listObjectsRequest);
            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                if (objectSummary.getKey().endsWith("note.json")) {
                    info = getNoteInfo(objectSummary.getKey());
                    if (info != null) {
                        infos.add(info);
                    }
                }
            }
            listObjectsRequest.setMarker(objectListing.getNextMarker());
        } while (objectListing.isTruncated());
    } catch (AmazonClientException ace) {
        throw new IOException("Unable to list objects in S3: " + ace, ace);
    }
    return infos;
}

From source file:org.apache.zeppelin.notebook.repo.OldS3NotebookRepo.java

License:Apache License

@Override
public void remove(String noteId, AuthenticationInfo subject) throws IOException {
    String key = user + "/" + "notebook" + "/" + noteId;
    final ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName)
            .withPrefix(key);/*from w  w w  .  jav  a  2s. c o m*/

    try {
        ObjectListing objects = s3client.listObjects(listObjectsRequest);
        do {
            for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
                s3client.deleteObject(bucketName, objectSummary.getKey());
            }
            objects = s3client.listNextBatchOfObjects(objects);
        } while (objects.isTruncated());
    } catch (AmazonClientException ace) {
        throw new IOException("Unable to remove note in S3: " + ace, ace);
    }
}

From source file:org.apache.zeppelin.notebook.repo.S3NotebookRepo.java

License:Apache License

@Override
public Map<String, NoteInfo> list(AuthenticationInfo subject) throws IOException {
    Map<String, NoteInfo> notesInfo = new HashMap<>();
    try {//w  w w  . j a  va 2 s.  c om
        ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName)
                .withPrefix(user + "/" + "notebook");
        ObjectListing objectListing;
        do {
            objectListing = s3client.listObjects(listObjectsRequest);
            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                if (objectSummary.getKey().endsWith(".zpln")) {
                    try {
                        NoteInfo info = getNoteInfo(objectSummary.getKey());
                        notesInfo.put(info.getId(), info);
                    } catch (IOException e) {
                        LOGGER.warn(e.getMessage());
                    }
                }
            }
            listObjectsRequest.setMarker(objectListing.getNextMarker());
        } while (objectListing.isTruncated());
    } catch (AmazonClientException ace) {
        throw new IOException("Unable to list objects in S3: " + ace, ace);
    }
    return notesInfo;
}

From source file:org.apache.zeppelin.notebook.repo.S3NotebookRepo.java

License:Apache License

@Override
public void remove(String noteId, String notePath, AuthenticationInfo subject) throws IOException {
    String key = rootFolder + "/" + buildNoteFileName(noteId, notePath);
    final ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName)
            .withPrefix(key);//from www  . j a  va 2  s .co  m
    try {
        ObjectListing objects = s3client.listObjects(listObjectsRequest);
        do {
            for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
                s3client.deleteObject(bucketName, objectSummary.getKey());
            }
            objects = s3client.listNextBatchOfObjects(objects);
        } while (objects.isTruncated());
    } catch (AmazonClientException ace) {
        throw new IOException("Unable to remove note in S3: " + ace, ace);
    }
}

From source file:org.benetech.secureapp.generator.AmazonS3Utils.java

License:Open Source License

public static String getUniqueBuildNumber(HttpSession session, String apkNameWithNoSagBuild)
        throws S3Exception {
    try {//from   w w w.  j a v a2 s . com
        String partialApkName = apkNameWithNoSagBuild.toLowerCase();
        int greatestBuildNumberFound = 0;

        AmazonS3 s3 = getS3();
        ObjectListing listing = s3.listObjects(getDownloadS3Bucket(), AMAZON_DOWNLOADS_DIRECTORY);
        List<S3ObjectSummary> summaries = listing.getObjectSummaries();
        SagLogger.logDebug(session, "S3 ListObjects");

        while (listing.isTruncated()) {
            listing = s3.listNextBatchOfObjects(listing);
            summaries.addAll(listing.getObjectSummaries());
        }
        SagLogger.logDebug(session, "S3 Summaries Added");

        if (!summaries.isEmpty()) {
            SagLogger.logDebug(session, "S3 Summarys Found:" + summaries.size());
            for (Iterator<S3ObjectSummary> iterator = summaries.iterator(); iterator.hasNext();) {
                S3ObjectSummary currentApk = iterator.next();
                String currentApkName = currentApk.getKey().toLowerCase();
                String amazonObjectPartialName = AMAZON_DOWNLOADS_DIRECTORY + partialApkName;
                if (currentApkName.startsWith(amazonObjectPartialName)) {
                    int buildStartPos = amazonObjectPartialName.length() + 1;
                    int buildEndPos = currentApkName.length() - AppConfiguration.APK_EXTENSION.length();
                    String currentBuildNumberString = currentApkName.substring(buildStartPos, buildEndPos);
                    int currentBuildNumber = Integer.parseInt(currentBuildNumberString);
                    if (currentBuildNumber > greatestBuildNumberFound)
                        greatestBuildNumberFound = currentBuildNumber;
                }
            }
        }

        int nextSagBuildNumber = greatestBuildNumberFound + 1;
        return Integer.toString(nextSagBuildNumber);
    } catch (Exception e) {
        throw new S3Exception(e);
    }
}

From source file:org.boriken.s3fileuploader.S3SampleRefactored.java

License:Open Source License

public static void listFiles(AmazonS3 s3, String bucketName, String prefix) {
    /*/* w  w w .ja v a 2  s . c  om*/
      * List objects in your bucket by prefix - There are many options for
      * listing the objects in your bucket.  Keep in mind that buckets with
      * many objects might truncate their results when listing their objects,
      * so be sure to check if the returned object listing is truncated, and
      * use the AmazonS3.listNextBatchOfObjects(...) operation to retrieve
      * additional results.
      */
    System.out.println("Listing objects");
    ObjectListing objectListing = s3
            .listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix(prefix));

    for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
        System.out.println(" - " + objectSummary.getKey() + "  " + "(size = " + objectSummary.getSize() + ")");
    }
    System.out.println();

}