Example usage for com.amazonaws.services.s3.model ObjectListing getObjectSummaries

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.model ObjectListing getObjectSummaries.

Prototype

public List<S3ObjectSummary> getObjectSummaries() 

Source Link

Document

Gets the list of object summaries describing the objects stored in the S3 bucket.

Usage

From source file:org.geoserver.taskmanager.external.impl.S3FileServiceImpl.java

License:Open Source License

@Override
public FileReference getVersioned(String filePath) {
    int index = filePath.indexOf(PLACEHOLDER_VERSION);
    if (index < 0) {
        return new FileReferenceImpl(this, filePath, filePath);
    }//  ww  w  .  jav  a  2 s.c  o  m

    SortedSet<Integer> set = new TreeSet<Integer>();
    Pattern pattern = Pattern
            .compile(Pattern.quote(filePath).replace(FileService.PLACEHOLDER_VERSION, "\\E(.*)\\Q"));

    ObjectListing listing = getS3Client().listObjects(rootFolder, filePath.substring(0, index));
    for (S3ObjectSummary summary : listing.getObjectSummaries()) {
        Matcher matcher = pattern.matcher(summary.getKey());
        if (matcher.matches()) {
            try {
                set.add(Integer.parseInt(matcher.group(1)));
            } catch (NumberFormatException e) {
                LOGGER.log(Level.WARNING, "could not parse version in versioned file " + summary.getKey(), e);
            }
        }
    }
    int last = set.isEmpty() ? 0 : set.last();
    return new FileReferenceImpl(this, filePath.replace(FileService.PLACEHOLDER_VERSION, last + ""),
            filePath.replace(FileService.PLACEHOLDER_VERSION, (last + 1) + ""));
}

---

From source file:org.geppetto.persistence.s3.S3Manager.java

License:Open Source License

public List<S3ObjectSummary> retrievePathsFromS3(String prefix) {
    ObjectListing listing = getS3Connection().listObjects(PersistenceHelper.BUCKET_NAME, prefix);
    List<S3ObjectSummary> allSummaries = new ArrayList<>();
    List<S3ObjectSummary> summaries = listing.getObjectSummaries();
    while (!summaries.isEmpty()) {
        allSummaries.addAll(summaries);/*from w  w w  .jav  a 2 s.c  om*/
        summaries.clear();
        if (listing.isTruncated()) {
            summaries = listing.getObjectSummaries();
        }
    }
    return allSummaries;
}

---

From source file:org.gradle.internal.resource.transport.aws.s3.S3Client.java

License:Apache License

private List<String> resolveResourceNames(ObjectListing objectListing) {
    List<String> results = new ArrayList<String>();
    List<S3ObjectSummary> objectSummaries = objectListing.getObjectSummaries();
    if (null != objectSummaries) {
        for (S3ObjectSummary objectSummary : objectSummaries) {
            String key = objectSummary.getKey();
            String fileName = extractResourceName(key);
            if (null != fileName) {
                results.add(fileName);/*from  w  ww.j  a va 2s.c o m*/
            }
        }
    }
    return results;
}

---

From source file:org.gradle.internal.resource.transport.aws.s3.S3ResourceResolver.java

License:Apache License

private List<String> resolveFileResourceNames(ObjectListing objectListing) {
    List<S3ObjectSummary> objectSummaries = objectListing.getObjectSummaries();
    if (null != objectSummaries) {
        return ImmutableList.copyOf(Iterables.filter(Iterables.transform(objectSummaries, EXTRACT_FILE_NAME),
                Predicates.notNull()));/*from   ww w  . ja v  a  2  s .  co  m*/
    }
    return Collections.emptyList();

}

---

From source file:org.gridgain.grid.spi.checkpoint.s3.GridS3CheckpointSpi.java

License:Open Source License

/** {@inheritDoc} */
@SuppressWarnings({ "BusyWait" })
@Override//  w  w  w . j av a2  s  .co m
public void spiStart(String gridName) throws GridSpiException {
    // Start SPI start stopwatch.
    startStopwatch();

    assertParameter(cred != null, "awsCredentials != null");

    if (log.isDebugEnabled()) {
        log.debug(configInfo("awsCredentials", cred));
        log.debug(configInfo("clientConfiguration", cfg));
        log.debug(configInfo("bucketNameSuffix", bucketNameSuffix));
    }

    if (cfg == null)
        U.warn(log, "Amazon client configuration is not set (will use default).");

    if (F.isEmpty(bucketNameSuffix)) {
        U.warn(log, "Bucket name suffix is null or empty (will use default bucket name).");

        bucketName = BUCKET_NAME_PREFIX + DFLT_BUCKET_NAME_SUFFIX;
    } else
        bucketName = BUCKET_NAME_PREFIX + bucketNameSuffix;

    s3 = cfg != null ? new AmazonS3Client(cred, cfg) : new AmazonS3Client(cred);

    if (!s3.doesBucketExist(bucketName)) {
        try {
            s3.createBucket(bucketName);

            if (log.isDebugEnabled())
                log.debug("Created S3 bucket: " + bucketName);

            while (!s3.doesBucketExist(bucketName))
                try {
                    U.sleep(200);
                } catch (GridInterruptedException e) {
                    throw new GridSpiException("Thread has been interrupted.", e);
                }
        } catch (AmazonClientException e) {
            try {
                if (!s3.doesBucketExist(bucketName))
                    throw new GridSpiException("Failed to create bucket: " + bucketName, e);
            } catch (AmazonClientException ignored) {
                throw new GridSpiException("Failed to create bucket: " + bucketName, e);
            }
        }
    }

    Collection<GridS3TimeData> s3TimeDataLst = new LinkedList<>();

    try {
        ObjectListing list = s3.listObjects(bucketName);

        while (true) {
            for (S3ObjectSummary sum : list.getObjectSummaries()) {
                GridS3CheckpointData data = read(sum.getKey());

                if (data != null) {
                    s3TimeDataLst.add(new GridS3TimeData(data.getExpireTime(), data.getKey()));

                    if (log.isDebugEnabled())
                        log.debug("Registered existing checkpoint from key: " + data.getKey());
                }
            }

            if (list.isTruncated())
                list = s3.listNextBatchOfObjects(list);
            else
                break;
        }
    } catch (AmazonClientException e) {
        throw new GridSpiException("Failed to read checkpoint bucket: " + bucketName, e);
    } catch (GridException e) {
        throw new GridSpiException("Failed to marshal/unmarshal objects in bucket: " + bucketName, e);
    }

    // Track expiration for only those data that are made by this node
    timeoutWrk = new GridS3TimeoutWorker();

    timeoutWrk.add(s3TimeDataLst);

    timeoutWrk.start();

    registerMBean(gridName, this, GridS3CheckpointSpiMBean.class);

    // Ack ok start.
    if (log.isDebugEnabled())
        log.debug(startInfo());
}

---

From source file:org.gridgain.grid.spi.discovery.tcp.ipfinder.s3.GridTcpDiscoveryS3IpFinder.java

License:Open Source License

/** {@inheritDoc} */
@Override// w  ww. j  a  va 2 s  .  c o  m
public Collection<InetSocketAddress> getRegisteredAddresses() throws GridSpiException {
    initClient();

    Collection<InetSocketAddress> addrs = new LinkedList<>();

    try {
        ObjectListing list = s3.listObjects(bucketName);

        while (true) {
            for (S3ObjectSummary sum : list.getObjectSummaries()) {
                String key = sum.getKey();

                StringTokenizer st = new StringTokenizer(key, DELIM);

                if (st.countTokens() != 2)
                    U.error(log, "Failed to parse S3 entry due to invalid format: " + key);
                else {
                    String addrStr = st.nextToken();
                    String portStr = st.nextToken();

                    int port = -1;

                    try {
                        port = Integer.parseInt(portStr);
                    } catch (NumberFormatException e) {
                        U.error(log, "Failed to parse port for S3 entry: " + key, e);
                    }

                    if (port != -1)
                        try {
                            addrs.add(new InetSocketAddress(addrStr, port));
                        } catch (IllegalArgumentException e) {
                            U.error(log, "Failed to parse port for S3 entry: " + key, e);
                        }
                }
            }

            if (list.isTruncated())
                list = s3.listNextBatchOfObjects(list);
            else
                break;
        }
    } catch (AmazonClientException e) {
        throw new GridSpiException("Failed to list objects in the bucket: " + bucketName, e);
    }

    return addrs;
}

---

From source file:org.gridgain.grid.spi.discovery.tcp.metricsstore.s3.GridTcpDiscoveryS3MetricsStore.java

License:Open Source License

/** {@inheritDoc} */
@Override//from   www .j  a  va  2 s  .c o m
protected Map<UUID, GridNodeMetrics> metrics0(Collection<UUID> nodeIds) throws GridSpiException {
    assert !F.isEmpty(nodeIds);

    initClient();

    Map<UUID, GridNodeMetrics> res = new HashMap<>();

    try {
        ObjectListing list = s3.listObjects(bucketName);

        while (true) {
            for (S3ObjectSummary sum : list.getObjectSummaries()) {
                UUID id = UUID.fromString(sum.getKey());

                if (!nodeIds.contains(id))
                    continue;

                InputStream in = null;

                try {
                    in = s3.getObject(bucketName, sum.getKey()).getObjectContent();

                    byte[] buf = new byte[GridDiscoveryMetricsHelper.METRICS_SIZE];

                    in.read(buf);

                    res.put(id, GridDiscoveryMetricsHelper.deserialize(buf, 0));
                } catch (IllegalArgumentException ignored) {
                    U.warn(log, "Failed to parse UUID from entry key: " + sum.getKey());
                } catch (IOException e) {
                    U.error(log, "Failed to get entry content [bucketName=" + bucketName + ", entry="
                            + id.toString() + ']', e);
                } finally {
                    U.closeQuiet(in);
                }
            }

            if (list.isTruncated())
                list = s3.listNextBatchOfObjects(list);
            else
                break;
        }
    } catch (AmazonClientException e) {
        throw new GridSpiException("Failed to list objects in the bucket: " + bucketName, e);
    }

    return res;
}

---

From source file:org.gridgain.grid.spi.discovery.tcp.metricsstore.s3.GridTcpDiscoveryS3MetricsStore.java

License:Open Source License

/** {@inheritDoc} */
@Override//from   w w w.  j ava2  s.co m
public Collection<UUID> allNodeIds() throws GridSpiException {
    initClient();

    Collection<UUID> res = new LinkedList<>();

    try {
        ObjectListing list = s3.listObjects(bucketName);

        while (true) {
            for (S3ObjectSummary sum : list.getObjectSummaries())
                try {
                    UUID id = UUID.fromString(sum.getKey());

                    res.add(id);
                } catch (IllegalArgumentException ignored) {
                    U.warn(log, "Failed to parse UUID from entry key: " + sum.getKey());
                }

            if (list.isTruncated())
                list = s3.listNextBatchOfObjects(list);
            else
                break;
        }
    } catch (AmazonClientException e) {
        throw new GridSpiException("Failed to list objects in the bucket: " + bucketName, e);
    }

    return res;
}

---

From source file:org.huahinframework.core.util.S3Utils.java

License:Apache License

/**
 * {@inheritDoc}//from www  .j  ava2s  .  c  o  m
 */
@Override
public void delete(String path) throws IOException, URISyntaxException {
    URI uri = new URI(path);
    String bucketName = uri.getHost();
    String key = uri.getPath().substring(1, uri.getPath().length());

    List<DeleteObjectsRequest.KeyVersion> keys = new ArrayList<DeleteObjectsRequest.KeyVersion>();
    String marker = null;
    for (;;) {
        ObjectListing ol = s3.listObjects(
                new ListObjectsRequest().withBucketName(bucketName).withPrefix(key).withMarker(marker));
        for (S3ObjectSummary objectSummary : ol.getObjectSummaries()) {
            keys.add(new DeleteObjectsRequest.KeyVersion(objectSummary.getKey()));
        }

        marker = ol.getNextMarker();
        if (marker == null) {
            break;
        }
    }

    s3.deleteObjects(new DeleteObjectsRequest(bucketName).withKeys(keys));
    s3.deleteObject(bucketName, key);
}

---

From source file:org.icgc.dcc.storage.server.repository.s3.S3ListingService.java

License:Open Source License

private void readBucket(String bucketName, String prefix, Consumer<S3ObjectSummary> callback) {
    val request = new ListObjectsRequest().withBucketName(bucketName).withPrefix(prefix);
    log.debug("Reading summaries from '{}/{}'...", bucketName, prefix);

    ObjectListing listing;
    do {//from  w w w  .  java 2s  .c  o  m
        listing = s3.listObjects(request);
        for (val objectSummary : listing.getObjectSummaries()) {
            callback.accept(objectSummary);
        }
        request.setMarker(listing.getNextMarker());
    } while (listing.isTruncated());
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• »