Example usage for com.amazonaws.services.s3.model S3ObjectSummary getKey

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.model S3ObjectSummary getKey.

Prototype

public String getKey() 

Source Link

Document

Gets the key under which this object is stored in Amazon S3.

Usage

From source file:it.openutils.mgnlaws.magnolia.datastore.S3DataStore.java

License:Open Source License

/**
 * {@inheritDoc}//from  w  w  w .  j  a  v  a 2  s.co m
 */
public int deleteAllOlderThan(long min) throws DataStoreException {
    int deleted = 0;

    for (Iterator<S3ObjectSummary> i = new S3Iterator(amazonS3, bucket, PREFIX); i.hasNext();) {
        S3ObjectSummary rec = (S3ObjectSummary) i.next();
        if (rec.getLastModified().getTime() < min) {
            deleteRecord(rec.getKey());
            if (useCache) {
                cache.remove(getDataIdentifier(rec.getKey()));
            }
            deleted++;
        }
    }

    return deleted;
}

---

From source file:it.openutils.mgnlaws.magnolia.datastore.S3IdentifierIterator.java

License:Open Source License

@Override
public DataIdentifier next() {
    S3ObjectSummary summary = iterator.next();
    if (summary != null) {
        return new DataIdentifier(StringUtils.substringAfterLast(summary.getKey(), "/"));
    }/*from w  w  w  . j av a2 s. com*/
    return null;
}

---

From source file:Java21.S3Files.java

License:Open Source License

public static void main(String[] args) throws IOException {

    /*// w ww. j a v  a 2 s. c  o  m
     * The ProfileCredentialsProvider will return your [default]
     * credential profile by reading from the credentials file located at
     * (~/.aws/credentials).
     */
    AWSCredentials credentials = null;
    try {
        credentials = new ProfileCredentialsProvider().getCredentials();
    } catch (Exception e) {
        throw new AmazonClientException("Cannot load the credentials from the credential profiles file. "
                + "Please make sure that your credentials file is at the correct "
                + "location (~/.aws/credentials), and is in valid format.", e);
    }

    AmazonS3 s3 = new AmazonS3Client(credentials);
    Region usWest2 = Region.getRegion(Regions.US_WEST_2);
    s3.setRegion(usWest2);

    String bucketName = "msm-gb-env-etl-iq/dev1-dwh/" + UUID.randomUUID();
    //String key = "";

    System.out.println("===========================================");
    System.out.println("Getting Started with Amazon S3");
    System.out.println("===========================================\n");

    /*
     * List objects in your bucket by prefix - There are many options for
     * listing the objects in your bucket.  Keep in mind that buckets with
     * many objects might truncate their results when listing their objects,
     * so be sure to check if the returned object listing is truncated, and
     * use the AmazonS3.listNextBatchOfObjects(...) operation to retrieve
     * additional results.
     */
    System.out.println("Listing objects");
    ObjectListing objectListing = s3
            .listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix("My"));
    for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
        System.out.println(" - " + objectSummary.getKey() + "  " + "(size = " + objectSummary.getSize() + ")");
    }
    System.out.println();

}

---

From source file:jenkins.plugins.itemstorage.s3.Downloads.java

License:Open Source License

public void startDownload(TransferManager manager, File base, String pathPrefix, S3ObjectSummary summary)
        throws AmazonServiceException, IOException {
    // calculate target file name
    File targetFile = FileUtils.getFile(base, summary.getKey().substring(pathPrefix.length() + 1));

    // if target file exists, only download it if newer
    if (targetFile.lastModified() < summary.getLastModified().getTime()) {
        // ensure directory above file exists
        FileUtils.forceMkdir(targetFile.getParentFile());

        // Start the download
        Download download = manager.download(summary.getBucketName(), summary.getKey(), targetFile);

        // Keep for later
        startedDownloads.add(new Memo(download, targetFile, summary.getLastModified().getTime()));
    }/*  w ww.j a va 2 s .com*/
}

---

From source file:jenkins.plugins.itemstorage.s3.S3Profile.java

License:Open Source License

public void delete(String bucketName, String pathPrefix) {
    ObjectListing listing = null;/*from w ww.j  a  va 2s. c  o  m*/
    do {
        listing = listing == null ? helper.client().listObjects(bucketName, pathPrefix)
                : helper.client().listNextBatchOfObjects(listing);

        DeleteObjectsRequest req = new DeleteObjectsRequest(bucketName);

        List<DeleteObjectsRequest.KeyVersion> keys = new ArrayList<>(listing.getObjectSummaries().size());
        for (S3ObjectSummary summary : listing.getObjectSummaries()) {
            keys.add(new DeleteObjectsRequest.KeyVersion(summary.getKey()));
        }
        req.withKeys(keys);

        helper.client().deleteObjects(req);
    } while (listing.isTruncated());
}

---

From source file:jenkins.plugins.itemstorage.s3.S3Profile.java

License:Open Source License

public void rename(String bucketName, String currentPathPrefix, String newPathPrefix) {

    ObjectListing listing = null;/* w w  w . j a v  a  2s .  c  o m*/
    do {
        listing = listing == null ? helper.client().listObjects(bucketName, currentPathPrefix)
                : helper.client().listNextBatchOfObjects(listing);
        for (S3ObjectSummary summary : listing.getObjectSummaries()) {
            String key = summary.getKey();

            helper.client().copyObject(bucketName, key, bucketName,
                    newPathPrefix + key.substring(currentPathPrefix.length()));
            helper.client().deleteObject(bucketName, key);
        }
    } while (listing.isTruncated());
}

---

From source file:jenkins.plugins.itemstorage.s3.S3UploadAllCallable.java

License:Open Source License

private Map<String, S3ObjectSummary> lookupExistingCacheEntries(AmazonS3 s3) {
    Map<String, S3ObjectSummary> summaries = new HashMap<>();

    ObjectListing listing = s3.listObjects(bucketName, pathPrefix);
    do {/*from   w w w . j  av a2s.  co  m*/
        for (S3ObjectSummary summary : listing.getObjectSummaries()) {
            summaries.put(summary.getKey(), summary);
        }
        listing = listing.isTruncated() ? s3.listNextBatchOfObjects(listing) : null;
    } while (listing != null);

    return summaries;
}

---

From source file:jp.co.tagbangers.jgroups.S3_CLIENT_PING.java

License:Apache License

@Override
protected void readAll(List<Address> members, String clustername, Responses responses) {
    if (clustername == null) {
        return;/*from   ww w. j  a va 2 s  .c om*/
    }

    try {
        clustername = sanitize(clustername);
        ObjectListing objectListing = amazonS3.listObjects(location, clustername);
        for (S3ObjectSummary summary : objectListing.getObjectSummaries()) {
            try {
                readResponse(summary, members, responses);
            } catch (Throwable t) {
                log.error("failed reading key %s: %s", summary.getKey(), t);
            }

        }
    } catch (AmazonServiceException ex) {
        log.error("failed reading addresses", ex);
    }
}

---

From source file:jp.co.tagbangers.jgroups.S3_CLIENT_PING.java

License:Apache License

protected void readResponse(S3ObjectSummary summary, List<Address> members, Responses responses) {
    S3Object object = amazonS3.getObject(summary.getBucketName(), summary.getKey());
    List<PingData> list;//from w ww  .  j  a  v a  2s.  c o  m
    try {
        list = read(object.getObjectContent());
        if (list != null) {
            for (PingData data : list) {
                if (members == null || members.contains(data.getAddress())) {
                    responses.addResponse(data, data.isCoord());
                }
                if (local_addr != null && !local_addr.equals(data.getAddress())) {
                    addDiscoveryResponseToCaches(data.getAddress(), data.getLogicalName(),
                            data.getPhysicalAddr());
                }
            }
        }
    } catch (Throwable e) {
        log.error("failed unmarshalling response", e);
    }
}

---

From source file:jp.sanix.analysis.java

License:Open Source License

public static void main(String[] args) throws IOException, SQLException, AmazonServiceException, JSONException,
        NullPointerException, ParseException {

    if (args.length < 1) {
        System.out.println("Please specify at least an arguments that is pvs_serial_id.");
        System.out.println("analysis A9990004 // get today's data");
        System.out.println("analysis A9990004 2015/05/30 // get from 2015/05/30 to today's data");
        System.out.println("analysis A9990004 2015/05/30 2015/06/30");
        System.exit(-1);//from   ww w  .  ja  v a 2  s  . c  o  m
    }

    String id = args[0];

    SimpleDateFormat format = new SimpleDateFormat("yyyy/MM/dd");
    format.setTimeZone(TimeZone.getTimeZone("JST"));
    SimpleDateFormat normalformat = new SimpleDateFormat("yyyy/MM/dd'T'HH:mm:ss'Z'");
    normalformat.setTimeZone(TimeZone.getTimeZone("JST"));
    SimpleDateFormat pgformat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
    pgformat.setTimeZone(TimeZone.getTimeZone("UTC"));

    Calendar cal = Calendar.getInstance();
    Calendar end = Calendar.getInstance();
    String today = toDate(cal);

    System.out.println(id);
    try {
        cal.setTime(format.parse(args[1]));
    } catch (ParseException e) {
    } catch (ArrayIndexOutOfBoundsException e) {
    }
    try {
        end.setTime(format.parse(args[2]));
    } catch (ParseException e) {
    } catch (ArrayIndexOutOfBoundsException e) {
    }
    end.add(Calendar.DAY_OF_MONTH, 1);

    AmazonS3 s3 = new AmazonS3Client();

    s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1));

    Connection db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS);
    Statement st = db.createStatement();
    ResultSet rs = st.executeQuery(
            "SELECT data, pvs_unique_code FROM data WHERE pvs_serial_id='" + id + "' OFFSET 0 LIMIT 1;");
    rs.next();
    String json = rs.getString(1);
    String key = rs.getString(2);
    rs.close();
    db.close();
    Date recent = new Date();

    while (cal.before(end)) {
        xlsSheet xls = new xlsSheet(json);
        System.out.println("Getting data of " + toDate(cal));

        /* AWS S3????? */
        String bucketName = "pvdata-storage-production";
        //         String bucketName = "pvdata-storage-staging";
        System.out.println("Get s3 data by key='" + bucketName + "/data/" + key + "/" + toDate(cal) + "/'");
        ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName)
                .withPrefix("data/" + key + "/" + toDate(cal) + "/"));

        /* get data from s3 */
        do {
            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                String keyname = objectSummary.getKey();
                S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname));
                BufferedReader reader = new BufferedReader(
                        new InputStreamReader(new GZIPInputStream(object.getObjectContent())));
                while (reader.ready()) {
                    String line = reader.readLine();
                    try {
                        json = line.substring(line.indexOf("{"));
                        recent = parseDate(line.substring(0, 25));
                        xls.putData(json);
                    } catch (NullPointerException e) {
                    }
                }
                reader.close();
                object.close();
            }
            objectListing = s3.listNextBatchOfObjects(objectListing);
        } while (objectListing.getMarker() != null);

        /* if today, read postgres to get recent data */
        if (toDate(cal).equals(today)) {
            System.out.println("Get recent data from postgres");
            try {
                db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS);
                st = db.createStatement();
                String sql = "SELECT data FROM data WHERE pvs_unique_code='" + key + "' AND created_at > '"
                        + pgformat.format(recent) + "';";
                System.out.println(sql);
                rs = st.executeQuery(sql);
                while (rs.next()) {
                    json = rs.getString(1);
                    xls.putData(json);
                }
                rs.close();
                db.close();
            } catch (PSQLException e) {
            } catch (ParseException e) {
            }
        }
        System.out.println("Write Buffer");
        xls.writeBuffer();

        File file = File.createTempFile("temp", ".xlsx");
        file.deleteOnExit();
        xls.putFile(new FileOutputStream(file));
        System.out.println("Put S3");
        s3.putObject(new PutObjectRequest("sanix-data-analysis",
                FOLDER + id + "-" + toDate(cal).replace("/", "-") + ".xlsx", file));
        System.out.println("Finished: " + toDate(cal));

        cal.add(Calendar.DAY_OF_MONTH, 1);
    }

    File file = File.createTempFile("temp", ".html");
    file.deleteOnExit();
    BufferedWriter bw = new BufferedWriter(new FileWriter(file));
    ObjectListing objectListing = s3
            .listObjects(new ListObjectsRequest().withBucketName("sanix-data-analysis").withPrefix(FOLDER));
    bw.write("<html><head></head><body><ul>\n");
    do {
        for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
            String keyname = objectSummary.getKey();
            if (Pattern.compile("\\.xlsx$|\\.html$").matcher(keyname).find()) {
                bw.write("<li><a href=\"https://s3-ap-northeast-1.amazonaws.com/sanix-data-analysis/" + keyname
                        + "\">" + keyname.replaceAll("^[^\\/]*\\/", "") + "</a></li>\n");
            }
        }
        objectListing = s3.listNextBatchOfObjects(objectListing);
    } while (objectListing.getMarker() != null);
    bw.write("</ul></body></html>\n");
    bw.flush();
    bw.close();

    s3.putObject(new PutObjectRequest("sanix-data-analysis", FOLDER + "index.html", file));
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• »