Example usage for com.amazonaws.services.s3.model ObjectMetadata getContentLength

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.model ObjectMetadata getContentLength.

Prototype

public long getContentLength() 

Source Link

Document

<p> Gets the Content-Length HTTP header indicating the size of the associated object in bytes.

Usage

From source file:alluxio.underfs.s3a.S3AUnderFileSystem.java

License:Apache License

@Override
public long getFileSize(String path) throws IOException {
    ObjectMetadata details = getObjectDetails(path);
    if (details != null) {
        return details.getContentLength();
    } else {/* w w  w  . j  av  a2s .  co m*/
        throw new FileNotFoundException(path);
    }
}

---

From source file:com.bigstep.S3Sampler.java

License:Apache License

@Override
public SampleResult runTest(JavaSamplerContext context) {
    // pull parameters
    String bucket = context.getParameter("bucket");
    String object = context.getParameter("object");
    String method = context.getParameter("method");
    String local_file_path = context.getParameter("local_file_path");
    String key_id = context.getParameter("key_id");
    String secret_key = context.getParameter("secret_key");
    String proxy_host = context.getParameter("proxy_host");
    String proxy_port = context.getParameter("proxy_port");
    String endpoint = context.getParameter("endpoint");

    log.debug("runTest:method=" + method + " local_file_path=" + local_file_path + " bucket=" + bucket
            + " object=" + object);

    SampleResult result = new SampleResult();
    result.sampleStart(); // start stopwatch

    try {//ww  w  .  j av  a2s . c  o  m
        ClientConfiguration config = new ClientConfiguration();
        if (proxy_host != null && !proxy_host.isEmpty()) {
            config.setProxyHost(proxy_host);
        }
        if (proxy_port != null && !proxy_port.isEmpty()) {
            config.setProxyPort(Integer.parseInt(proxy_port));
        }
        //config.setProtocol(Protocol.HTTP);

        AWSCredentials credentials = new BasicAWSCredentials(key_id, secret_key);

        AmazonS3 s3Client = new AmazonS3Client(credentials, config);
        if (endpoint != null && !endpoint.isEmpty()) {
            s3Client.setEndpoint(endpoint);
        }
        ObjectMetadata meta = null;

        if (method.equals("GET")) {
            File file = new File(local_file_path);
            //meta= s3Client.getObject(new GetObjectRequest(bucket, object), file);
            S3Object s3object = s3Client.getObject(bucket, object);
            S3ObjectInputStream stream = s3object.getObjectContent();
            //while(stream.skip(1024*1024)>0);
            stream.close();
        } else if (method.equals("PUT")) {
            File file = new File(local_file_path);
            s3Client.putObject(bucket, object, file);
        }

        result.sampleEnd(); // stop stopwatch
        result.setSuccessful(true);
        if (meta != null) {
            result.setResponseMessage(
                    "OK on url:" + bucket + "/" + object + ". Length=" + meta.getContentLength());
        } else {
            result.setResponseMessage("OK on url:" + bucket + "/" + object + ".No metadata");
        }
        result.setResponseCodeOK(); // 200 code

    } catch (Exception e) {
        result.sampleEnd(); // stop stopwatch
        result.setSuccessful(false);
        result.setResponseMessage("Exception: " + e);

        // get stack trace as a String to return as document data
        java.io.StringWriter stringWriter = new java.io.StringWriter();
        e.printStackTrace(new java.io.PrintWriter(stringWriter));
        result.setResponseData(stringWriter.toString());
        result.setDataType(org.apache.jmeter.samplers.SampleResult.TEXT);
        result.setResponseCode("500");
    }

    return result;
}

---

From source file:com.emc.ecs.sync.model.object.S3SyncObject.java

License:Open Source License

protected SyncMetadata toSyncMeta(ObjectMetadata s3meta) {
    SyncMetadata meta = new SyncMetadata();

    meta.setCacheControl(s3meta.getCacheControl());
    meta.setContentDisposition(s3meta.getContentDisposition());
    meta.setContentEncoding(s3meta.getContentEncoding());
    if (s3meta.getContentMD5() != null)
        meta.setChecksum(new Checksum("MD5", s3meta.getContentMD5()));
    meta.setContentType(s3meta.getContentType());
    meta.setHttpExpires(s3meta.getHttpExpiresDate());
    meta.setExpirationDate(s3meta.getExpirationTime());
    meta.setModificationTime(s3meta.getLastModified());
    meta.setContentLength(s3meta.getContentLength());
    meta.setUserMetadata(toMetaMap(s3meta.getUserMetadata()));

    return meta;//from   w w  w  .  jav  a2 s.com
}

---

From source file:com.emc.ecs.sync.target.S3Target.java

License:Open Source License

@Override
public void filter(SyncObject obj) {
    try {//from   w w w.j a v a2 s  .c o  m
        // skip the root of the bucket since it obviously exists
        if ("".equals(rootKey + obj.getRelativePath())) {
            log.debug("Target is bucket root; skipping");
            return;
        }

        // some sync objects lazy-load their metadata (i.e. AtmosSyncObject)
        // since this may be a timed operation, ensure it loads outside of other timed operations
        if (!(obj instanceof S3ObjectVersion) || !((S3ObjectVersion) obj).isDeleteMarker())
            obj.getMetadata();

        // Compute target key
        String targetKey = getTargetKey(obj);
        obj.setTargetIdentifier(AwsS3Util.fullPath(bucketName, targetKey));

        if (includeVersions) {
            ListIterator<S3ObjectVersion> sourceVersions = s3Source.versionIterator((S3SyncObject) obj);
            ListIterator<S3ObjectVersion> targetVersions = versionIterator(obj);

            boolean newVersions = false, replaceVersions = false;
            if (force) {
                replaceVersions = true;
            } else {

                // special workaround for bug where objects are listed, but they have no versions
                if (sourceVersions.hasNext()) {

                    // check count and etag/delete-marker to compare version chain
                    while (sourceVersions.hasNext()) {
                        S3ObjectVersion sourceVersion = sourceVersions.next();

                        if (targetVersions.hasNext()) {
                            S3ObjectVersion targetVersion = targetVersions.next();

                            if (sourceVersion.isDeleteMarker()) {

                                if (!targetVersion.isDeleteMarker())
                                    replaceVersions = true;
                            } else {

                                if (targetVersion.isDeleteMarker())
                                    replaceVersions = true;

                                else if (!sourceVersion.getETag().equals(targetVersion.getETag()))
                                    replaceVersions = true; // different checksum
                            }

                        } else if (!replaceVersions) { // source has new versions, but existing target versions are ok
                            newVersions = true;
                            sourceVersions.previous(); // back up one
                            putIntermediateVersions(sourceVersions, targetKey); // add any new intermediary versions (current is added below)
                        }
                    }

                    if (targetVersions.hasNext())
                        replaceVersions = true; // target has more versions

                    if (!newVersions && !replaceVersions) {
                        log.info("Source and target versions are the same. Skipping {}", obj.getRelativePath());
                        return;
                    }
                }
            }

            // something's off; must delete all versions of the object
            if (replaceVersions) {
                log.info(
                        "[{}]: version history differs between source and target; re-placing target version history with that from source.",
                        obj.getRelativePath());

                // collect versions in target
                List<DeleteObjectsRequest.KeyVersion> deleteVersions = new ArrayList<>();
                while (targetVersions.hasNext())
                    targetVersions.next(); // move cursor to end
                while (targetVersions.hasPrevious()) { // go in reverse order
                    S3ObjectVersion version = targetVersions.previous();
                    deleteVersions.add(new DeleteObjectsRequest.KeyVersion(targetKey, version.getVersionId()));
                }

                // batch delete all versions in target
                log.debug("[{}]: deleting all versions in target", obj.getRelativePath());
                s3.deleteObjects(new DeleteObjectsRequest(bucketName).withKeys(deleteVersions));

                // replay version history in target
                while (sourceVersions.hasPrevious())
                    sourceVersions.previous(); // move cursor to beginning
                putIntermediateVersions(sourceVersions, targetKey);
            }

        } else { // normal sync (no versions)
            Date sourceLastModified = obj.getMetadata().getModificationTime();
            long sourceSize = obj.getMetadata().getContentLength();

            // Get target metadata.
            ObjectMetadata destMeta = null;
            try {
                destMeta = s3.getObjectMetadata(bucketName, targetKey);
            } catch (AmazonS3Exception e) {
                if (e.getStatusCode() != 404)
                    throw new RuntimeException("Failed to check target key '" + targetKey + "' : " + e, e);
            }

            if (!force && obj.getFailureCount() == 0 && destMeta != null) {

                // Check overwrite
                Date destLastModified = destMeta.getLastModified();
                long destSize = destMeta.getContentLength();

                if (destLastModified.equals(sourceLastModified) && sourceSize == destSize) {
                    log.info("Source and target the same.  Skipping {}", obj.getRelativePath());
                    return;
                }
                if (destLastModified.after(sourceLastModified)) {
                    log.info("Target newer than source.  Skipping {}", obj.getRelativePath());
                    return;
                }
            }
        }

        // at this point we know we are going to write the object
        // Put [current object version]
        if (obj instanceof S3ObjectVersion && ((S3ObjectVersion) obj).isDeleteMarker()) {

            // object has version history, but is currently deleted
            log.debug("[{}]: deleting object in target to replicate delete marker in source.",
                    obj.getRelativePath());
            s3.deleteObject(bucketName, targetKey);
        } else {
            putObject(obj, targetKey);

            // if object has new metadata after the stream (i.e. encryption checksum), we must update S3 again
            if (obj.requiresPostStreamMetadataUpdate()) {
                log.debug("[{}]: updating metadata after sync as required", obj.getRelativePath());
                CopyObjectRequest cReq = new CopyObjectRequest(bucketName, targetKey, bucketName, targetKey);
                cReq.setNewObjectMetadata(AwsS3Util.s3MetaFromSyncMeta(obj.getMetadata()));
                s3.copyObject(cReq);
            }
        }
    } catch (Exception e) {
        throw new RuntimeException("Failed to store object: " + e, e);
    }
}

---

From source file:com.eucalyptus.objectstorage.providers.s3.S3ProviderClient.java

License:Open Source License

protected void populateResponseMetadata(final ObjectStorageDataResponseType reply,
        final ObjectMetadata metadata) {
    reply.setSize(metadata.getContentLength());
    reply.setContentDisposition(metadata.getContentDisposition());
    reply.setContentType(metadata.getContentType());
    reply.setEtag(metadata.getETag());//from w w  w. j a  v a2s.  co  m
    reply.setLastModified(metadata.getLastModified());

    if (metadata.getUserMetadata() != null && metadata.getUserMetadata().size() > 0) {
        if (reply.getMetaData() == null)
            reply.setMetaData(new ArrayList<MetaDataEntry>());

        for (String k : metadata.getUserMetadata().keySet()) {
            reply.getMetaData().add(new MetaDataEntry(k, metadata.getUserMetadata().get(k)));
        }
    }

}

---

From source file:com.facebook.presto.hive.PrestoS3FileSystem.java

License:Apache License

@Override
public FileStatus getFileStatus(Path path) throws IOException {
    if (path.getName().isEmpty()) {
        // the bucket root requires special handling
        if (getS3ObjectMetadata(path) != null) {
            return new FileStatus(0, true, 1, 0, 0, qualifiedPath(path));
        }// w ww. j  a  v a 2s .  c om
        throw new FileNotFoundException("File does not exist: " + path);
    }

    ObjectMetadata metadata = getS3ObjectMetadata(path);

    if (metadata == null) {
        // check if this path is a directory
        Iterator<LocatedFileStatus> iterator = listPrefix(path);
        if ((iterator != null) && iterator.hasNext()) {
            return new FileStatus(0, true, 1, 0, 0, qualifiedPath(path));
        }
        throw new FileNotFoundException("File does not exist: " + path);
    }

    return new FileStatus(metadata.getContentLength(), false, 1, BLOCK_SIZE.toBytes(),
            lastModifiedTime(metadata), qualifiedPath(path));
}

---

From source file:com.facebook.presto.hive.s3.PrestoS3FileSystem.java

License:Apache License

private static long getObjectSize(Path path, ObjectMetadata metadata) throws IOException {
    Map<String, String> userMetadata = metadata.getUserMetadata();
    String length = userMetadata.get(UNENCRYPTED_CONTENT_LENGTH);
    if (userMetadata.containsKey(SERVER_SIDE_ENCRYPTION) && length == null) {
        throw new IOException(
                format("%s header is not set on an encrypted object: %s", UNENCRYPTED_CONTENT_LENGTH, path));
    }/*from ww  w  . ja v a 2s  .  co  m*/
    return (length != null) ? Long.parseLong(length) : metadata.getContentLength();
}

---

From source file:com.github.rholder.esthree.command.GetMultipart.java

License:Apache License

@Override
public Integer call() throws Exception {
    ObjectMetadata om = amazonS3Client.getObjectMetadata(bucket, key);
    contentLength = om.getContentLength();

    // this is the most up to date digest, it's initialized here but later holds the most up to date valid digest
    currentDigest = MessageDigest.getInstance("MD5");
    chunkSize = chunkSize == null ? DEFAULT_CHUNK_SIZE : chunkSize;
    fileParts = Parts.among(contentLength, chunkSize);
    for (Part fp : fileParts) {

        /*// ww  w  .  ja v  a  2  s . co m
         * We'll need to compute the digest on the full incoming stream for
         * each valid chunk that comes in. Invalid chunks will need to be
         * recomputed and fed through a copy of the MD5 that was valid up
         * until the latest chunk.
         */
        currentDigest = retryingGetWithRange(fp.start, fp.end);
    }

    // TODO fix total content length progress bar
    if (progressListener != null) {
        progressListener.progressChanged(new ProgressEvent(ProgressEventType.TRANSFER_STARTED_EVENT));
    }

    String fullETag = om.getETag();
    if (!fullETag.contains("-")) {
        byte[] expected = BinaryUtils.fromHex(fullETag);
        byte[] current = currentDigest.digest();
        if (!Arrays.equals(expected, current)) {
            throw new AmazonClientException("Unable to verify integrity of data download.  "
                    + "Client calculated content hash didn't match hash calculated by Amazon S3.  "
                    + "The data may be corrupt.");
        }
    } else {
        // TODO log warning that we can't validate the MD5
        if (verbose) {
            System.err.println("\nMD5 does not exist on AWS for file, calculated value: "
                    + BinaryUtils.toHex(currentDigest.digest()));
        }
    }
    // TODO add ability to resume from previously downloaded chunks
    // TODO add rate limiter

    return 0;
}

---

From source file:com.hpe.caf.worker.datastore.s3.S3DataStore.java

License:Apache License

@Override
public long size(String reference) throws DataStoreException {
    LOG.debug("Received size request for {}", reference);

    try (QuietResource<S3Object> s3Object = new QuietResource<>(
            amazonS3Client.getObject(bucketName, reference))) {
        ObjectMetadata objectMetadata = s3Object.get().getObjectMetadata();
        return objectMetadata.getContentLength();
    } catch (Exception e) {
        errors.incrementAndGet();//from ww w  .  j a  va 2  s .c  om
        throw new DataStoreException("Failed to get data size for reference " + reference, e);
    }
}

---

From source file:com.ibm.stocator.fs.cos.COSAPIClient.java

License:Apache License

private FileStatus getFileStatusKeyBased(String key, Path path) throws AmazonS3Exception {
    LOG.trace("internal method - get file status by key {}, path {}", key, path);
    FileStatus cachedFS = memoryCache.getFileStatus(path.toString());
    if (cachedFS != null) {
        return cachedFS;
    }//from   w w  w.  j a v  a  2 s  .  c  om
    ObjectMetadata meta = mClient.getObjectMetadata(mBucket, key);
    String sparkOrigin = meta.getUserMetaDataOf("data-origin");
    boolean stocatorCreated = false;
    if (sparkOrigin != null) {
        String tmp = (String) sparkOrigin;
        if (tmp.equals("stocator")) {
            stocatorCreated = true;
        }
    }
    mCachedSparkOriginated.put(key, Boolean.valueOf(stocatorCreated));
    FileStatus fs = createFileStatus(meta.getContentLength(), key, meta.getLastModified(), path);
    memoryCache.putFileStatus(path.toString(), fs);
    return fs;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• »