Example usage for com.amazonaws.services.s3.model ObjectMetadata ObjectMetadata

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.model ObjectMetadata ObjectMetadata.

Prototype

public ObjectMetadata() 

Source Link

Usage

From source file:org.crypto.sse.IEX2LevAMAZON.java

License:Open Source License

/**
 * @param args//from w  w w .j ava2  s  .  c  o m
 * @throws Exception
 */
@SuppressWarnings("null")
public static void main(String[] args) throws Exception {

    //First Job
    Configuration conf = new Configuration();

    Job job = Job.getInstance(conf, "IEX-2Lev");

    job.setJarByClass(IEX2LevAMAZON.class);

    job.setMapperClass(MLK1.class);

    job.setReducerClass(RLK1.class);

    job.setMapOutputKeyClass(Text.class);

    job.setMapOutputValueClass(Text.class);

    job.setOutputKeyClass(Text.class);

    job.setNumReduceTasks(1);

    job.setOutputValueClass(ArrayListWritable.class);

    job.setInputFormatClass(FileNameKeyInputFormat.class);

    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));

    //Second Job
    Configuration conf2 = new Configuration();

    Job job2 = Job.getInstance(conf2, "IEX-2Lev");

    job2.setJarByClass(IEX2LevAMAZON.class);

    job2.setMapperClass(MLK2.class);

    job2.setReducerClass(RLK2.class);

    job2.setNumReduceTasks(1);

    job2.setMapOutputKeyClass(Text.class);

    job2.setMapOutputValueClass(Text.class);

    job2.setOutputKeyClass(Text.class);

    job2.setOutputValueClass(ArrayListWritable.class);

    job2.setInputFormatClass(FileNameKeyInputFormat.class);

    FileInputFormat.addInputPath(job2, new Path(args[0]));
    FileOutputFormat.setOutputPath(job2, new Path(args[2]));

    job.waitForCompletion(true);
    job2.waitForCompletion(true);

    //Here add your Amazon Credentials

    AWSCredentials credentials = new BasicAWSCredentials("XXXXXXXXXXXXXXXX", "XXXXXXXXXXXXXXXX");
    // create a client connection based on credentials
    AmazonS3 s3client = new AmazonS3Client(credentials);

    // create bucket - name must be unique for all S3 users
    String bucketName = "iexmaptest";

    S3Object s3object = s3client.getObject(new GetObjectRequest(bucketName, args[4]));
    System.out.println(s3object.getObjectMetadata().getContentType());
    System.out.println(s3object.getObjectMetadata().getContentLength());
    List<String> lines = new ArrayList<String>();

    String folderName = "2";

    BufferedReader reader = new BufferedReader(new InputStreamReader(s3object.getObjectContent()));
    String line;
    int counter = 0;
    while ((line = reader.readLine()) != null) {
        // can copy the content locally as well
        // using a buffered writer
        lines.add(line);
        System.out.println(line);
        // upload file to folder 
        String fileName = folderName + "/" + Integer.toString(counter);
        ByteArrayInputStream input = new ByteArrayInputStream(line.getBytes());
        s3client.putObject(bucketName, fileName, input, new ObjectMetadata());
        counter++;
    }

    Multimap<String, String> lookup = ArrayListMultimap.create();

    for (int i = 0; i < lines.size(); i++) {
        String[] tokens = lines.get(i).split("\\s+");
        for (int j = 1; j < tokens.length; j++) {
            lookup.put(tokens[0], tokens[j]);
        }
    }

    // Loading inverted index that associates files identifiers to keywords
    lines = new ArrayList<String>();
    s3object = s3client.getObject(new GetObjectRequest(bucketName, args[5]));
    System.out.println(s3object.getObjectMetadata().getContentType());
    System.out.println(s3object.getObjectMetadata().getContentLength());

    // Loading inverted index that associates keywords to identifiers

    reader = new BufferedReader(new InputStreamReader(s3object.getObjectContent()));
    while ((line = reader.readLine()) != null) {
        lines.add(line);
    }
    Multimap<String, String> lookup2 = ArrayListMultimap.create();
    for (int i = 0; i < lines.size(); i++) {
        String[] tokens = lines.get(i).split("\\s+");
        for (int j = 1; j < tokens.length; j++) {
            lookup2.put(tokens[0], tokens[j]);
        }
    }

    // Delete File
    try {
        s3client.deleteObject(new DeleteObjectRequest(bucketName, args[4]));
    } catch (AmazonServiceException ase) {
        System.out.println("Caught an AmazonServiceException.");
        System.out.println("Error Message:    " + ase.getMessage());
        System.out.println("HTTP Status Code: " + ase.getStatusCode());
        System.out.println("AWS Error Code:   " + ase.getErrorCode());
        System.out.println("Error Type:       " + ase.getErrorType());
        System.out.println("Request ID:       " + ase.getRequestId());
    } catch (AmazonClientException ace) {
        System.out.println("Caught an AmazonClientException.");
        System.out.println("Error Message: " + ace.getMessage());
    }

    /*
     * Start of IEX-2Lev construction
     */

    // Generation of keys for IEX-2Lev
    BufferedReader keyRead = new BufferedReader(new InputStreamReader(System.in));
    System.out.println("Enter your password :");
    String pass = keyRead.readLine();

    // You can change the size of the key; Here we set it to 128

    List<byte[]> listSK = IEX2Lev.keyGen(128, pass, "salt/salt", 100);

    // Generation of Local Multi-maps with Mapper job only without reducer

    Configuration conf3 = new Configuration();

    String testSerialization1 = new String(Base64.encodeBase64(Serializer.serialize(lookup)));
    String testSerialization2 = new String(Base64.encodeBase64(Serializer.serialize(lookup2)));

    String testSerialization3 = new String(Base64.encodeBase64(Serializer.serialize(listSK)));

    //String testSerialization2 = gson.toJson(lookup2);
    conf3.set("lookup", testSerialization1);
    conf3.set("lookup2", testSerialization2);
    conf3.set("setKeys", testSerialization3);

    Job job3 = Job.getInstance(conf3, "Local MM");

    job3.setJarByClass(IEX2LevAMAZON.class);

    job3.setMapperClass(LocalMM.class);

    job3.setNumReduceTasks(0);

    FileInputFormat.addInputPath(job3, new Path(args[2]));
    FileOutputFormat.setOutputPath(job3, new Path(args[3]));

    job3.waitForCompletion(true);

}

---

From source file:org.deeplearning4j.aws.s3.uploader.S3Uploader.java

License:Apache License

public void upload(InputStream is, String name, String bucketName) {
    AmazonS3 client = getClient();//from   w ww  . j a  v  a2s .co m
    bucketName = ensureValidBucketName(bucketName);
    List<Bucket> buckets = client.listBuckets();
    ObjectMetadata med = new ObjectMetadata();
    for (Bucket b : buckets)
        if (b.getName().equals(bucketName)) {
            client.putObject(bucketName, name, is, med);
            return;
        }

    //bucket didn't exist: createComplex it
    client.createBucket(bucketName);
    client.putObject(bucketName, name, is, med);

}

---

From source file:org.dspace.globus.S3Client.java

License:Open Source License

public String createObject(String metadata) {
    UUID key = UUID.randomUUID();
    try {//w  w  w.  j  av a 2  s. c o m
        byte[] metadataBytes = metadata.getBytes("UTF-8");
        InputStream stream = new ByteArrayInputStream(metadataBytes);
        ObjectMetadata s3ObjectMeta = new ObjectMetadata();
        s3ObjectMeta.setContentLength(metadataBytes.length);
        s3Client.putObject(new PutObjectRequest(bucketName, key.toString(), stream, s3ObjectMeta));
        return key.toString();
    } catch (AmazonServiceException ase) {
        logger.error("AmazonServiceException: " + ase.getMessage());
    } catch (AmazonClientException ace) {
        logger.error("AmazonClientException: " + ace.getMessage());
    } catch (Exception e) {
        logger.error("Exception: " + e.getMessage());
    }
    return null;
}

---

From source file:org.duracloud.s3storage.S3StorageProvider.java

License:Apache License

/**
 * Adds content to a hidden space./* ww w  . j  a  va 2  s .c  o  m*/
 *
 * @param spaceId         hidden spaceId
 * @param contentId
 * @param contentMimeType
 * @param content
 * @return
 */
public String addHiddenContent(String spaceId, String contentId, String contentMimeType, InputStream content) {
    log.debug("addHiddenContent(" + spaceId + ", " + contentId + ", " + contentMimeType + ")");

    // Will throw if bucket does not exist
    String bucketName = getBucketName(spaceId);

    // Wrap the content in order to be able to retrieve a checksum

    if (contentMimeType == null || contentMimeType.equals("")) {
        contentMimeType = DEFAULT_MIMETYPE;
    }

    ObjectMetadata objMetadata = new ObjectMetadata();
    objMetadata.setContentType(contentMimeType);

    PutObjectRequest putRequest = new PutObjectRequest(bucketName, contentId, content, objMetadata);
    putRequest.setStorageClass(DEFAULT_STORAGE_CLASS);
    putRequest.setCannedAcl(CannedAccessControlList.Private);

    try {
        PutObjectResult putResult = s3Client.putObject(putRequest);
        return putResult.getETag();
    } catch (AmazonClientException e) {
        String err = "Could not add content " + contentId + " with type " + contentMimeType + " to S3 bucket "
                + bucketName + " due to error: " + e.getMessage();
        throw new StorageException(err, e, NO_RETRY);
    }

}

---

From source file:org.duracloud.s3storage.S3StorageProvider.java

License:Apache License

/**
 * {@inheritDoc}/*from   ww  w  . j av a 2s  . c  o m*/
 */
public String addContent(String spaceId, String contentId, String contentMimeType,
        Map<String, String> userProperties, long contentSize, String contentChecksum, InputStream content) {
    log.debug("addContent(" + spaceId + ", " + contentId + ", " + contentMimeType + ", " + contentSize + ", "
            + contentChecksum + ")");

    // Will throw if bucket does not exist
    String bucketName = getBucketName(spaceId);

    // Wrap the content in order to be able to retrieve a checksum
    ChecksumInputStream wrappedContent = new ChecksumInputStream(content, contentChecksum);

    String contentEncoding = removeContentEncoding(userProperties);

    userProperties = removeCalculatedProperties(userProperties);

    if (contentMimeType == null || contentMimeType.equals("")) {
        contentMimeType = DEFAULT_MIMETYPE;
    }

    ObjectMetadata objMetadata = new ObjectMetadata();
    objMetadata.setContentType(contentMimeType);
    if (contentSize > 0) {
        objMetadata.setContentLength(contentSize);
    }
    if (null != contentChecksum && !contentChecksum.isEmpty()) {
        String encodedChecksum = ChecksumUtil.convertToBase64Encoding(contentChecksum);
        objMetadata.setContentMD5(encodedChecksum);
    }

    if (contentEncoding != null) {
        objMetadata.setContentEncoding(contentEncoding);
    }

    if (userProperties != null) {
        for (String key : userProperties.keySet()) {
            String value = userProperties.get(key);

            if (log.isDebugEnabled()) {
                log.debug("[" + key + "|" + value + "]");
            }

            objMetadata.addUserMetadata(getSpaceFree(encodeHeaderKey(key)), encodeHeaderValue(value));
        }
    }

    PutObjectRequest putRequest = new PutObjectRequest(bucketName, contentId, wrappedContent, objMetadata);
    putRequest.setStorageClass(DEFAULT_STORAGE_CLASS);
    putRequest.setCannedAcl(CannedAccessControlList.Private);

    // Add the object
    String etag;
    try {
        PutObjectResult putResult = s3Client.putObject(putRequest);
        etag = putResult.getETag();
    } catch (AmazonClientException e) {
        if (e instanceof AmazonS3Exception) {
            AmazonS3Exception s3Ex = (AmazonS3Exception) e;
            String errorCode = s3Ex.getErrorCode();
            Integer statusCode = s3Ex.getStatusCode();
            String message = MessageFormat.format(
                    "exception putting object {0} into {1}: errorCode={2},"
                            + "  statusCode={3}, errorMessage={4}",
                    contentId, bucketName, errorCode, statusCode, e.getMessage());

            if (errorCode.equals("InvalidDigest") || errorCode.equals("BadDigest")) {
                log.error(message, e);

                String err = "Checksum mismatch detected attempting to add " + "content " + contentId
                        + " to S3 bucket " + bucketName + ". Content was not added.";
                throw new ChecksumMismatchException(err, e, NO_RETRY);
            } else if (errorCode.equals("IncompleteBody")) {
                log.error(message, e);
                throw new StorageException("The content body was incomplete for " + contentId + " to S3 bucket "
                        + bucketName + ". Content was not added.", e, NO_RETRY);
            } else if (!statusCode.equals(HttpStatus.SC_SERVICE_UNAVAILABLE)
                    && !statusCode.equals(HttpStatus.SC_NOT_FOUND)) {
                log.error(message, e);
            } else {
                log.warn(message, e);
            }
        } else {
            String err = MessageFormat.format("exception putting object {0} into {1}: {2}", contentId,
                    bucketName, e.getMessage());
            log.error(err, e);
        }

        // Check to see if file landed successfully in S3, despite the exception
        etag = doesContentExistWithExpectedChecksum(bucketName, contentId, contentChecksum);
        if (null == etag) {
            String err = "Could not add content " + contentId + " with type " + contentMimeType + " and size "
                    + contentSize + " to S3 bucket " + bucketName + " due to error: " + e.getMessage();
            throw new StorageException(err, e, NO_RETRY);
        }
    }

    // Compare checksum
    String providerChecksum = getETagValue(etag);
    String checksum = wrappedContent.getMD5();
    StorageProviderUtil.compareChecksum(providerChecksum, spaceId, contentId, checksum);
    return providerChecksum;
}

---

From source file:org.duracloud.s3storage.S3StorageProvider.java

License:Apache License

/**
 * {@inheritDoc}/*from w w  w  . j av a  2s.com*/
 */
public void setContentProperties(String spaceId, String contentId, Map<String, String> contentProperties) {
    log.debug("setContentProperties(" + spaceId + ", " + contentId + ")");

    // Will throw if bucket does not exist
    String bucketName = getBucketName(spaceId);

    String contentEncoding = removeContentEncoding(contentProperties);

    contentProperties = removeCalculatedProperties(contentProperties);

    // Determine mimetype, from properties list or existing value
    String mimeType = contentProperties.remove(PROPERTIES_CONTENT_MIMETYPE);
    if (mimeType == null || mimeType.equals("")) {
        Map<String, String> existingMeta = getContentProperties(spaceId, contentId);
        String existingMime = existingMeta.get(StorageProvider.PROPERTIES_CONTENT_MIMETYPE);
        if (existingMime != null) {
            mimeType = existingMime;
        }
    }

    // Collect all object properties
    ObjectMetadata objMetadata = new ObjectMetadata();
    for (String key : contentProperties.keySet()) {
        if (log.isDebugEnabled()) {
            log.debug("[" + key + "|" + contentProperties.get(key) + "]");
        }
        objMetadata.addUserMetadata(getSpaceFree(key), contentProperties.get(key));
    }

    // Set Content-Type
    if (mimeType != null && !mimeType.equals("")) {
        objMetadata.setContentType(mimeType);
    }

    // Set Content-Encoding
    if (contentEncoding != null && !contentEncoding.equals("")) {
        objMetadata.setContentEncoding(contentEncoding);
    }

    updateObjectProperties(bucketName, contentId, objMetadata);
}

---

From source file:org.eclipse.hawkbit.artifact.repository.S3Repository.java

License:Open Source License

private ObjectMetadata createObjectMetadata(final String mdMD5Hash16, final String contentType,
        final File file) {
    final ObjectMetadata objectMetadata = new ObjectMetadata();
    final String mdMD5Hash64 = BaseEncoding.base64()
            .encode(BaseEncoding.base16().lowerCase().decode(mdMD5Hash16));
    objectMetadata.setContentMD5(mdMD5Hash64);
    objectMetadata.setContentType(contentType);
    objectMetadata.setContentLength(file.length());
    objectMetadata.setHeader("x-amz-meta-md5chksum", mdMD5Hash64);
    if (s3Properties.isServerSideEncryption()) {
        objectMetadata.setHeader(Headers.SERVER_SIDE_ENCRYPTION,
                s3Properties.getServerSideEncryptionAlgorithm());
    }/*from w  ww . j  a v a 2  s  .  co  m*/
    return objectMetadata;
}

---

From source file:org.elasticsearch.cloud.aws.blobstore.DefaultS3OutputStream.java

License:Apache License

protected void doUpload(S3BlobStore blobStore, String bucketName, String blobName, InputStream is, int length,
        boolean serverSideEncryption) throws AmazonS3Exception {
    ObjectMetadata md = new ObjectMetadata();
    if (serverSideEncryption) {
        md.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
    }//w ww  .  j  av a  2s.  c om
    md.setContentLength(length);
    blobStore.client().putObject(bucketName, blobName, is, md);
}

---

From source file:org.elasticsearch.cloud.aws.blobstore.DefaultS3OutputStream.java

License:Apache License

protected String doInitialize(S3BlobStore blobStore, String bucketName, String blobName,
        boolean serverSideEncryption) {
    InitiateMultipartUploadRequest request = new InitiateMultipartUploadRequest(bucketName, blobName);
    if (serverSideEncryption) {
        ObjectMetadata md = new ObjectMetadata();
        md.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
        request.setObjectMetadata(md);// w  w w.  ja v a  2  s  .c om
    }
    return blobStore.client().initiateMultipartUpload(request).getUploadId();
}

---

From source file:org.elasticsearch.cloud.aws.blobstore.MockAmazonS3.java

License:Apache License

@Override
public ObjectMetadata getObjectMetadata(GetObjectMetadataRequest getObjectMetadataRequest)
        throws AmazonClientException, AmazonServiceException {
    String blobName = getObjectMetadataRequest.getKey();

    if (!blobs.containsKey(blobName)) {
        throw new AmazonS3Exception("[" + blobName + "] does not exist.");
    }//from  ww  w.j  a v a 2  s.c  om

    return new ObjectMetadata(); // nothing is done with it
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• »