Example usage for com.amazonaws.services.s3.model PutObjectRequest PutObjectRequest

List of usage examples for com.amazonaws.services.s3.model PutObjectRequest PutObjectRequest

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.services.s3.model.*
  5. PutObjectRequest
  6. PutObjectRequest

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.model PutObjectRequest PutObjectRequest.

Prototype

public PutObjectRequest(String bucketName, String key, String redirectLocation)

Source Link

Document

Constructs a new PutObjectRequest object to perform a redirect for the specified bucket and key.

Usage

From source file:org.apache.hadoop.fs.s3a.S3AOutputStream.java

License:Apache License

@Override
public synchronized void close() throws IOException {
    if (closed) {
        return;//from   w w  w .  java  2  s .c o  m
    }

    backupStream.close();
    LOG.info("OutputStream for key '" + key + "' closed. Now beginning upload");
    LOG.info("Minimum upload part size: " + partSize + " threshold " + partSizeThreshold);

    try {
        TransferManagerConfiguration transferConfiguration = new TransferManagerConfiguration();
        transferConfiguration.setMinimumUploadPartSize(partSize);
        transferConfiguration.setMultipartUploadThreshold(partSizeThreshold);

        TransferManager transfers = new TransferManager(client);
        transfers.setConfiguration(transferConfiguration);

        final ObjectMetadata om = new ObjectMetadata();
        if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) {
            om.setServerSideEncryption(serverSideEncryptionAlgorithm);
        }

        PutObjectRequest putObjectRequest = new PutObjectRequest(bucket, key, backupFile);
        putObjectRequest.setCannedAcl(cannedACL);
        putObjectRequest.setMetadata(om);

        Upload upload = transfers.upload(putObjectRequest);

        ProgressableProgressListener listener = new ProgressableProgressListener(upload, progress, statistics);
        upload.addProgressListener(listener);

        upload.waitForUploadResult();

        long delta = upload.getProgress().getBytesTransferred() - listener.getLastBytesTransferred();
        if (statistics != null && delta != 0) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("S3A write delta changed after finished: " + delta + " bytes");
            }
            statistics.incrementBytesWritten(delta);
        }

        // This will delete unnecessary fake parent directories
        fs.finishedWrite(key);
    } catch (InterruptedException e) {
        throw new IOException(e);
    } finally {
        if (!backupFile.delete()) {
            LOG.warn("Could not delete temporary s3a file: " + backupFile);
        }
        super.close();
        closed = true;
    }

    LOG.info("OutputStream for key '" + key + "' upload complete");
}

From source file:org.apache.hadoop.fs.s3r.S3RFileSystem.java

License:Apache License

/**
 * The src file is on the local disk.  Add it to FS at
 * the given dst name./*  w  w  w. j av a 2 s. com*/
 *
 * This version doesn't need to create a temporary file to calculate the md5.
 * Sadly this doesn't seem to be used by the shell cp :(
 *
 * delSrc indicates if the source should be removed
 * @param delSrc whether to delete the src
 * @param overwrite whether to overwrite an existing file
 * @param src path
 * @param dst path
 */
@Override
public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path src, Path dst) throws IOException {
    String key = pathToKey(dst);

    if (!overwrite && exists(dst)) {
        throw new IOException(dst + " already exists");
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Copying local file from " + src + " to " + dst);
    }

    // Since we have a local file, we don't need to stream into a temporary file
    LocalFileSystem local = getLocal(getConf());
    File srcfile = local.pathToFile(src);

    final ObjectMetadata om = new ObjectMetadata();
    if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) {
        om.setServerSideEncryption(serverSideEncryptionAlgorithm);
    }
    PutObjectRequest putObjectRequest = new PutObjectRequest(bucket, key, srcfile);
    putObjectRequest.setCannedAcl(cannedACL);
    putObjectRequest.setMetadata(om);

    ProgressListener progressListener = new ProgressListener() {
        public void progressChanged(ProgressEvent progressEvent) {
            switch (progressEvent.getEventCode()) {
            case ProgressEvent.PART_COMPLETED_EVENT_CODE:
                statistics.incrementWriteOps(1);
                break;
            default:
                break;
            }
        }
    };

    Upload up = transfers.upload(putObjectRequest);
    up.addProgressListener(progressListener);
    try {
        up.waitForUploadResult();
        statistics.incrementWriteOps(1);
    } catch (InterruptedException e) {
        throw new IOException("Got interrupted, cancelling");
    }

    // This will delete unnecessary fake parent directories
    finishedWrite(key);

    if (delSrc) {
        local.delete(src, false);
    }
}

From source file:org.apache.hadoop.fs.s3r.S3ROutputStream.java

License:Apache License

@Override
public synchronized void close() throws IOException {
    if (closed) {
        return;/*  w ww . j a  v a 2s  . co  m*/
    }

    backupStream.close();
    if (LOG.isDebugEnabled()) {
        LOG.debug("OutputStream for key '" + key + "' closed. Now beginning upload");
        LOG.debug("Minimum upload part size: " + partSize + " threshold " + partSizeThreshold);
    }

    try {
        final ObjectMetadata om = new ObjectMetadata();
        if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) {
            om.setServerSideEncryption(serverSideEncryptionAlgorithm);
        }
        PutObjectRequest putObjectRequest = new PutObjectRequest(bucket, key, backupFile);
        putObjectRequest.setCannedAcl(cannedACL);
        putObjectRequest.setMetadata(om);

        Upload upload = transfers.upload(putObjectRequest);

        ProgressableProgressListener listener = new ProgressableProgressListener(upload, progress, statistics);
        upload.addProgressListener(listener);

        upload.waitForUploadResult();

        long delta = upload.getProgress().getBytesTransferred() - listener.getLastBytesTransferred();
        if (statistics != null && delta != 0) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("S3A write delta changed after finished: " + delta + " bytes");
            }
            statistics.incrementBytesWritten(delta);
        }

        // This will delete unnecessary fake parent directories
        fs.finishedWrite(key);
    } catch (InterruptedException e) {
        throw new IOException(e);
    } finally {
        if (!backupFile.delete()) {
            LOG.warn("Could not delete temporary s3a file: {}", backupFile);
        }
        super.close();
        closed = true;
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("OutputStream for key '" + key + "' upload complete");
    }
}

From source file:org.apache.jackrabbit.aws.ext.ds.S3Backend.java

License:Apache License

private void write(DataIdentifier identifier, File file, boolean asyncUpload, AsyncUploadCallback callback)
        throws DataStoreException {
    String key = getKeyName(identifier);
    ObjectMetadata objectMetaData = null;
    long start = System.currentTimeMillis();
    ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
    try {//  w  w  w . ja  v  a2 s .c  o  m
        Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
        // check if the same record already exists
        try {
            objectMetaData = s3service.getObjectMetadata(bucket, key);
        } catch (AmazonServiceException ase) {
            if (ase.getStatusCode() != 404) {
                throw ase;
            }
        }
        if (objectMetaData != null) {
            long l = objectMetaData.getContentLength();
            if (l != file.length()) {
                throw new DataStoreException(
                        "Collision: " + key + " new length: " + file.length() + " old length: " + l);
            }
            LOG.debug("[{}]'s exists, lastmodified = [{}]", key, objectMetaData.getLastModified().getTime());
            CopyObjectRequest copReq = new CopyObjectRequest(bucket, key, bucket, key);
            copReq.setNewObjectMetadata(objectMetaData);
            s3service.copyObject(copReq);
            LOG.debug("lastModified of [{}] updated successfully.", identifier);
            if (callback != null) {
                callback.onSuccess(new AsyncUploadResult(identifier, file));
            }
        }

        if (objectMetaData == null) {
            try {
                // start multipart parallel upload using amazon sdk
                Upload up = tmx.upload(new PutObjectRequest(bucket, key, file));
                // wait for upload to finish
                if (asyncUpload) {
                    up.addProgressListener(new S3UploadProgressListener(up, identifier, file, callback));
                    LOG.debug("added upload progress listener to identifier [{}]", identifier);
                } else {
                    up.waitForUploadResult();
                    LOG.debug("synchronous upload to identifier [{}] completed.", identifier);
                    if (callback != null) {
                        callback.onSuccess(new AsyncUploadResult(identifier, file));
                    }
                }
            } catch (Exception e2) {
                if (!asyncUpload) {
                    callback.onAbort(new AsyncUploadResult(identifier, file));
                }
                throw new DataStoreException("Could not upload " + key, e2);
            }
        }
    } finally {
        if (contextClassLoader != null) {
            Thread.currentThread().setContextClassLoader(contextClassLoader);
        }
    }
    LOG.debug("write of [{}], length=[{}], in async mode [{}], in [{}]ms",
            new Object[] { identifier, file.length(), asyncUpload, (System.currentTimeMillis() - start) });
}

From source file:org.apache.jackrabbit.oak.blob.cloud.aws.s3.S3Backend.java

License:Apache License

private void write(DataIdentifier identifier, File file, boolean asyncUpload, AsyncUploadCallback callback)
        throws DataStoreException {
    String key = getKeyName(identifier);
    ObjectMetadata objectMetaData = null;
    long start = System.currentTimeMillis();
    ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
    try {//from w w w.  j  av a2s . c o m
        Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
        // check if the same record already exists
        try {
            objectMetaData = s3service.getObjectMetadata(bucket, key);
        } catch (AmazonServiceException ase) {
            if (!(ase.getStatusCode() == 404 || ase.getStatusCode() == 403)) {
                throw ase;
            }
        }
        if (objectMetaData != null) {
            long l = objectMetaData.getContentLength();
            if (l != file.length()) {
                throw new DataStoreException(
                        "Collision: " + key + " new length: " + file.length() + " old length: " + l);
            }
            LOG.debug("[{}]'s exists, lastmodified = [{}]", key, objectMetaData.getLastModified().getTime());
            CopyObjectRequest copReq = new CopyObjectRequest(bucket, key, bucket, key);
            copReq.setNewObjectMetadata(objectMetaData);
            Copy copy = tmx.copy(s3ReqDecorator.decorate(copReq));
            try {
                copy.waitForCopyResult();
                LOG.debug("lastModified of [{}] updated successfully.", identifier);
                if (callback != null) {
                    callback.onSuccess(new AsyncUploadResult(identifier, file));
                }
            } catch (Exception e2) {
                AsyncUploadResult asyncUpRes = new AsyncUploadResult(identifier, file);
                asyncUpRes.setException(e2);
                if (callback != null) {
                    callback.onAbort(asyncUpRes);
                }
                throw new DataStoreException("Could not upload " + key, e2);
            }
        }

        if (objectMetaData == null) {
            try {
                // start multipart parallel upload using amazon sdk
                Upload up = tmx.upload(s3ReqDecorator.decorate(new PutObjectRequest(bucket, key, file)));
                // wait for upload to finish
                if (asyncUpload) {
                    up.addProgressListener(new S3UploadProgressListener(up, identifier, file, callback));
                    LOG.debug("added upload progress listener to identifier [{}]", identifier);
                } else {
                    up.waitForUploadResult();
                    LOG.debug("synchronous upload to identifier [{}] completed.", identifier);
                    if (callback != null) {
                        callback.onSuccess(new AsyncUploadResult(identifier, file));
                    }
                }
            } catch (Exception e2) {
                AsyncUploadResult asyncUpRes = new AsyncUploadResult(identifier, file);
                asyncUpRes.setException(e2);
                if (callback != null) {
                    callback.onAbort(asyncUpRes);
                }
                throw new DataStoreException("Could not upload " + key, e2);
            }
        }
    } finally {
        if (contextClassLoader != null) {
            Thread.currentThread().setContextClassLoader(contextClassLoader);
        }
    }
    LOG.debug("write of [{}], length=[{}], in async mode [{}], in [{}]ms",
            new Object[] { identifier, file.length(), asyncUpload, (System.currentTimeMillis() - start) });
}

From source file:org.apache.nifi.processors.aws.s3.AbstractS3IT.java

License:Apache License

protected void putTestFile(String key, File file) throws AmazonS3Exception {
    PutObjectRequest putRequest = new PutObjectRequest(BUCKET_NAME, key, file);

    client.putObject(putRequest);//www. j a  v  a  2s.c o m
}

From source file:org.apache.oodt.cas.filemgr.datatransfer.S3DataTransferer.java

License:Apache License

@Override
public void transferProduct(Product product) throws DataTransferException, IOException {
    for (Reference ref : product.getProductReferences()) {
        String origRef = stripProtocol(ref.getOrigReference(), false);
        String dataStoreRef = stripProtocol(ref.getDataStoreReference(), true);
        try {//ww  w.  ja  v a2  s.c om
            PutObjectRequest request = new PutObjectRequest(bucketName, dataStoreRef, new File(origRef));
            if (encrypt) {
                ObjectMetadata requestMetadata = new ObjectMetadata();
                requestMetadata.setServerSideEncryption(AES_256_SERVER_SIDE_ENCRYPTION);
                request.setMetadata(requestMetadata);
            }
            s3Client.putObject(request);
        } catch (AmazonClientException e) {
            throw new DataTransferException(
                    String.format("Failed to upload product reference %s to S3 at %s", origRef, dataStoreRef),
                    e);
        }
    }
}

From source file:org.apache.zeppelin.notebook.repo.OldS3NotebookRepo.java

License:Apache License

@Override
public void save(Note note, AuthenticationInfo subject) throws IOException {
    String json = note.toJson();//from  w  w w.  j  a va  2s  .  c  om
    String key = user + "/" + "notebook" + "/" + note.getId() + "/" + "note.json";

    File file = File.createTempFile("note", "json");
    try {
        Writer writer = new OutputStreamWriter(new FileOutputStream(file));
        writer.write(json);
        writer.close();

        PutObjectRequest putRequest = new PutObjectRequest(bucketName, key, file);

        if (useServerSideEncryption) {
            // Request server-side encryption.
            ObjectMetadata objectMetadata = new ObjectMetadata();
            objectMetadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
            putRequest.setMetadata(objectMetadata);
        }

        s3client.putObject(putRequest);
    } catch (AmazonClientException ace) {
        throw new IOException("Unable to store note in S3: " + ace, ace);
    } finally {
        FileUtils.deleteQuietly(file);
    }
}

From source file:org.apache.zeppelin.notebook.repo.S3NotebookRepo.java

License:Apache License

@Override
public void save(Note note, AuthenticationInfo subject) throws IOException {
    String json = note.toJson();/*from  w  w  w.j a v a2s  .  c  o  m*/
    String key = rootFolder + "/" + buildNoteFileName(note);
    File file = File.createTempFile("note", "zpln");
    try {
        Writer writer = new OutputStreamWriter(new FileOutputStream(file));
        writer.write(json);
        writer.close();
        PutObjectRequest putRequest = new PutObjectRequest(bucketName, key, file);
        if (useServerSideEncryption) {
            // Request server-side encryption.
            ObjectMetadata objectMetadata = new ObjectMetadata();
            objectMetadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
            putRequest.setMetadata(objectMetadata);
        }
        s3client.putObject(putRequest);
    } catch (AmazonClientException ace) {
        throw new IOException("Unable to store note in S3: " + ace, ace);
    } finally {
        FileUtils.deleteQuietly(file);
    }
}

From source file:org.applicationMigrator.migrationclient.FileTransferClient.java

License:Apache License

private void uploadFile(AWSCredentials awsCredentials, String sourcePathString, String destinationPathString,
        boolean forceUpload) throws FileNotFoundException {
    // TODO Think about one file being used by many apps (e.g HP1.pdf read
    // through Adobe reader and OpenOffice)
    AmazonS3 s3client = new AmazonS3Client(awsCredentials);
    boolean fileIsPresentOnServer = checkIfFileIsPresentOnServer(s3client, BUCKET_NAME, destinationPathString);
    if (fileIsPresentOnServer && !forceUpload)
        return;//from   ww w  . j a  va 2 s  . co  m
    try {
        File file = new File(sourcePathString);
        if (!file.exists())
            throw new FileNotFoundException();
        s3client.putObject(new PutObjectRequest(BUCKET_NAME, destinationPathString, file));
    } catch (AmazonServiceException ase) {
        throw ase;
    } catch (AmazonClientException ace) {
        throw ace;
    }
    // TODO:verify completion of upload operation

}