List of usage examples for com.amazonaws.services.s3.model PutObjectRequest PutObjectRequest
public PutObjectRequest(String bucketName, String key, InputStream input, ObjectMetadata metadata)
From source file:org.pieShare.pieDrive.adapter.s3.S3Adapter.java
@Override public synchronized void upload(PieDriveFile file, InputStream stream) throws AdaptorException { try {//from www. j a v a 2s. co m ObjectMetadata meta = new ObjectMetadata(); meta.setContentLength(file.getSize()); PutObjectRequest req = new PutObjectRequest(bucketName, file.getUuid(), stream, meta); //req.getRequestClientOptions().setReadLimit(64); s3Auth.getClient().putObject(req); //Thread.sleep(2000); PieLogger.trace(S3Adapter.class, "{} uploaded", file.getUuid()); } catch (AmazonServiceException ase) { throw new AdaptorException(ase); } catch (AmazonClientException ace) { throw new AdaptorException(ace); } catch (Exception e) { } }
From source file:org.rdswitchboard.harvesters.pmh.Harvester.java
License:Open Source License
public void downloadRecords(SetStatus set) throws HarvesterException, UnsupportedEncodingException, IOException, InterruptedException, XPathExpressionException, SAXException, ParserConfigurationException { // Generate the URL of request String url = null;/*from w w w .j a v a2 s. c o m*/ ; if (set.hasToken()) { try { url = repoUrl + String.format(URL_LIST_RECORDS_RESUMPTION_TOKEN, URLEncoder.encode(set.getToken(), "UTF-8")); } catch (UnsupportedEncodingException e) { e.printStackTrace(); } } if (null == url) { if (!set.hasName()) url = repoUrl + String.format(URL_LIST_DEFAULT_RECORDS, metadataPrefix); else url = repoUrl + String.format(URL_LIST_RECORDS, URLEncoder.encode(set.getName(), "UTF-8"), metadataPrefix); } System.out.println("Downloading records: " + url); String xml = null; // Get XML document URLConnection conn = new URL(url).openConnection(); if (connectionTimeout > 0) conn.setConnectTimeout(connectionTimeout); if (readTimeout > 0) conn.setReadTimeout(readTimeout); try (InputStream is = conn.getInputStream()) { if (null != is) xml = IOUtils.toString(is, StandardCharsets.UTF_8.name()); } // Check if xml has been returned and check what it had a valid root element if (null == xml) throw new HarvesterException("The XML document is empty"); Document doc = dbf.newDocumentBuilder().parse(new InputSource(new StringReader(xml))); // Extract root node Node root = (Node) XPATH_OAI_PMH.evaluate(doc, XPathConstants.NODE); if (null == root) throw new HarvesterException("The document is not an OAI:PMH file"); // Check for error node Node error = (Node) XPATH_ERROR.evaluate(root, XPathConstants.NODE); if (null != error && error instanceof Element) { String code = ((Element) error).getAttribute("code"); String message = ((Element) error).getTextContent(); if (ERR_NO_RECORDS_MATCH.equals(code)) { System.out.println("Error: The set is empty"); set.setFiles(0); set.resetToken(); return; } else throw new HarvesterException(code, message); } Node nodeToken = (Node) XPATH_RECORDS_RESUMPTION_TOKEN.evaluate(root, XPathConstants.NODE); if (null != nodeToken && nodeToken instanceof Element) { String tokenString = ((Element) nodeToken).getTextContent(); if (null != tokenString && !tokenString.isEmpty()) set.setToken(tokenString); else set.resetToken(); set.setCursor(((Element) nodeToken).getAttribute("cursor")); set.setSize(((Element) nodeToken).getAttribute("completeListSize")); set.dumpToken(System.out); } else set.resetToken(); String filePath = repoPrefix + "/" + metadataPrefix + "/" + harvestDate + "/" + set.getNameSafe() + "/" + set.getFiles() + ".xml"; if (StringUtils.isNullOrEmpty(bucketName)) { FileUtils.writeStringToFile(new File(folderName, filePath), xml); } else { byte[] bytes = xml.getBytes(StandardCharsets.UTF_8); ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentEncoding(StandardCharsets.UTF_8.name()); metadata.setContentType("text/xml"); metadata.setContentLength(bytes.length); InputStream inputStream = new ByteArrayInputStream(bytes); PutObjectRequest request = new PutObjectRequest(bucketName, filePath, inputStream, metadata); s3client.putObject(request); } set.incFiles(); }
From source file:org.rdswitchboard.harvesters.pmh.Harvester.java
License:Open Source License
/** * Alternative function to organize the harvest process. The difference with another function * is in data storage. The harvest2 function will store files in the raw format as they come * from the server./*from w ww . j a v a2s . co m*/ * The harvesting method should never be mixed. The harvesting folder must be wiped out if * switching to this method, or function will fail. * @param prefix A metadata prefix * @throws Exception */ public boolean harvest() throws Exception { if (StringUtils.isNullOrEmpty(metadataPrefix)) throw new IllegalArgumentException("The OAI:PMH Metadata Prefix can not be empty"); System.out.println("Downloading set list"); boolean result = false; if (null == whiteList || whiteList.isEmpty()) { System.out.println( "There is no whitelist found. Proceeding with downloading the list of all available sets."); // download all sets in the repository Map<String, String> mapSets = listSets(); if (null == mapSets || mapSets.isEmpty()) { System.out.println("Processing default set"); result = harvestSet(new SetStatus(null, "Default")); } else { result = false; for (Map.Entry<String, String> entry : mapSets.entrySet()) { SetStatus set = new SetStatus(entry.getKey().trim(), URLDecoder.decode(entry.getValue(), StandardCharsets.UTF_8.name())); // if black list exists and item is blacklisted, continue if (null != blackList && blackList.contains(set)) { set.setFiles(-2); saveSetStats(set); // set was ignored continue; } System.out.println("Processing set: " + URLDecoder.decode(entry.getValue(), StandardCharsets.UTF_8.name())); if (!harvestSet(set)) { System.err.println( "The harvesting job has been aborted due to an error. If you want harvesting to be continued, please set option 'fail.on.error' to 'false' in the configuration file"); result = false; break; } else result = true; } } } else { for (String item : whiteList) { if (!harvestSet(new SetStatus(item, item))) { System.err.println( "The harvesting job has been aborted due to an error. If you want harvesting to be continued, please set option 'fail.on.error' to 'false' in the configuration file"); result = false; break; } else result = true; } } if (result) { String filePath = repoPrefix + "/" + metadataPrefix + "/latest.txt"; if (StringUtils.isNullOrEmpty(bucketName)) { FileUtils.writeStringToFile(new File(folderName, filePath), harvestDate); } else { byte[] bytes = harvestDate.getBytes(StandardCharsets.UTF_8); ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentEncoding(StandardCharsets.UTF_8.name()); metadata.setContentType("text/plain"); metadata.setContentLength(bytes.length); InputStream inputStream = new ByteArrayInputStream(bytes); PutObjectRequest request = new PutObjectRequest(bucketName, filePath, inputStream, metadata); s3client.putObject(request); } } return result; }
From source file:org.researchgraph.crossref.CrossRef.java
private void saveCacheFile(String file, String json) throws IOException { if (null != file && null != json && !json.isEmpty()) { if (null != cache) { FileUtils.write(new File(cache, file), json); } else if (null != s3Client) { byte[] bytes = json.getBytes(StandardCharsets.UTF_8); ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentEncoding(StandardCharsets.UTF_8.name()); metadata.setContentType("text/json"); metadata.setContentLength(bytes.length); InputStream inputStream = new ByteArrayInputStream(bytes); s3Client.putObject(new PutObjectRequest(s3Bucket, getS3Key(file), inputStream, metadata)); }//from w w w .j a v a2 s . c om } }
From source file:org.springfield.lou.servlet.LouServlet.java
License:Open Source License
private String handleFileUpload(HttpServletRequest request) { System.out.println("HANDLE FILE UPLOAD"); try {//from w ww .j a v a 2 s . c om String targetid = request.getParameter("targetid"); System.out.println("TARGETID UPLOAD=" + targetid); String screenid = request.getParameter("screenid"); String cfilename = request.getParameter("cfilename"); System.out.println("CFILENAME=" + cfilename); String cfilesize = request.getParameter("cfilesize"); System.out.println("CFILESIZE=" + cfilesize); Html5ApplicationInterface app = null; String url = request.getRequestURI(); int pos = url.indexOf("/domain/"); if (pos != -1) { String tappname = url.substring(pos); app = ApplicationManager.instance().getApplication(tappname); } Screen eventscreen = app.getScreen(screenid); if (eventscreen == null) return null; String method = eventscreen.getModel() .getProperty("/screen['upload']/target['" + targetid + "']/method"); System.out.println("METHOD=" + method); String destpath = eventscreen.getModel() .getProperty("/screen['upload']/target['" + targetid + "']/destpath"); System.out.println("DESTPATH=" + destpath + " T=" + targetid); if (destpath == null || destpath.equals("")) { setUploadError(eventscreen, targetid, "destpath not set"); return null; } String destname_prefix = eventscreen.getModel() .getProperty("/screen['upload']/target['" + targetid + "']/destname_prefix"); if (destname_prefix == null || destname_prefix.equals("")) { setUploadError(eventscreen, targetid, "destname_prefix not set"); return null; } String filetype = eventscreen.getModel() .getProperty("/screen['upload']/target['" + targetid + "']/filetype"); if (filetype == null || filetype.equals("")) { setUploadError(eventscreen, targetid, "filetype not set"); return null; } String fileext = eventscreen.getModel() .getProperty("/screen['upload']/target['" + targetid + "']/fileext"); if (fileext == null || fileext.equals("")) { setUploadError(eventscreen, targetid, "fileext not set"); return null; } String checkupload = eventscreen.getModel() .getProperty("/screen['upload']/target['" + targetid + "']/checkupload"); if (checkupload == null || checkupload.equals("")) { setUploadError(eventscreen, targetid, "checkupload not set"); return null; } String storagehost = eventscreen.getModel() .getProperty("/screen['upload']/target['" + targetid + "']/storagehost"); if (storagehost == null || storagehost.equals("")) { setUploadError(eventscreen, targetid, "storagehost not set"); return null; } String destname_type = eventscreen.getModel() .getProperty("/screen['upload']/target['" + targetid + "']/destname_type"); if (destname_type == null || destname_type.equals("")) { setUploadError(eventscreen, targetid, "destname_type not set"); return null; } String publicpath = eventscreen.getModel() .getProperty("/screen['upload']/target['" + targetid + "']/publicpath"); if (publicpath == null || publicpath.equals("")) { setUploadError(eventscreen, targetid, "publicpath not set"); return null; } // here we can check if its a valid upload based on filename and other specs and kill if needed, also map real extension fileext = getValidExtension(fileext, cfilename); if (fileext == null) return null; // kill the request its not a valid format if (method.equals("s3amazon")) { String bucketname = eventscreen.getModel() .getProperty("/screen['upload']/target['" + targetid + "']/bucketname"); if (bucketname == null || bucketname.equals("")) { setUploadError(eventscreen, targetid, "bucketname not set"); return null; } AmazonS3 s3Client = AmazonS3ClientBuilder.standard() .withCredentials(new EnvironmentVariableCredentialsProvider()).build(); String filename = "unknown"; int storageport = 22; if (destname_type.equals("epoch")) { filename = destpath + destname_prefix + "" + new Date().getTime(); } String publicurl = publicpath + bucketname + "/" + filename + "." + fileext; FsPropertySet ps = new FsPropertySet(); // we will use this to send status reports back ps.setProperty("action", "start"); ps.setProperty("progress", "0"); ps.setProperty("cfilename", cfilename); ps.setProperty("url", publicurl); eventscreen.getModel().setProperties("/screen/upload/" + targetid, ps); try { InputStream inst = request.getInputStream(); int read = 0; int readtotal = 0; int b; while ((b = inst.read()) != 44) { // skip the base64 tagline, not sure how todo this better } Base64InputStream b64i = new Base64InputStream(inst); //System.out.println("Uploading a new object to S3 from a stream "+bucketname+"/"+filename+"."+fileext); ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentType(filetype + "/" + fileext); PutObjectRequest or = new PutObjectRequest(bucketname, filename + "." + fileext, b64i, metadata); or.setGeneralProgressListener(new UploadProgressListener(eventscreen.getModel(), publicurl, cfilename, cfilesize, targetid)); s3Client.putObject(or); } catch (AmazonServiceException ase) { ase.printStackTrace(); } ps.setProperty("action", "done"); ps.setProperty("progress", "100"); ps.setProperty("cfilename", cfilename); ps.setProperty("url", publicurl); eventscreen.getModel().setProperties("/screen/upload/" + targetid, ps); return bucketname + "/" + filename + "." + fileext; } else if (method.equals("scp")) { String pemfile = eventscreen.getModel() .getProperty("/screen['upload']/target['" + targetid + "']/pemfile"); if (destpath == null || destpath.equals("")) { setUploadError(eventscreen, targetid, "destpath not set"); return null; } String storagename = eventscreen.getModel() .getProperty("/screen['upload']/target['" + targetid + "']/storagename"); if (storagename == null || storagehost.equals("")) { setUploadError(eventscreen, targetid, "storagename not set"); return null; } String filename = "unknown"; int storageport = 22; if (destname_type.equals("epoch")) { filename = destname_prefix + "" + new Date().getTime(); } String publicurl = publicpath + filename + "." + fileext; FsPropertySet ps = new FsPropertySet(); // we will use this to send status reports back ps.setProperty("action", "start"); ps.setProperty("progress", "0"); ps.setProperty("url", publicurl); eventscreen.getModel().setProperties("/screen/upload/" + targetid, ps); JSch jsch = new JSch(); jsch.addIdentity(pemfile); jsch.setConfig("StrictHostKeyChecking", "no"); Session session = jsch.getSession(storagename, storagehost, storageport); session.connect(); Channel channel = session.openChannel("sftp"); channel.connect(); ChannelSftp channelSftp = (ChannelSftp) channel; channelSftp.cd(destpath); InputStream inst = request.getInputStream(); int read = 0; int readtotal = 0; int b; while ((b = inst.read()) != 44) { // skip the base64 tagline, not sure how todo this better } Base64InputStream b64i = new Base64InputStream(inst); channelSftp.put(b64i, filename + "." + fileext); ps.setProperty("action", "done"); ps.setProperty("progress", "100"); ps.setProperty("url", publicurl); eventscreen.getModel().setProperties("/screen/upload/" + targetid, ps); return filename + "." + fileext; } } catch (Exception e) { e.printStackTrace(); } return null; }
From source file:org.springframework.build.aws.maven.PrivateS3Wagon.java
License:Apache License
private PutObjectRequest createDirectoryPutObjectRequest(String key) { ByteArrayInputStream inputStream = new ByteArrayInputStream(new byte[0]); ObjectMetadata objectMetadata = new ObjectMetadata(); objectMetadata.setContentLength(0);//from w ww .j a v a 2s . c om return new PutObjectRequest(this.bucketName, key, inputStream, objectMetadata); }
From source file:org.springframework.integration.aws.outbound.S3MessageHandler.java
License:Apache License
private Transfer upload(Message<?> requestMessage) { Object payload = requestMessage.getPayload(); String bucketName = obtainBucket(requestMessage); String key = null;// w ww . j a v a 2s .c o m if (this.keyExpression != null) { key = this.keyExpression.getValue(this.evaluationContext, requestMessage, String.class); } if (payload instanceof File && ((File) payload).isDirectory()) { File fileToUpload = (File) payload; if (key == null) { key = fileToUpload.getName(); } return this.transferManager.uploadDirectory(bucketName, key, fileToUpload, true, new MessageHeadersObjectMetadataProvider(requestMessage.getHeaders())); } else { ObjectMetadata metadata = new ObjectMetadata(); if (this.uploadMetadataProvider != null) { this.uploadMetadataProvider.populateMetadata(metadata, requestMessage); } InputStream inputStream; if (payload instanceof InputStream) { inputStream = (InputStream) payload; } else if (payload instanceof File) { File fileToUpload = (File) payload; if (key == null) { key = fileToUpload.getName(); } try { inputStream = new FileInputStream(fileToUpload); if (metadata.getContentLength() == 0) { metadata.setContentLength(fileToUpload.length()); } if (metadata.getContentType() == null) { metadata.setContentType(Mimetypes.getInstance().getMimetype(fileToUpload)); } } catch (FileNotFoundException e) { throw new AmazonClientException(e); } } else if (payload instanceof byte[]) { inputStream = new ByteArrayInputStream((byte[]) payload); } else { throw new IllegalArgumentException("Unsupported payload type: [" + payload.getClass() + "]. The only supported payloads for the upload request are " + "java.io.File, java.io.InputStream, byte[] and PutObjectRequest."); } Assert.state(key != null, "The 'keyExpression' must not be null for non-File payloads and can't evaluate to null. " + "Root object is: " + requestMessage); if (key == null) { if (this.keyExpression != null) { throw new IllegalStateException( "The 'keyExpression' [" + this.keyExpression.getExpressionString() + "] must not evaluate to null. Root object is: " + requestMessage); } else { throw new IllegalStateException("Specify a 'keyExpression' for non-java.io.File payloads"); } } if (metadata.getContentMD5() == null) { String contentMd5 = null; try { contentMd5 = Md5Utils.md5AsBase64(StreamUtils.copyToByteArray(inputStream)); if (inputStream.markSupported()) { inputStream.reset(); } metadata.setContentMD5(contentMd5); } catch (IOException e) { throw new MessageHandlingException(requestMessage, e); } } PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, key, inputStream, metadata); S3ProgressListener progressListener = this.s3ProgressListener; if (this.objectAclExpression != null) { Object acl = this.objectAclExpression.getValue(this.evaluationContext, requestMessage); Assert.state(acl instanceof AccessControlList || acl instanceof CannedAccessControlList, "The 'objectAclExpression' [" + this.objectAclExpression.getExpressionString() + "] must evaluate to com.amazonaws.services.s3.model.AccessControlList " + "or must evaluate to com.amazonaws.services.s3.model.CannedAccessControlList. " + "Gotten: [" + acl + "]"); SetObjectAclRequest aclRequest; if (acl instanceof AccessControlList) { aclRequest = new SetObjectAclRequest(bucketName, key, (AccessControlList) acl); } else { aclRequest = new SetObjectAclRequest(bucketName, key, (CannedAccessControlList) acl); } final SetObjectAclRequest theAclRequest = aclRequest; progressListener = new S3ProgressListener() { @Override public void onPersistableTransfer(PersistableTransfer persistableTransfer) { } @Override public void progressChanged(ProgressEvent progressEvent) { if (ProgressEventType.TRANSFER_COMPLETED_EVENT.equals(progressEvent.getEventType())) { S3MessageHandler.this.transferManager.getAmazonS3Client().setObjectAcl(theAclRequest); } } }; if (this.s3ProgressListener != null) { progressListener = new S3ProgressListenerChain(this.s3ProgressListener, progressListener); } } if (progressListener != null) { return this.transferManager.upload(putObjectRequest, progressListener); } else { return this.transferManager.upload(putObjectRequest); } } }
From source file:org.symphonyoss.vb.util.AwsS3Client.java
License:Apache License
public void putObject(String destBucket, String key, InputStream inputStream, ObjectMetadata metaData) { try {//from w w w. j a va 2s . c o m logger.info("Put object for s3://{}/{}", destBucket, key); byte[] bytes = IOUtils.toByteArray(inputStream); if (metaData == null) metaData = new ObjectMetadata(); metaData.setContentLength(bytes.length); ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes); s3Client.putObject(new PutObjectRequest(destBucket, key, byteArrayInputStream, metaData)); } catch (AmazonServiceException ase) { logger.error("Caught an AmazonServiceException, " + "which means your request made it " + "to Amazon S3, but was rejected with an error response " + "for some reason."); logger.error("Error Message: " + ase.getMessage()); logger.error("HTTP Status Code: " + ase.getStatusCode()); logger.error("AWS Error Code: " + ase.getErrorCode()); logger.error("Error Type: " + ase.getErrorType()); logger.error("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { logger.error("Caught an AmazonClientException, " + "which means the client encountered " + "an internal error while trying to communicate" + " with S3, " + "such as not being able to access the network."); logger.error("Error Message: " + ace.getMessage()); } catch (IOException e) { logger.error("Obtaining length", e); } }
From source file:org.zalando.stups.fullstop.s3.S3Writer.java
License:Apache License
public void putObjectToS3(String bucket, String fileName, String keyName, ObjectMetadata metadata, InputStream stream) {/*from w w w .jav a 2 s . c o m*/ AmazonS3 s3client = new AmazonS3Client(); try { logger.info("Uploading a new object to S3 from a file"); s3client.putObject( new PutObjectRequest(bucket, Paths.get(keyName, fileName).toString(), stream, metadata)); } catch (AmazonServiceException ase) { logger.error("Caught an AmazonServiceException, which " + "means your request made it " + "to Amazon S3, but was rejected with an error response" + " for some reason."); logger.error("Error Message: " + ase.getMessage()); logger.error("HTTP Status Code: " + ase.getStatusCode()); logger.error("AWS Error Code: " + ase.getErrorCode()); logger.error("Error Type: " + ase.getErrorType()); logger.error("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { logger.error("Caught an AmazonClientException, which " + "means the client encountered " + "an internal error while trying to " + "communicate with S3, " + "such as not being able to access the network."); logger.error("Error Message: " + ace.getMessage()); } }
From source file:oulib.aws.s3.S3Util.java
/** * Creates an AWS S3 folder//www. jav a2s . c om * * @param bucketName * @param folderName * @param client */ public static void createFolder(String bucketName, String folderName, AmazonS3 client) { try { ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentLength(0); InputStream emptyContent = new ByteArrayInputStream(new byte[0]); PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, folderName + "/", emptyContent, metadata); client.putObject(putObjectRequest); System.out .println("Sucessfully created the folder of " + folderName + " in the bucket of " + bucketName); } catch (Exception ex) { System.out.println("Failed to create the folder of " + folderName + " in the bucket of " + bucketName); // Logger.getLogger(AwsS3Processor.class.getName()).log(Level.SEVERE, null, ex); } }