List of usage examples for com.amazonaws.event ProgressEvent getEventType
public ProgressEventType getEventType()
From source file:com.github.rholder.esthree.progress.PrintingProgressListener.java
License:Apache License
@Override public void progressChanged(ProgressEvent progressEvent) { ProgressEventType type = progressEvent.getEventType(); if (type.equals(TRANSFER_COMPLETED_EVENT) || type.equals(TRANSFER_STARTED_EVENT)) { out.println();//from w w w . ja v a 2 s . co m } if (type.isByteCountEvent()) { long timeLeft = getTimeLeft(); if (lastTimeLeft < 1 && timeLeft > 0) { // prime this value with a sane starting point lastTimeLeft = timeLeft; } // use an exponential moving average to smooth the estimate lastTimeLeft += 0.90 * (timeLeft - lastTimeLeft); out.print(String.format("\r%1$s %2$s / %3$s %4$s ", generate(saturatedCast(round(completed + (progress.getPercentTransferred() * multiplier)))), humanReadableByteCount(progress.getBytesTransferred(), true), humanReadableByteCount(progress.getTotalBytesToTransfer(), true), fromSeconds(lastTimeLeft))); out.flush(); } }
From source file:com.vmware.photon.controller.model.adapters.awsadapter.AWSCostStatsService.java
License:Open Source License
private void downloadAndParse(AWSCostStatsCreationContext statsData, String awsBucketname, int year, int month, AWSCostStatsCreationStages next) throws IOException { // Creating a working directory for downloanding and processing the bill final Path workingDirPath = Paths.get(System.getProperty(TEMP_DIR_LOCATION), UUID.randomUUID().toString()); Files.createDirectories(workingDirPath); String accountId = statsData.computeDesc.customProperties.getOrDefault(AWSConstants.AWS_ACCOUNT_ID_KEY, null);//from www.j a v a 2 s. c o m AWSCsvBillParser parser = new AWSCsvBillParser(); final String csvBillZipFileName = parser.getCsvBillFileName(month, year, accountId, true); Path csvBillZipFilePath = Paths.get(workingDirPath.toString(), csvBillZipFileName); GetObjectRequest getObjectRequest = new GetObjectRequest(awsBucketname, csvBillZipFileName); Download download = statsData.s3Client.download(getObjectRequest, csvBillZipFilePath.toFile()); final StatelessService service = this; download.addProgressListener(new ProgressListener() { @Override public void progressChanged(ProgressEvent progressEvent) { try { ProgressEventType eventType = progressEvent.getEventType(); if (ProgressEventType.TRANSFER_COMPLETED_EVENT.equals(eventType)) { LocalDate monthDate = new LocalDate(year, month, 1); statsData.accountDetailsMap = parser.parseDetailedCsvBill(statsData.ignorableInvoiceCharge, csvBillZipFilePath, monthDate); deleteTempFiles(); OperationContext.restoreOperationContext(statsData.opContext); statsData.stage = next; handleCostStatsCreationRequest(statsData); } else if (ProgressEventType.TRANSFER_FAILED_EVENT.equals(eventType)) { deleteTempFiles(); throw new IOException("Download of AWS CSV Bill '" + csvBillZipFileName + "' failed."); } } catch (IOException e) { logSevere(e); AdapterUtils.sendFailurePatchToProvisioningTask(service, statsData.statsRequest.taskReference, e); } } private void deleteTempFiles() { try { Files.deleteIfExists(csvBillZipFilePath); Files.deleteIfExists(workingDirPath); } catch (IOException e) { // Ignore IO exception while cleaning files. } } }); }
From source file:io.dockstore.common.FileProvisioning.java
License:Apache License
public void provisionOutputFile(FileInfo file, String cwlOutputPath) { File sourceFile = new File(cwlOutputPath); long inputSize = sourceFile.length(); if (file.getUrl().startsWith("s3://")) { AmazonS3 s3Client = FileProvisioning.getAmazonS3Client(config); String trimmedPath = file.getUrl().replace("s3://", ""); List<String> splitPathList = Lists.newArrayList(trimmedPath.split("/")); String bucketName = splitPathList.remove(0); PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, Joiner.on("/").join(splitPathList), sourceFile);//from w ww. ja va2 s. co m putObjectRequest.setGeneralProgressListener(new ProgressListener() { ProgressPrinter printer = new ProgressPrinter(); long runningTotal = 0; @Override public void progressChanged(ProgressEvent progressEvent) { if (progressEvent.getEventType() == ProgressEventType.REQUEST_BYTE_TRANSFER_EVENT) { runningTotal += progressEvent.getBytesTransferred(); } printer.handleProgress(runningTotal, inputSize); } }); try { s3Client.putObject(putObjectRequest); } finally { System.out.println(); } } else { try { FileSystemManager fsManager; // trigger a copy from the URL to a local file path that's a UUID to avoid collision fsManager = VFS.getManager(); // check for a local file path FileObject dest = fsManager.resolveFile(file.getUrl()); FileObject src = fsManager.resolveFile(sourceFile.getAbsolutePath()); copyFromInputStreamToOutputStream(src.getContent().getInputStream(), inputSize, dest.getContent().getOutputStream()); } catch (IOException e) { throw new RuntimeException("Could not provision output files", e); } } }
From source file:org.anhonesteffort.p25.wav.WaveFileS3Sender.java
License:Open Source License
@Override public void progressChanged(ProgressEvent event) { switch (event.getEventType()) { case TRANSFER_COMPLETED_EVENT: ImbeefMetrics.getInstance().wavPutSuccess(); log.info(proto.toString(channelId) + " wave file successfully put to s3"); chunks.forEach(CheckpointingAudioChunk::checkpoint); break;/*ww w . j a va 2 s . c o m*/ case TRANSFER_CANCELED_EVENT: case TRANSFER_FAILED_EVENT: case TRANSFER_PART_FAILED_EVENT: ImbeefMetrics.getInstance().wavPutFailure(); log.error(proto.toString(channelId) + " error sending wave file " + event.toString()); break; } }
From source file:org.apache.hadoop.fs.s3a.ProgressableProgressListener.java
License:Apache License
@Override public void progressChanged(ProgressEvent progressEvent) { if (progress != null) { progress.progress();/*ww w. jav a 2s . c o m*/ } // There are 3 http ops here, but this should be close enough for now ProgressEventType pet = progressEvent.getEventType(); if (pet == TRANSFER_PART_STARTED_EVENT || pet == TRANSFER_COMPLETED_EVENT) { fs.incrementWriteOperations(); } long transferred = upload.getProgress().getBytesTransferred(); long delta = transferred - lastBytesTransferred; fs.incrementPutProgressStatistics(key, delta); lastBytesTransferred = transferred; }
From source file:org.springframework.integration.aws.outbound.S3MessageHandler.java
License:Apache License
private Transfer upload(Message<?> requestMessage) { Object payload = requestMessage.getPayload(); String bucketName = obtainBucket(requestMessage); String key = null;/*from w w w . j a v a 2 s .c o m*/ if (this.keyExpression != null) { key = this.keyExpression.getValue(this.evaluationContext, requestMessage, String.class); } if (payload instanceof File && ((File) payload).isDirectory()) { File fileToUpload = (File) payload; if (key == null) { key = fileToUpload.getName(); } return this.transferManager.uploadDirectory(bucketName, key, fileToUpload, true, new MessageHeadersObjectMetadataProvider(requestMessage.getHeaders())); } else { ObjectMetadata metadata = new ObjectMetadata(); if (this.uploadMetadataProvider != null) { this.uploadMetadataProvider.populateMetadata(metadata, requestMessage); } InputStream inputStream; if (payload instanceof InputStream) { inputStream = (InputStream) payload; } else if (payload instanceof File) { File fileToUpload = (File) payload; if (key == null) { key = fileToUpload.getName(); } try { inputStream = new FileInputStream(fileToUpload); if (metadata.getContentLength() == 0) { metadata.setContentLength(fileToUpload.length()); } if (metadata.getContentType() == null) { metadata.setContentType(Mimetypes.getInstance().getMimetype(fileToUpload)); } } catch (FileNotFoundException e) { throw new AmazonClientException(e); } } else if (payload instanceof byte[]) { inputStream = new ByteArrayInputStream((byte[]) payload); } else { throw new IllegalArgumentException("Unsupported payload type: [" + payload.getClass() + "]. The only supported payloads for the upload request are " + "java.io.File, java.io.InputStream, byte[] and PutObjectRequest."); } Assert.state(key != null, "The 'keyExpression' must not be null for non-File payloads and can't evaluate to null. " + "Root object is: " + requestMessage); if (key == null) { if (this.keyExpression != null) { throw new IllegalStateException( "The 'keyExpression' [" + this.keyExpression.getExpressionString() + "] must not evaluate to null. Root object is: " + requestMessage); } else { throw new IllegalStateException("Specify a 'keyExpression' for non-java.io.File payloads"); } } if (metadata.getContentMD5() == null) { String contentMd5 = null; try { contentMd5 = Md5Utils.md5AsBase64(StreamUtils.copyToByteArray(inputStream)); if (inputStream.markSupported()) { inputStream.reset(); } metadata.setContentMD5(contentMd5); } catch (IOException e) { throw new MessageHandlingException(requestMessage, e); } } PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, key, inputStream, metadata); S3ProgressListener progressListener = this.s3ProgressListener; if (this.objectAclExpression != null) { Object acl = this.objectAclExpression.getValue(this.evaluationContext, requestMessage); Assert.state(acl instanceof AccessControlList || acl instanceof CannedAccessControlList, "The 'objectAclExpression' [" + this.objectAclExpression.getExpressionString() + "] must evaluate to com.amazonaws.services.s3.model.AccessControlList " + "or must evaluate to com.amazonaws.services.s3.model.CannedAccessControlList. " + "Gotten: [" + acl + "]"); SetObjectAclRequest aclRequest; if (acl instanceof AccessControlList) { aclRequest = new SetObjectAclRequest(bucketName, key, (AccessControlList) acl); } else { aclRequest = new SetObjectAclRequest(bucketName, key, (CannedAccessControlList) acl); } final SetObjectAclRequest theAclRequest = aclRequest; progressListener = new S3ProgressListener() { @Override public void onPersistableTransfer(PersistableTransfer persistableTransfer) { } @Override public void progressChanged(ProgressEvent progressEvent) { if (ProgressEventType.TRANSFER_COMPLETED_EVENT.equals(progressEvent.getEventType())) { S3MessageHandler.this.transferManager.getAmazonS3Client().setObjectAcl(theAclRequest); } } }; if (this.s3ProgressListener != null) { progressListener = new S3ProgressListenerChain(this.s3ProgressListener, progressListener); } } if (progressListener != null) { return this.transferManager.upload(putObjectRequest, progressListener); } else { return this.transferManager.upload(putObjectRequest); } } }