List of usage examples for com.amazonaws.services.s3.model S3ObjectSummary getKey
public String getKey()
From source file:com.haskins.cloudtrailviewer.dialog.S3FileChooser.java
License:Open Source License
private void reloadContents() { loadingLabel.setVisible(true);// w w w.j av a2s .c om this.s3ListModel.clear(); String bucketName = currentAccount.getBucket(); ListObjectsRequest listObjectsRequest = new ListObjectsRequest(); listObjectsRequest.setBucketName(bucketName); listObjectsRequest.setPrefix(prefix); listObjectsRequest.setDelimiter("/"); AWSCredentials credentials = new BasicAWSCredentials(currentAccount.getKey(), currentAccount.getSecret()); AmazonS3 s3Client = new AmazonS3Client(credentials); try { ObjectListing objectListing = s3Client.listObjects(listObjectsRequest); // Add .. if not at root if (prefix.trim().length() != 0) { S3ListModel model = new S3ListModel(MOVE_BACK, MOVE_BACK, S3ListModel.FILE_BACK); this.s3ListModel.addElement(model); } // these are directories List<String> directories = objectListing.getCommonPrefixes(); for (String directory : directories) { String dir = stripPrefix(directory); int lastSlash = dir.lastIndexOf("/"); String strippeDir = dir.substring(0, lastSlash); String alias = dir; if (isAccountNumber(strippeDir)) { if (aliasMap.containsKey(strippeDir)) { alias = aliasMap.get(strippeDir); } } S3ListModel model = new S3ListModel(dir, alias, S3ListModel.FILE_DIR); this.s3ListModel.addElement(model); } // these are files List<S3ObjectSummary> objectSummaries = objectListing.getObjectSummaries(); for (final S3ObjectSummary objectSummary : objectSummaries) { String file = stripPrefix(objectSummary.getKey()); S3ListModel model = new S3ListModel(file, file, S3ListModel.FILE_DOC); this.s3ListModel.addElement(model); } loadingLabel.setVisible(false); } catch (Exception e) { e.printStackTrace(); } }
From source file:com.ibm.stocator.fs.cos.COSAPIClient.java
License:Apache License
private FileStatus createFileStatus(S3ObjectSummary objSummary, String hostName, Path path) throws IllegalArgumentException, IOException { String objKey = objSummary.getKey(); String newMergedPath = getMergedPath(hostName, path, objKey); return createFileStatus(objSummary.getSize(), objKey, objSummary.getLastModified(), new Path(newMergedPath)); }
From source file:com.ibm.stocator.fs.cos.COSAPIClient.java
License:Apache License
/** * {@inheritDoc}/*from w w w. j av a2 s. c om*/ * * Prefix based * Return everything that starts with the prefix * Fill listing * Return all objects, even zero size * If fileStatus is null means the path is part of some name, neither object * or pseudo directory. Was called by Globber * * @param hostName hostName * @param path path * @param fullListing Return all objects, even zero size * @param prefixBased Return everything that starts with the prefix * @return list * @throws IOException if error */ /* public FileStatus[] list(String hostName, Path path, boolean fullListing, boolean prefixBased) throws IOException { String key = pathToKey(hostName, path); ArrayList<FileStatus> tmpResult = new ArrayList<FileStatus>(); ListObjectsRequest request = new ListObjectsRequest().withBucketName(mBucket).withPrefix(key); String curObj; if (path.toString().equals(mBucket)) { curObj = ""; } else if (path.toString().startsWith(mBucket + "/")) { curObj = path.toString().substring(mBucket.length() + 1); } else if (path.toString().startsWith(hostName)) { curObj = path.toString().substring(hostName.length()); } else { curObj = path.toString(); } ObjectListing objectList = mClient.listObjects(request); List<S3ObjectSummary> objectSummaries = objectList.getObjectSummaries(); if (objectSummaries.size() == 0) { FileStatus[] emptyRes = {}; LOG.debug("List for bucket {} is empty", mBucket); return emptyRes; } boolean objectScanContinue = true; S3ObjectSummary prevObj = null; while (objectScanContinue) { for (S3ObjectSummary obj : objectSummaries) { if (prevObj == null) { prevObj = obj; continue; } String objKey = obj.getKey(); String unifiedObjectName = extractUnifiedObjectName(objKey); if (!prefixBased && !curObj.equals("") && !path.toString().endsWith("/") && !unifiedObjectName.equals(curObj) && !unifiedObjectName.startsWith(curObj + "/")) { LOG.trace("{} does not match {}. Skipped", unifiedObjectName, curObj); continue; } if (isSparkOrigin(unifiedObjectName) && !fullListing) { LOG.trace("{} created by Spark", unifiedObjectName); if (!isJobSuccessful(unifiedObjectName)) { LOG.trace("{} created by failed Spark job. Skipped", unifiedObjectName); if (fModeAutomaticDelete) { delete(hostName, new Path(objKey), true); } continue; } else { // if we here - data created by spark and job completed // successfully // however there be might parts of failed tasks that // were not aborted // we need to make sure there are no failed attempts if (nameWithoutTaskID(objKey).equals(nameWithoutTaskID(prevObj.getKey()))) { // found failed that was not aborted. LOG.trace("Colisiion found between {} and {}", prevObj.getKey(), objKey); if (prevObj.getSize() < obj.getSize()) { LOG.trace("New candidate is {}. Removed {}", obj.getKey(), prevObj.getKey()); prevObj = obj; } continue; } } } if (prevObj.getSize() > 0 || fullListing) { FileStatus fs = getFileStatusObjSummaryBased(prevObj, hostName, path); tmpResult.add(fs); } prevObj = obj; } boolean isTruncated = objectList.isTruncated(); if (isTruncated) { objectList = mClient.listNextBatchOfObjects(objectList); objectSummaries = objectList.getObjectSummaries(); } else { objectScanContinue = false; } } if (prevObj != null && (prevObj.getSize() > 0 || fullListing)) { FileStatus fs = getFileStatusObjSummaryBased(prevObj, hostName, path); tmpResult.add(fs); } if (LOG.isTraceEnabled()) { LOG.trace("COS List to return length {}", tmpResult.size()); for (FileStatus fs: tmpResult) { LOG.trace("{}", fs.getPath()); } } return tmpResult.toArray(new FileStatus[tmpResult.size()]); } */ @Override public FileStatus[] list(String hostName, Path path, boolean fullListing, boolean prefixBased, Boolean isDirectory, boolean flatListing, PathFilter filter) throws FileNotFoundException, IOException { LOG.debug("Native direct list status for {}", path); ArrayList<FileStatus> tmpResult = new ArrayList<FileStatus>(); String key = pathToKey(hostName, path); if (isDirectory != null && isDirectory.booleanValue() && !key.endsWith("/")) { key = key + "/"; LOG.debug("listNativeDirect modify key to {}", key); } Map<String, FileStatus> emptyObjects = new HashMap<String, FileStatus>(); ListObjectsRequest request = new ListObjectsRequest(); request.setBucketName(mBucket); request.setMaxKeys(5000); request.setPrefix(key); if (!flatListing) { request.setDelimiter("/"); } ObjectListing objectList = mClient.listObjects(request); List<S3ObjectSummary> objectSummaries = objectList.getObjectSummaries(); List<String> commonPrefixes = objectList.getCommonPrefixes(); boolean objectScanContinue = true; S3ObjectSummary prevObj = null; // start FTA logic boolean stocatorOrigin = isSparkOrigin(key, path.toString()); if (stocatorOrigin) { LOG.debug("Stocator origin is true for {}", key); if (!isJobSuccessful(key)) { LOG.debug("{} created by failed Spark job. Skipped", key); if (fModeAutomaticDelete) { delete(hostName, new Path(key), true); } return new FileStatus[0]; } } while (objectScanContinue) { for (S3ObjectSummary obj : objectSummaries) { if (prevObj == null) { prevObj = obj; continue; } String objKey = obj.getKey(); String unifiedObjectName = extractUnifiedObjectName(objKey); LOG.debug("list candidate {}, unified name {}", objKey, unifiedObjectName); if (stocatorOrigin && !fullListing) { LOG.trace("{} created by Spark", unifiedObjectName); // if we here - data created by spark and job completed // successfully // however there be might parts of failed tasks that // were not aborted // we need to make sure there are no failed attempts if (nameWithoutTaskID(objKey).equals(nameWithoutTaskID(prevObj.getKey()))) { // found failed that was not aborted. LOG.trace("Colisiion found between {} and {}", prevObj.getKey(), objKey); if (prevObj.getSize() < obj.getSize()) { LOG.trace("New candidate is {}. Removed {}", obj.getKey(), prevObj.getKey()); prevObj = obj; } continue; } } FileStatus fs = createFileStatus(prevObj, hostName, path); if (fs.getLen() > 0 || fullListing) { LOG.debug("Native direct list. Adding {} size {}", fs.getPath(), fs.getLen()); if (filter == null) { tmpResult.add(fs); } else if (filter != null && filter.accept(fs.getPath())) { tmpResult.add(fs); } else { LOG.trace("{} rejected by path filter during list. Filter {}", fs.getPath(), filter); } } else { emptyObjects.put(fs.getPath().toString(), fs); } prevObj = obj; } boolean isTruncated = objectList.isTruncated(); if (isTruncated) { objectList = mClient.listNextBatchOfObjects(objectList); objectSummaries = objectList.getObjectSummaries(); } else { objectScanContinue = false; } } if (prevObj != null) { FileStatus fs = createFileStatus(prevObj, hostName, path); LOG.debug("Adding the last object from the list {}", fs.getPath()); if (fs.getLen() > 0 || fullListing) { LOG.debug("Native direct list. Adding {} size {}", fs.getPath(), fs.getLen()); if (filter == null) { memoryCache.putFileStatus(fs.getPath().toString(), fs); tmpResult.add(fs); } else if (filter != null && filter.accept(fs.getPath())) { memoryCache.putFileStatus(fs.getPath().toString(), fs); tmpResult.add(fs); } else { LOG.trace("{} rejected by path filter during list. Filter {}", fs.getPath(), filter); } } else if (!fs.getPath().getName().equals(HADOOP_SUCCESS)) { emptyObjects.put(fs.getPath().toString(), fs); } } // get common prefixes for (String comPrefix : commonPrefixes) { LOG.debug("Common prefix is {}", comPrefix); if (emptyObjects.containsKey(keyToQualifiedPath(hostName, comPrefix).toString()) || emptyObjects.isEmpty()) { FileStatus status = new COSFileStatus(true, false, keyToQualifiedPath(hostName, comPrefix)); LOG.debug("Match between common prefix and empty object {}. Adding to result", comPrefix); if (filter == null) { memoryCache.putFileStatus(status.getPath().toString(), status); tmpResult.add(status); } else if (filter != null && filter.accept(status.getPath())) { memoryCache.putFileStatus(status.getPath().toString(), status); tmpResult.add(status); } else { LOG.trace("Common prefix {} rejected by path filter during list. Filter {}", status.getPath(), filter); } } } return tmpResult.toArray(new FileStatus[tmpResult.size()]); }
From source file:com.ibm.stocator.fs.cos.COSUtils.java
License:Apache License
/** * Create a files status instance from a listing. * @param keyPath path to entry/*from w w w .j av a2s .c o m*/ * @param summary summary from AWS * @param blockSize block size to declare * @return a status entry */ public static COSFileStatus createFileStatus(Path keyPath, S3ObjectSummary summary, long blockSize) { long size = summary.getSize(); return createFileStatus(keyPath, objectRepresentsDirectory(summary.getKey(), size), size, summary.getLastModified(), blockSize); }
From source file:com.ibm.stocator.fs.cos.COSUtils.java
License:Apache License
public static String stringify(S3ObjectSummary summary) { StringBuilder builder = new StringBuilder(summary.getKey().length() + 100); builder.append(summary.getKey()).append(' '); builder.append("size=").append(summary.getSize()); return builder.toString(); }
From source file:com.ikanow.infinit.e.harvest.extraction.document.file.AwsInfiniteFile.java
License:Open Source License
@Override public InfiniteFile[] listFiles(Date optionalFilterDate, int maxDocs) { InfiniteFile[] fileList = null;//from w ww . ja v a 2 s . co m ObjectListing list = null; _overwriteTime = 0L; ListObjectsRequest listRequest = new ListObjectsRequest().withBucketName(_awsBucketName); if (null != _awsObjectName) { listRequest.withPrefix(_awsObjectName); } listRequest.withDelimiter("/"); list = ((AmazonS3Client) _awsClient).listObjects(listRequest); fileList = new InfiniteFile[list.getObjectSummaries().size() + list.getCommonPrefixes().size()]; //TESTED (3.2) int nAdded = 0; // Get the sub-directories for (String subDir : list.getCommonPrefixes()) { // Create directories: fileList[nAdded] = new AwsInfiniteFile(_awsBucketName, subDir, null, _awsClient); nAdded++; } //TESTED (3b.3) // Get the files: for (S3ObjectSummary s3Obj : list.getObjectSummaries()) { if (!s3Obj.getKey().endsWith("/")) { fileList[nAdded] = new AwsInfiniteFile(s3Obj.getBucketName(), s3Obj.getKey(), s3Obj.getLastModified(), _awsClient); long fileTime = fileList[nAdded].getDate(); if (fileTime > _overwriteTime) { _overwriteTime = fileTime; } //TESTED (3.2) nAdded++; } } return fileList; }
From source file:com.images3.data.impl.ImageContentAccessImplS3.java
License:Apache License
private void deleteAllImageContent(AmazonS3 client, AmazonS3Bucket bucket, ObjectListing objList) { boolean isFinished = false; while (!isFinished) { DeleteObjectsRequest request = new DeleteObjectsRequest(bucket.getName()); List<KeyVersion> keys = new ArrayList<KeyVersion>(objList.getMaxKeys()); for (S3ObjectSummary sum : objList.getObjectSummaries()) { keys.add(new KeyVersion(sum.getKey())); }//from ww w.j a va2 s . c om request.setKeys(keys); client.deleteObjects(request); if (objList.isTruncated()) { objList = client.listNextBatchOfObjects(objList); } else { isFinished = true; } } }
From source file:com.intuit.s3encrypt.S3Encrypt.java
License:Open Source License
public static void main(String[] args) throws IOException, NoSuchAlgorithmException, InvalidKeySpecException { // create Options object Options options = new Options(); options.addOption(create_bucket);/*from w ww . j ava 2 s . c o m*/ options.addOption(create_key); options.addOption(delete_bucket); options.addOption(get); options.addOption(help); options.addOption(inspect); options.addOption(keyfile); options.addOption(list_buckets); options.addOption(list_objects); options.addOption(put); options.addOption(remove); options.addOption(rotate); options.addOption(rotateall); options.addOption(rotateKey); // CommandLineParser parser = new GnuParser(); // Changed from above GnuParser to below PosixParser because I found code which allows for multiple arguments PosixParser parser = new PosixParser(); CommandLine cmd; try { cmd = parser.parse(options, args); Logger.getRootLogger().setLevel(Level.OFF); if (cmd.hasOption("help")) { HelpFormatter help = new HelpFormatter(); System.out.println(); help.printHelp("S3Encrypt", options); System.out.println(); System.exit(1); } else if (cmd.hasOption("create_key")) { keyname = cmd.getOptionValue("keyfile"); createKeyFile(keyname); key = new File(keyname); } else { if (cmd.hasOption("keyfile")) { keyname = cmd.getOptionValue("keyfile"); } key = new File(keyname); } if (!(key.exists())) { System.out.println("Key does not exist or not provided"); System.exit(1); } // AmazonS3 s3 = new AmazonS3Client(new ClasspathPropertiesFileCredentialsProvider()); ClasspathPropertiesFileCredentialsProvider credentials = new ClasspathPropertiesFileCredentialsProvider( ".s3encrypt"); EncryptionMaterials encryptionMaterials = new EncryptionMaterials(getKeyFile(keyname)); AmazonS3EncryptionClient s3 = new AmazonS3EncryptionClient(credentials.getCredentials(), encryptionMaterials); // Region usWest2 = Region.getRegion(Regions.US_WEST_2); // s3.setRegion(usWest2); if (cmd.hasOption("create_bucket")) { String bucket = cmd.getOptionValue("create_bucket"); System.out.println("Creating bucket " + bucket + "\n"); s3.createBucket(bucket); } else if (cmd.hasOption("delete_bucket")) { String bucket = cmd.getOptionValue("delete_bucket"); System.out.println("Deleting bucket " + bucket + "\n"); s3.deleteBucket(bucket); } else if (cmd.hasOption("get")) { String[] searchArgs = cmd.getOptionValues("get"); String bucket = searchArgs[0]; String filename = searchArgs[1]; getS3Object(cmd, s3, bucket, filename); } else if (cmd.hasOption("inspect")) { String[] searchArgs = cmd.getOptionValues("inspect"); String bucket = searchArgs[0]; String filename = searchArgs[1]; String keyname = "encryption_key"; String metadata = inspectS3Object(cmd, s3, bucket, filename, keyname); System.out.println(metadata); } else if (cmd.hasOption("list_buckets")) { System.out.println("Listing buckets"); for (Bucket bucket : s3.listBuckets()) { System.out.println(bucket.getName()); } System.out.println(); } else if (cmd.hasOption("list_objects")) { String bucket = cmd.getOptionValue("list_objects"); System.out.println("Listing objects"); ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucket)); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { System.out.println(objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")"); } System.out.println(); } else if (cmd.hasOption("put")) { String[] searchArgs = cmd.getOptionValues("put"); String bucket = searchArgs[0]; String filename = searchArgs[1]; String metadataKeyname = "encryption_key"; String key = keyname; putS3Object(cmd, s3, bucket, filename, metadataKeyname, key); } else if (cmd.hasOption("remove")) { String[] searchArgs = cmd.getOptionValues("remove"); String bucket = searchArgs[0]; String filename = searchArgs[1]; System.out.println("Removing object in S3 from BUCKET = " + bucket + " FILENAME = " + filename); s3.deleteObject(new DeleteObjectRequest(bucket, filename)); System.out.println(); } else if (cmd.hasOption("rotate")) { String[] searchArgs = cmd.getOptionValues("rotate"); String bucket = searchArgs[0]; String filename = searchArgs[1]; String key1 = cmd.getOptionValue("keyfile"); String key2 = cmd.getOptionValue("rotateKey"); String metadataKeyname = "encryption_key"; System.out.println("Supposed to get object from here OPTION VALUE = " + bucket + " FILENAME = " + filename + " KEY1 = " + key1 + " KEY2 = " + key2); EncryptionMaterials rotateEncryptionMaterials = new EncryptionMaterials(getKeyFile(key2)); AmazonS3EncryptionClient rotateS3 = new AmazonS3EncryptionClient(credentials.getCredentials(), rotateEncryptionMaterials); getS3Object(cmd, s3, bucket, filename); putS3Object(cmd, rotateS3, bucket, filename, metadataKeyname, key2); } else if (cmd.hasOption("rotateall")) { String[] searchArgs = cmd.getOptionValues("rotateall"); String bucket = searchArgs[0]; String key1 = searchArgs[1]; String key2 = searchArgs[2]; System.out.println("Supposed to rotateall here for BUCKET NAME = " + bucket + " KEY1 = " + key1 + " KEY2 = " + key2); } else { System.out.println("Something went wrong... "); System.exit(1); } } catch (ParseException e) { e.printStackTrace(); } catch (AmazonServiceException ase) { System.out.println("Caught an AmazonServiceException, which " + "means your request made it " + "to Amazon S3, but was rejected with an error response" + " for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { System.out.println("Caught an AmazonClientException, which " + "means the client encountered " + "an internal error while trying to " + "communicate with S3, " + "such as not being able to access the network."); System.out.println("Error Message: " + ace.getMessage()); } }
From source file:com.ipcglobal.fredimportaws.TsvsToRedshift.java
License:Apache License
/** * Empty bucket contents./*from w w w .ja v a 2 s . c om*/ * * @throws Exception the exception */ private void emptyBucketContents() throws Exception { try { while (true) { ObjectListing objectListing = s3Client.listObjects(new ListObjectsRequest() .withBucketName(awsBucketName).withPrefix(awsBucketTsvPrefix + "/").withDelimiter("/")); if (objectListing.getObjectSummaries().size() == 0) break; List<KeyVersion> keyVersions = new ArrayList<KeyVersion>(); for (S3ObjectSummary s3ObjectSummary : objectListing.getObjectSummaries()) keyVersions.add(new KeyVersion(s3ObjectSummary.getKey())); if (keyVersions.size() > 0) { DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(awsBucketName); deleteObjectsRequest.setKeys(keyVersions); s3Client.deleteObjects(deleteObjectsRequest); } } } catch (AmazonServiceException ase) { log.error(ase); throw ase; } catch (AmazonClientException ace) { log.error(ace); throw ace; } }
From source file:com.jeet.s3.AmazonS3ClientWrapper.java
License:Open Source License
public List<String> getListOfFiles(String basepath) { try {//from w w w . j av a 2s . c o m List<String> keys = new ArrayList(); ListObjectsRequest listObjectsRequest = new ListObjectsRequest(); listObjectsRequest.setBucketName(Constants.BUCKET_NAME); // listObjectsRequest.setDelimiter(basepath); listObjectsRequest.setPrefix(basepath); ObjectListing objectListing; do { objectListing = s3Client.listObjects(listObjectsRequest); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { // System.out.println(" - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")"); keys.add(objectSummary.getKey()); } listObjectsRequest.setMarker(objectListing.getNextMarker()); } while (objectListing.isTruncated()); return keys; } catch (Exception e) { e.printStackTrace(); } return new ArrayList<>(); }