List of usage examples for com.amazonaws.services.s3.model S3ObjectSummary getLastModified
public Date getLastModified()
From source file:core.connector.s3.sync.S3Connector.java
License:GNU General Public License
@Override public List<FileInfo> listDirectory(String path) throws ConnectorException { try {/*from www . j av a2s. co m*/ ObjectListing bucketListing = s3 .listObjects(new ListObjectsRequest().withBucketName(info.getBucketName()).withPrefix(path)); List<FileInfo> listing = new ArrayList<FileInfo>(bucketListing.getObjectSummaries().size()); boolean finished = false; while (!finished) { for (S3ObjectSummary s3s : bucketListing.getObjectSummaries()) { String key = s3s.getKey(); long size = s3s.getSize(); Date date = s3s.getLastModified(); listing.add(new FileInfo(key, key.substring(path.length() + 1), size, date, s3s.getETag())); } if (bucketListing.isTruncated()) bucketListing = s3.listNextBatchOfObjects(bucketListing); else finished = true; } return listing; } catch (Exception e) { throw new ConnectorException(e); } }
From source file:eu.stratosphere.nephele.fs.s3.S3FileSystem.java
License:Apache License
private S3FileStatus[] listBucketContent(final Path f, final S3BucketObjectPair bop) throws IOException { ObjectListing listing = null;/*from ww w . j a v a2 s . c o m*/ final List<S3FileStatus> resultList = new ArrayList<S3FileStatus>(); final int depth = (bop.hasObject() ? getDepth(bop.getObject()) + 1 : 0); while (true) { if (listing == null) { if (bop.hasObject()) { listing = this.s3Client.listObjects(bop.getBucket(), bop.getObject()); } else { listing = this.s3Client.listObjects(bop.getBucket()); } } else { listing = this.s3Client.listNextBatchOfObjects(listing); } final List<S3ObjectSummary> list = listing.getObjectSummaries(); final Iterator<S3ObjectSummary> it = list.iterator(); while (it.hasNext()) { final S3ObjectSummary os = it.next(); String key = os.getKey(); final int childDepth = getDepth(os.getKey()); if (childDepth != depth) { continue; } // Remove the prefix if (bop.hasObject()) { if (key.startsWith(bop.getObject())) { key = key.substring(bop.getObject().length()); } // This has been the prefix itself if (key.isEmpty()) { continue; } } final long modificationDate = dateToLong(os.getLastModified()); S3FileStatus fileStatus; if (objectRepresentsDirectory(os)) { fileStatus = new S3FileStatus(extendPath(f, key), 0, true, modificationDate, 0L); } else { fileStatus = new S3FileStatus(extendPath(f, key), os.getSize(), false, modificationDate, 0L); } resultList.add(fileStatus); } if (!listing.isTruncated()) { break; } } /* * System.out.println("---- RETURN CONTENT ----"); * for (final FileStatus entry : resultList) { * System.out.println(entry.getPath()); * } * System.out.println("------------------------"); */ return resultList.toArray(new S3FileStatus[0]); }
From source file:gov.noaa.pfel.coastwatch.util.FileVisitorDNLS.java
License:Open Source License
/** * This is a convenience method for using this class. * <p>This works with Amazon AWS S3 bucket URLs. Internal /'s in the keys will be * treated as folder separators. If there aren't any /'s, all the keys will * be in the root directory./* w w w .j a v a2 s . c o m*/ * * @param tDir The starting directory, with \\ or /, with or without trailing slash. * The resulting directoryPA will contain dirs with matching slashes and trailing slash. * @param tPathRegex a regex to constrain which subdirs to include. * This is ignored if recursive is false. * null or "" is treated as .* (i.e., match everything). * @param tDirectoriesToo if true, each directory name will get its own rows * in the results. * @return a table with columns with DIRECTORY, NAME, LASTMODIFIED, and SIZE columns. * LASTMODIFIED and SIZE are LongArrays -- For directories when the values * are otherwise unknown, the value will be Long.MAX_VALUE. * If directoriesToo=true, the original dir won't be included and any * directory's file NAME will be "". * @throws IOException if trouble */ public static Table oneStep(String tDir, String tFileNameRegex, boolean tRecursive, String tPathRegex, boolean tDirectoriesToo) throws IOException { long time = System.currentTimeMillis(); //is tDir an http URL? if (tDir.matches(FileVisitorDNLS.HTTP_REGEX)) { //Is it an S3 bucket with "files"? //If testing a "dir", url should have a trailing slash. Matcher matcher = AWS_S3_PATTERN.matcher(File2.addSlash(tDir)); //force trailing slash if (matcher.matches()) { //http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGET.html //If files have file-system-like names, e.g., // http://bucketname.s3.amazonaws.com/dir1/dir2/fileName.ext) // http://nasanex.s3.amazonaws.com/NEX-DCP30/BCSD/rcp26/mon/atmos/tasmin/r1i1p1/v1.0/CONUS/tasmin_amon_BCSD_rcp26_r1i1p1_CONUS_NorESM1-M_209601-209912.nc // you still can't request just dir2 info because they aren't directories. // They are just object keys with internal slashes. //So specify prefix in request. Table table = makeEmptyTable(); StringArray directoryPA = (StringArray) table.getColumn(DIRECTORY); StringArray namePA = (StringArray) table.getColumn(NAME); LongArray lastModifiedPA = (LongArray) table.getColumn(LASTMODIFIED); LongArray sizePA = (LongArray) table.getColumn(SIZE); String bucketName = matcher.group(1); String prefix = matcher.group(2); String baseURL = tDir.substring(0, matcher.start(2)); AmazonS3 s3client = new AmazonS3Client(new ProfileCredentialsProvider()); try { if (verbose) String2.log("FileVisitorDNLS.oneStep getting info from AWS S3 at" + "\nURL=" + tDir); //"\nbucket=" + bucketName + " prefix=" + prefix); //I wanted to generate lastMod for dir based on lastMod of files //but it would be inconsistent for different requests (recursive, fileNameRegex). //so just a set of dir names. HashSet<String> dirHashSet = new HashSet(); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName) .withPrefix(prefix); ObjectListing objectListing; do { objectListing = s3client.listObjects(listObjectsRequest); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { String keyFullName = objectSummary.getKey(); String keyDir = File2.getDirectory(baseURL + keyFullName); String keyName = File2.getNameAndExtension(keyFullName); if (debugMode) String2.log( "keyFullName=" + keyFullName + "\nkeyDir=" + keyDir + "\n tDir=" + tDir); if (keyDir.startsWith(tDir) && //it should (tRecursive || keyDir.length() == tDir.length())) { //store this dir if (tDirectoriesToo) { //S3 only returns object keys. I must infer/collect directories. //Store this dir and parents back to tDir. String choppedKeyDir = keyDir; while (choppedKeyDir.length() >= tDir.length()) { if (!dirHashSet.add(choppedKeyDir)) break; //hash set already had this, so will already have parents //chop off last subdirectory choppedKeyDir = File2.getDirectory( choppedKeyDir.substring(0, choppedKeyDir.length() - 1)); //remove trailing / } } //store this file's information //Sometimes directories appear as files are named "" with size=0. //I don't store those as files. if (debugMode) String2.log("keyName=" + keyFullName + "\n tFileNameRegex=" + tFileNameRegex + " matches=" + keyName.matches(tFileNameRegex)); if (keyName.length() > 0 && keyName.matches(tFileNameRegex)) { directoryPA.add(keyDir); namePA.add(keyName); lastModifiedPA.add(objectSummary.getLastModified().getTime()); //epoch millis sizePA.add(objectSummary.getSize()); //long } } } listObjectsRequest.setMarker(objectListing.getNextMarker()); } while (objectListing.isTruncated()); //add directories to the table if (tDirectoriesToo) { Iterator<String> it = dirHashSet.iterator(); while (it.hasNext()) { directoryPA.add(it.next()); namePA.add(""); lastModifiedPA.add(Long.MAX_VALUE); sizePA.add(Long.MAX_VALUE); } } table.leftToRightSortIgnoreCase(2); return table; } catch (AmazonServiceException ase) { throw new IOException("AmazonServiceException: " + ase.getErrorType() + " ERROR, HTTP Code=" + ase.getStatusCode() + ": " + ase.getMessage(), ase); } catch (AmazonClientException ace) { throw new IOException(ace.getMessage(), ace); } } //HYRAX before THREDDS //http://dods.jpl.nasa.gov/opendap/ocean_wind/ccmp/L3.5a/data/flk/1988/ matcher = HYRAX_PATTERN.matcher(tDir); if (matcher.matches()) { try { if (verbose) String2.log("FileVisitorDNLS.oneStep getting info from Hyrax at" + "\nURL=" + tDir); Table table = makeEmptyTable(); StringArray directoryPA = (StringArray) table.getColumn(DIRECTORY); StringArray namePA = (StringArray) table.getColumn(NAME); LongArray lastModifiedPA = (LongArray) table.getColumn(LASTMODIFIED); LongArray sizePA = (LongArray) table.getColumn(SIZE); DoubleArray lastModDA = new DoubleArray(); addToHyraxUrlList(tDir, tFileNameRegex, tRecursive, tPathRegex, tDirectoriesToo, namePA, lastModDA, sizePA); lastModifiedPA.append(lastModDA); int n = namePA.size(); for (int i = 0; i < n; i++) { String fn = namePA.get(i); directoryPA.add(File2.getDirectory(fn)); namePA.set(i, File2.getNameAndExtension(fn)); } table.leftToRightSortIgnoreCase(2); return table; } catch (Throwable t) { throw new IOException(t.getMessage(), t); } } //THREDDS matcher = THREDDS_PATTERN.matcher(tDir); if (matcher.matches()) { try { if (verbose) String2.log("FileVisitorDNLS.oneStep getting info from THREDDS at" + "\nURL=" + tDir); Table table = makeEmptyTable(); StringArray directoryPA = (StringArray) table.getColumn(DIRECTORY); StringArray namePA = (StringArray) table.getColumn(NAME); LongArray lastModifiedPA = (LongArray) table.getColumn(LASTMODIFIED); LongArray sizePA = (LongArray) table.getColumn(SIZE); DoubleArray lastModDA = new DoubleArray(); addToThreddsUrlList(tDir, tFileNameRegex, tRecursive, tPathRegex, tDirectoriesToo, namePA, lastModDA, sizePA); lastModifiedPA.append(lastModDA); int n = namePA.size(); for (int i = 0; i < n; i++) { String fn = namePA.get(i); directoryPA.add(File2.getDirectory(fn)); namePA.set(i, File2.getNameAndExtension(fn)); } table.leftToRightSortIgnoreCase(2); return table; } catch (Throwable t) { throw new IOException(t.getMessage(), t); } } //default: Apache-style WAF try { if (verbose) String2.log("FileVisitorDNLS.oneStep getting info from Apache-style WAF at" + "\nURL=" + tDir); Table table = makeEmptyTable(); StringArray directorySA = (StringArray) table.getColumn(DIRECTORY); StringArray nameSA = (StringArray) table.getColumn(NAME); LongArray lastModLA = (LongArray) table.getColumn(LASTMODIFIED); LongArray sizeLA = (LongArray) table.getColumn(SIZE); addToWAFUrlList(tDir, tFileNameRegex, tRecursive, tPathRegex, tDirectoriesToo, directorySA, nameSA, lastModLA, sizeLA); table.leftToRightSortIgnoreCase(2); return table; } catch (Throwable t) { throw new IOException(t.getMessage(), t); } } //local files //follow symbolic links: https://docs.oracle.com/javase/7/docs/api/java/nio/file/FileVisitor.html //But this doesn't follow Windows symbolic link .lnk's: // http://bugs.java.com/bugdatabase/view_bug.do?bug_id=4237760 FileVisitorDNLS fv = new FileVisitorDNLS(tDir, tFileNameRegex, tRecursive, tPathRegex, tDirectoriesToo); EnumSet<FileVisitOption> opts = EnumSet.of(FileVisitOption.FOLLOW_LINKS); Files.walkFileTree(FileSystems.getDefault().getPath(tDir), opts, //follow symbolic links Integer.MAX_VALUE, //maxDepth fv); fv.table.leftToRightSortIgnoreCase(2); if (verbose) String2.log("FileVisitorDNLS.oneStep(local) finished successfully. n=" + fv.directoryPA.size() + " time=" + (System.currentTimeMillis() - time) + "ms"); return fv.table; }
From source file:imperial.modaclouds.monitoring.datacollectors.monitors.DetailedCostMonitor.java
License:BSD License
@Override public void run() { String accessKeyId = null;/*from w w w . j a v a 2 s . co m*/ String secretKey = null; ObjectListing objects = null; AmazonS3Client s3Client = null; String key = null; long startTime = 0; while (!dcmt.isInterrupted()) { if (System.currentTimeMillis() - startTime > 10000) { cost_nonspot = new HashMap<String, Double>(); cost_spot = new HashMap<String, Double>(); for (String metric : getProvidedMetrics()) { try { VM resource = new VM(Config.getInstance().getVmType(), Config.getInstance().getVmId()); if (dcAgent.shouldMonitor(resource, metric)) { Map<String, String> parameters = dcAgent.getParameters(resource, metric); accessKeyId = parameters.get("accessKey"); secretKey = parameters.get("secretKey"); bucketName = parameters.get("bucketName"); filePath = parameters.get("filePath"); period = Integer.valueOf(parameters.get("samplingTime")) * 1000; } } catch (NumberFormatException e) { e.printStackTrace(); } catch (ConfigurationException e) { e.printStackTrace(); } } startTime = System.currentTimeMillis(); AWSCredentials credentials = new BasicAWSCredentials(accessKeyId, secretKey); s3Client = new AmazonS3Client(credentials); objects = s3Client.listObjects(bucketName); key = "aws-billing-detailed-line-items-with-resources-and-tags-"; SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM"); String date = sdf.format(new Date()); key = key + date + ".csv.zip"; } String fileName = null; do { for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) { System.out.println(objectSummary.getKey() + "\t" + objectSummary.getSize() + "\t" + StringUtils.fromDate(objectSummary.getLastModified())); if (objectSummary.getKey().contains(key)) { fileName = objectSummary.getKey(); s3Client.getObject(new GetObjectRequest(bucketName, fileName), new File(filePath + fileName)); break; } } objects = s3Client.listNextBatchOfObjects(objects); } while (objects.isTruncated()); try { ZipFile zipFile = new ZipFile(filePath + fileName); zipFile.extractAll(filePath); } catch (ZipException e) { e.printStackTrace(); } String csvFileName = fileName.replace(".zip", ""); AnalyseFile(filePath + csvFileName); try { Thread.sleep(period); } catch (InterruptedException e) { Thread.currentThread().interrupt(); break; } } }
From source file:io.druid.storage.s3.S3TimestampVersionedDataFinder.java
License:Apache License
/** * Gets the key with the most recently modified timestamp. * `pattern` is evaluated against the entire key AFTER the path given in `uri`. * The substring `pattern` is matched against will have a leading `/` removed. * For example `s3://some_bucket/some_prefix/some_key` with a URI of `s3://some_bucket/some_prefix` will match against `some_key`. * `s3://some_bucket/some_prefixsome_key` with a URI of `s3://some_bucket/some_prefix` will match against `some_key` * `s3://some_bucket/some_prefix//some_key` with a URI of `s3://some_bucket/some_prefix` will match against `/some_key` * * @param uri The URI of in the form of `s3://some_bucket/some_key` * @param pattern The pattern matcher to determine if a *key* is of interest, or `null` to match everything. * * @return A URI to the most recently modified object which matched the pattern. *///from w ww.j av a 2 s. c o m @Override public URI getLatestVersion(final URI uri, final @Nullable Pattern pattern) { try { return RetryUtils.retry(() -> { final S3Coords coords = new S3Coords(checkURI(uri)); long mostRecent = Long.MIN_VALUE; URI latest = null; final Iterator<S3ObjectSummary> objectSummaryIterator = S3Utils.objectSummaryIterator(s3Client, coords.bucket, coords.path, MAX_LISTING_KEYS); while (objectSummaryIterator.hasNext()) { final S3ObjectSummary objectSummary = objectSummaryIterator.next(); String keyString = objectSummary.getKey().substring(coords.path.length()); if (keyString.startsWith("/")) { keyString = keyString.substring(1); } if (pattern != null && !pattern.matcher(keyString).matches()) { continue; } final long latestModified = objectSummary.getLastModified().getTime(); if (latestModified >= mostRecent) { mostRecent = latestModified; latest = new URI(StringUtils.format("s3://%s/%s", objectSummary.getBucketName(), objectSummary.getKey())); } } return latest; }, shouldRetryPredicate(), DEFAULT_RETRY_COUNT); } catch (Exception e) { throw Throwables.propagate(e); } }
From source file:it.openutils.mgnlaws.magnolia.datastore.S3DataStore.java
License:Open Source License
/** * {@inheritDoc}//from ww w. jav a2 s. c o m */ public int deleteAllOlderThan(long min) throws DataStoreException { int deleted = 0; for (Iterator<S3ObjectSummary> i = new S3Iterator(amazonS3, bucket, PREFIX); i.hasNext();) { S3ObjectSummary rec = (S3ObjectSummary) i.next(); if (rec.getLastModified().getTime() < min) { deleteRecord(rec.getKey()); if (useCache) { cache.remove(getDataIdentifier(rec.getKey())); } deleted++; } } return deleted; }
From source file:jenkins.plugins.itemstorage.s3.Downloads.java
License:Open Source License
public void startDownload(TransferManager manager, File base, String pathPrefix, S3ObjectSummary summary) throws AmazonServiceException, IOException { // calculate target file name File targetFile = FileUtils.getFile(base, summary.getKey().substring(pathPrefix.length() + 1)); // if target file exists, only download it if newer if (targetFile.lastModified() < summary.getLastModified().getTime()) { // ensure directory above file exists FileUtils.forceMkdir(targetFile.getParentFile()); // Start the download Download download = manager.download(summary.getBucketName(), summary.getKey(), targetFile); // Keep for later startedDownloads.add(new Memo(download, targetFile, summary.getLastModified().getTime())); }/*from ww w .j av a 2 s.co m*/ }
From source file:jenkins.plugins.itemstorage.s3.S3UploadAllCallable.java
License:Open Source License
/** * Upload from slave//from w w w.j a v a 2 s . c o m */ @Override public Integer invoke(final TransferManager transferManager, File base, VirtualChannel channel) throws IOException, InterruptedException { if (!base.exists()) return 0; final AtomicInteger count = new AtomicInteger(0); final Uploads uploads = new Uploads(); final Map<String, S3ObjectSummary> summaries = lookupExistingCacheEntries( transferManager.getAmazonS3Client()); // Find files to upload that match scan scanner.scan(base, new FileVisitor() { @Override public void visit(File f, String relativePath) throws IOException { if (f.isFile()) { String key = pathPrefix + "/" + relativePath; S3ObjectSummary summary = summaries.get(key); if (summary == null || f.lastModified() > summary.getLastModified().getTime()) { final ObjectMetadata metadata = buildMetadata(f); uploads.startUploading(transferManager, f, IOUtils.toBufferedInputStream(FileUtils.openInputStream(f)), new Destination(bucketName, key), metadata); if (uploads.count() > 20) { waitForUploads(count, uploads); } } } } }); // Wait for each file to complete before returning waitForUploads(count, uploads); return uploads.count(); }
From source file:n3phele.storage.s3.CloudStorageImpl.java
License:Open Source License
public List<FileNode> getFileList(Repository repo, String prefix, int max) { List<FileNode> result = new ArrayList<FileNode>(); Credential credential = repo.getCredential().decrypt(); AmazonS3Client s3 = new AmazonS3Client( new BasicAWSCredentials(credential.getAccount(), credential.getSecret())); s3.setEndpoint(repo.getTarget().toString()); Policy p = null;// w ww . j a va2 s .c o m try { BucketPolicy bp = s3.getBucketPolicy(repo.getRoot()); log.info("Policy text " + bp.getPolicyText()); if (bp != null && bp.getPolicyText() != null) { p = PolicyHelper.parse(bp.getPolicyText()); log.info("Policy object is " + (p == null ? null : p.toJson())); } } catch (Exception e) { log.log(Level.WARNING, "Policy not supported", e); } try { ObjectListing s3Objects = s3 .listObjects(new ListObjectsRequest(repo.getRoot(), prefix, null, "/", max)); if (s3Objects.getCommonPrefixes() != null) { for (String dirName : s3Objects.getCommonPrefixes()) { String name = dirName; if (dirName.endsWith("/")) { name = dirName.substring(0, dirName.length() - 1); } boolean isPublic = isPublicFolder(p, repo.getRoot(), name); name = name.substring(name.lastIndexOf("/") + 1); FileNode folder = FileNode.newFolder(name, prefix, repo, isPublic); log.info("Folder:" + folder); result.add(folder); } } if (s3Objects.getObjectSummaries() != null) { for (S3ObjectSummary fileSummary : s3Objects.getObjectSummaries()) { String key = fileSummary.getKey(); if (key != null && !key.equals(prefix)) { String name = key.substring(key.lastIndexOf("/") + 1); UriBuilder builder = UriBuilder.fromUri(repo.getTarget()).path(repo.getRoot()); if (prefix != null && !prefix.isEmpty()) { builder = builder.path(prefix); } FileNode file = FileNode.newFile(name, prefix, repo, fileSummary.getLastModified(), fileSummary.getSize(), builder.path(name).toString()); log.info("File:" + file); result.add(file); } } } } catch (AmazonServiceException e) { log.log(Level.WARNING, "Service Error processing " + repo, e); } catch (AmazonClientException e) { log.log(Level.SEVERE, "Client Error processing " + repo, e); } return result; }
From source file:net.solarnetwork.node.backup.s3.SdkS3Client.java
License:Open Source License
@Override public Set<S3ObjectReference> listObjects(String prefix) throws IOException { AmazonS3 client = getClient();// w ww . ja v a 2 s . c o m Set<S3ObjectReference> result = new LinkedHashSet<>(100); try { final ListObjectsV2Request req = new ListObjectsV2Request(); req.setBucketName(bucketName); req.setMaxKeys(maximumKeysPerRequest); req.setPrefix(prefix); ListObjectsV2Result listResult; do { listResult = client.listObjectsV2(req); for (S3ObjectSummary objectSummary : listResult.getObjectSummaries()) { result.add(new S3ObjectReference(objectSummary.getKey(), objectSummary.getSize(), objectSummary.getLastModified())); } req.setContinuationToken(listResult.getNextContinuationToken()); } while (listResult.isTruncated() == true); } catch (AmazonServiceException e) { log.warn("AWS error: {}; HTTP code {}; AWS code {}; type {}; request ID {}", e.getMessage(), e.getStatusCode(), e.getErrorCode(), e.getErrorType(), e.getRequestId()); throw new RemoteServiceException("Error listing S3 objects at " + prefix, e); } catch (AmazonClientException e) { log.debug("Error communicating with AWS: {}", e.getMessage()); throw new IOException("Error communicating with AWS", e); } return result; }