List of usage examples for com.amazonaws.services.s3 AmazonS3Client AmazonS3Client
@SdkInternalApi AmazonS3Client(AmazonS3ClientParams s3ClientParams)
From source file:eu.crowdrec.flume.plugins.source.DirectIdomaarSource.java
License:Apache License
private BufferedReader createReaderFromUrl() throws IOException { if (url.startsWith("s3://")) { logger.info("Reading resource {} from Amazon S3.", url); AmazonS3 s3 = new AmazonS3Client(createAWSCreditentials()); AmazonS3URI amazonURI = new AmazonS3URI(url); GeneratePresignedUrlRequest request = new GeneratePresignedUrlRequest(amazonURI.getBucket(), amazonURI.getKey());/*from ww w . j av a 2 s . c o m*/ URL httpUrl = s3.generatePresignedUrl(request); InputStream inp = httpUrl.openStream(); return new BufferedReader(new InputStreamReader(inp)); } throw new RuntimeException("Can only handle s3:// scheme. Cannot use URL " + url); }
From source file:eu.stratosphere.nephele.fs.s3.S3FileSystem.java
License:Apache License
/** * {@inheritDoc}//from www.ja v a2s. c o m */ @Override public void initialize(URI name) throws IOException { this.host = name.getHost(); if (this.host == null) { LOG.debug("Provided URI does not provide a host to connect to, using configuration..."); this.host = GlobalConfiguration.getString(S3_HOST_KEY, DEFAULT_S3_HOST); } this.port = name.getPort(); if (this.port == -1) { LOG.debug("Provided URI does not provide a port to connect to, using configuration..."); this.port = GlobalConfiguration.getInteger(S3_PORT_KEY, DEFAULT_S3_PORT); } final String userInfo = name.getUserInfo(); String awsAccessKey = null; String awsSecretKey = null; if (userInfo != null) { final String[] splits = userInfo.split(":"); if (splits.length > 1) { awsAccessKey = URLDecoder.decode(splits[0], URL_ENCODE_CHARACTER); awsSecretKey = URLDecoder.decode(splits[1], URL_ENCODE_CHARACTER); } } if (awsAccessKey == null) { LOG.debug("Provided URI does not provide an access key to Amazon S3, using configuration..."); awsAccessKey = GlobalConfiguration.getString(S3_ACCESS_KEY_KEY, null); if (awsAccessKey == null) { throw new IOException("Cannot determine access key to Amazon S3"); } } if (awsSecretKey == null) { LOG.debug("Provided URI does not provide a secret key to Amazon S3, using configuration..."); awsSecretKey = GlobalConfiguration.getString(S3_SECRET_KEY_KEY, null); if (awsSecretKey == null) { throw new IOException("Cannot determine secret key to Amazon S3"); } } final AWSCredentials credentials = new BasicAWSCredentials(awsAccessKey, awsSecretKey); this.s3Client = new AmazonS3Client(credentials); initializeDirectoryStructure(name); }
From source file:eu.stratosphere.runtime.fs.s3.S3FileSystem.java
License:Apache License
@Override public void initialize(URI name) throws IOException { this.host = name.getHost(); if (this.host == null) { LOG.debug("Provided URI does not provide a host to connect to, using configuration..."); this.host = GlobalConfiguration.getString(S3_HOST_KEY, DEFAULT_S3_HOST); }//from w w w.j a v a2 s .c o m this.port = name.getPort(); if (this.port == -1) { LOG.debug("Provided URI does not provide a port to connect to, using configuration..."); this.port = GlobalConfiguration.getInteger(S3_PORT_KEY, DEFAULT_S3_PORT); } final String userInfo = name.getUserInfo(); String awsAccessKey = null; String awsSecretKey = null; if (userInfo != null) { final String[] splits = userInfo.split(":"); if (splits.length > 1) { awsAccessKey = URLDecoder.decode(splits[0], URL_ENCODE_CHARACTER); awsSecretKey = URLDecoder.decode(splits[1], URL_ENCODE_CHARACTER); } } if (awsAccessKey == null) { LOG.debug("Provided URI does not provide an access key to Amazon S3, using configuration..."); awsAccessKey = GlobalConfiguration.getString(S3_ACCESS_KEY_KEY, null); if (awsAccessKey == null) { throw new IOException("Cannot determine access key to Amazon S3"); } } if (awsSecretKey == null) { LOG.debug("Provided URI does not provide a secret key to Amazon S3, using configuration..."); awsSecretKey = GlobalConfiguration.getString(S3_SECRET_KEY_KEY, null); if (awsSecretKey == null) { throw new IOException("Cannot determine secret key to Amazon S3"); } } final AWSCredentials credentials = new BasicAWSCredentials(awsAccessKey, awsSecretKey); this.s3Client = new AmazonS3Client(credentials); initializeDirectoryStructure(name); }
From source file:example.uploads3.UploadS3.java
License:Apache License
public static void main(String[] args) throws Exception { String uploadFileName = args[0]; String bucketName = "haos3"; String keyName = "test/byspark.txt"; // Create a Java Spark Context. SparkConf conf = new SparkConf().setAppName("UploadS3"); JavaSparkContext sc = new JavaSparkContext(conf); AmazonS3 s3client = new AmazonS3Client(new ProfileCredentialsProvider()); try {/* w ww . j ava 2 s . c o m*/ System.out.println("Uploading a new object to S3 from a file\n"); File file = new File(uploadFileName); PutObjectRequest putRequest = new PutObjectRequest(bucketName, keyName, file); // Request server-side encryption. ObjectMetadata objectMetadata = new ObjectMetadata(); objectMetadata.setServerSideEncryption("AES256"); putRequest.setMetadata(objectMetadata); s3client.putObject(putRequest); } catch (AmazonServiceException ase) { System.out.println("Caught an AmazonServiceException, which " + "means your request made it " + "to Amazon S3, but was rejected with an error response" + " for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { System.out.println("Caught an AmazonClientException, which " + "means the client encountered " + "an internal error while trying to " + "communicate with S3, " + "such as not being able to access the network."); System.out.println("Error Message: " + ace.getMessage()); } }
From source file:exemplos.S3Sample.java
License:Open Source License
public static void main(String[] args) throws IOException { /*/*w w w .ja va 2s . c om*/ * This credentials provider implementation loads your AWS credentials * from a properties file at the root of your classpath. * * Important: Be sure to fill in your AWS access credentials in the * AwsCredentials.properties file before you try to run this * sample. * http://aws.amazon.com/security-credentials */ AmazonS3 s3 = new AmazonS3Client(new ClasspathPropertiesFileCredentialsProvider()); Region usWest2 = Region.getRegion(Regions.US_WEST_2); s3.setRegion(usWest2); String bucketName = "my-first-s3-bucket-" + UUID.randomUUID(); String key = "MyObjectKey"; System.out.println("==========================================="); System.out.println("Getting Started with Amazon S3"); System.out.println("===========================================\n"); try { /* * Create a new S3 bucket - Amazon S3 bucket names are globally unique, * so once a bucket name has been taken by any user, you can't create * another bucket with that same name. * * You can optionally specify a location for your bucket if you want to * keep your data closer to your applications or users. */ System.out.println("Creating bucket " + bucketName + "\n"); s3.createBucket(bucketName); /* * List the buckets in your account */ System.out.println("Listing buckets"); for (Bucket bucket : s3.listBuckets()) { System.out.println(" - " + bucket.getName()); } System.out.println(); /* * Upload an object to your bucket - You can easily upload a file to * S3, or upload directly an InputStream if you know the length of * the data in the stream. You can also specify your own metadata * when uploading to S3, which allows you set a variety of options * like content-type and content-encoding, plus additional metadata * specific to your applications. */ System.out.println("Uploading a new object to S3 from a file\n"); s3.putObject(new PutObjectRequest(bucketName, key, createSampleFile())); /* * Download an object - When you download an object, you get all of * the object's metadata and a stream from which to read the contents. * It's important to read the contents of the stream as quickly as * possibly since the data is streamed directly from Amazon S3 and your * network connection will remain open until you read all the data or * close the input stream. * * GetObjectRequest also supports several other options, including * conditional downloading of objects based on modification times, * ETags, and selectively downloading a range of an object. */ System.out.println("Downloading an object"); S3Object object = s3.getObject(new GetObjectRequest(bucketName, key)); System.out.println("Content-Type: " + object.getObjectMetadata().getContentType()); displayTextInputStream(object.getObjectContent()); /* * List objects in your bucket by prefix - There are many options for * listing the objects in your bucket. Keep in mind that buckets with * many objects might truncate their results when listing their objects, * so be sure to check if the returned object listing is truncated, and * use the AmazonS3.listNextBatchOfObjects(...) operation to retrieve * additional results. */ System.out.println("Listing objects"); ObjectListing objectListing = s3 .listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix("My")); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { System.out.println( " - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")"); } System.out.println(); /* * Delete an object - Unless versioning has been turned on for your bucket, * there is no way to undelete an object, so use caution when deleting objects. */ System.out.println("Deleting an object\n"); s3.deleteObject(bucketName, key); /* * Delete a bucket - A bucket must be completely empty before it can be * deleted, so remember to delete any objects from your buckets before * you try to delete them. */ System.out.println("Deleting bucket " + bucketName + "\n"); s3.deleteBucket(bucketName); } catch (AmazonServiceException ase) { System.out.println("Caught an AmazonServiceException, which means your request made it " + "to Amazon S3, but was rejected with an error response for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { System.out.println("Caught an AmazonClientException, which means the client encountered " + "a serious internal problem while trying to communicate with S3, " + "such as not being able to access the network."); System.out.println("Error Message: " + ace.getMessage()); } }
From source file:fr.ens.biologie.genomique.eoulsan.data.protocols.S3DataProtocol.java
License:LGPL
/** * Get the AmazonS3 object./*from w w w. j av a2 s . c o m*/ * @return an AmazonS3 */ private AmazonS3 getS3() { if (this.s3 == null) { final Settings settings = EoulsanRuntime.getSettings(); this.s3 = new AmazonS3Client( new BasicAWSCredentials(settings.getAWSAccessKey(), settings.getAWSSecretKey())); getLogger().info("AWS S3 account owner: " + this.s3.getS3AccountOwner()); this.tx = new TransferManager(this.s3); } return this.s3; }
From source file:fr.eurecom.hybris.kvs.drivers.AmazonKvs.java
License:Apache License
public AmazonKvs(String id, final String accessKey, final String secretKey, String container, boolean enabled, int cost) throws IOException { super(id, container, enabled, cost); BasicAWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey); this.s3 = new AmazonS3Client(credentials); this.tm = new TransferManager(credentials); TransferManagerConfiguration tmc = new TransferManagerConfiguration(); tmc.setMultipartUploadThreshold(30000000L); // 30 MB tmc.setMinimumUploadPartSize(10000000); // 10 MB this.tm.setConfiguration(tmc); this.createContainer(); }
From source file:fsi_admin.JAwsS3Conn.java
License:Open Source License
@SuppressWarnings({ "rawtypes", "unchecked" })
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
String ERROR = null, codErr = null;
try {/*ww w . ja va 2 s . co m*/
Properties parametros = new Properties();
Vector archivos = new Vector();
DiskFileUpload fu = new DiskFileUpload();
List items = fu.parseRequest(request);
Iterator iter = items.iterator();
while (iter.hasNext()) {
FileItem item = (FileItem) iter.next();
if (item.isFormField())
parametros.put(item.getFieldName(), item.getString());
else
archivos.addElement(item);
}
if (parametros.getProperty("SERVER") == null || parametros.getProperty("DATABASE") == null
|| parametros.getProperty("USER") == null || parametros.getProperty("PASSWORD") == null
|| parametros.getProperty("ACTION") == null) {
System.out.println("No recibi parametros de conexin antes del archivo");
ERROR = "ERROR: El servidor no recibi todos los parametros de conexion (SERVER,DATABASE,USER,PASSWORD,ACTION) antes del archivo";
codErr = "3";
ingresarRegistroFallido(request.getRemoteAddr(), request.getRemoteHost(),
parametros.getProperty("SERVER"), parametros.getProperty("USER"),
parametros.getProperty("PASSWORD"), request, ERROR, 3);
}
//Hasta aqui se han enviado todos los parametros ninguno nulo
if (ERROR == null) {
StringBuffer msj = new StringBuffer(), S3BUKT = new StringBuffer(), S3USR = new StringBuffer(),
S3PASS = new StringBuffer();
MutableBoolean COBRAR = new MutableBoolean(false);
MutableDouble COSTO = new MutableDouble(0.0), SALDO = new MutableDouble(0.0);
// Primero obtiene info del S3
if (!obtenInfoAWSS3(request.getRemoteAddr(), request.getRemoteHost(),
parametros.getProperty("SERVER"), parametros.getProperty("DATABASE"),
parametros.getProperty("USER"), parametros.getProperty("PASSWORD"), S3BUKT, S3USR, S3PASS,
msj, COSTO, SALDO, COBRAR)) {
System.out.println("El usuario y contrasea de servicio estan mal");
ERROR = msj.toString();
codErr = "2";
ingresarRegistroFallido(request.getRemoteAddr(), request.getRemoteHost(),
parametros.getProperty("SERVER"), parametros.getProperty("USER"),
parametros.getProperty("PASSWORD"), request, ERROR, 2);
} else {
AWSCredentials credentials = new BasicAWSCredentials(S3USR.toString(), S3PASS.toString());
AmazonS3 s3 = new AmazonS3Client(credentials);
Region usWest2 = Region.getRegion(Regions.US_WEST_2);
s3.setRegion(usWest2);
//System.out.println("AwsConn:" + parametros.getProperty("NOMBRE") + ":parametros.getProperty(NOMBRE)");
String nombre = parametros.getProperty("SERVER") + parametros.getProperty("DATABASE")
+ parametros.getProperty("ID_MODULO") + parametros.getProperty("OBJIDS")
+ parametros.getProperty("IDSEP") + parametros.getProperty("NOMBRE");
//System.out.println("AwsConn_Nombre:" + nombre + ":nombre");
if (parametros.getProperty("ACTION").equals("SUBIR")) {
Double TOTBITES = new Double(Double.parseDouble(parametros.getProperty("TOTBITES")));
Double TAMBITES = new Double(Double.parseDouble(parametros.getProperty("TAMBITES")));
if (COBRAR.booleanValue() && SALDO
.doubleValue() < (COSTO.doubleValue() * (((TOTBITES + TAMBITES) / 1024) / 1024))) {
System.out
.println("El servicio S3 de subida tiene un costo que no alcanza en el saldo");
ERROR = "El servicio S3 de subida tiene un costo que no alcanza en el saldo";
codErr = "2";
ingresarRegistroFallido(request.getRemoteAddr(), request.getRemoteHost(),
parametros.getProperty("SERVER"), parametros.getProperty("USER"),
parametros.getProperty("PASSWORD"), request, ERROR, 2);
} else {
if (!subirArchivo(msj, s3, S3BUKT.toString(), nombre, archivos)) {
System.out.println("No se permiti subir el archivo al s3");
ERROR = msj.toString();
codErr = "3";
ingresarRegistroFallido(request.getRemoteAddr(), request.getRemoteHost(),
parametros.getProperty("SERVER"), parametros.getProperty("USER"),
parametros.getProperty("PASSWORD"), request, ERROR, 3);
} else {
ingresarRegistroExitoso(parametros.getProperty("SERVER"),
parametros.getProperty("DATABASE"), parametros.getProperty("ID_MODULO"),
parametros.getProperty("OBJIDS"), parametros.getProperty("IDSEP"),
parametros.getProperty("NOMBRE"), parametros.getProperty("TAMBITES"));
}
}
} else if (parametros.getProperty("ACTION").equals("ELIMINAR")) {
Double TOTBITES = new Double(Double.parseDouble(parametros.getProperty("TOTBITES")));
if (COBRAR.booleanValue()
&& SALDO.doubleValue() < (COSTO.doubleValue() * ((TOTBITES / 1024) / 1024))) {
System.out
.println("El servicio S3 de borrado tiene un costo que no alcanza en el saldo");
ERROR = "El servicio S3 de borrado tiene un costo que no alcanza en el saldo";
codErr = "2";
ingresarRegistroFallido(request.getRemoteAddr(), request.getRemoteHost(),
parametros.getProperty("SERVER"), parametros.getProperty("USER"),
parametros.getProperty("PASSWORD"), request, ERROR, 2);
} else {
if (!eliminarArchivo(msj, s3, S3BUKT.toString(), nombre)) {
System.out.println("No se permiti eliminar el archivo del s3");
ERROR = msj.toString();
codErr = "3";
ingresarRegistroFallido(request.getRemoteAddr(), request.getRemoteHost(),
parametros.getProperty("SERVER"), parametros.getProperty("USER"),
parametros.getProperty("PASSWORD"), request, ERROR, 3);
} else {
eliminarRegistroExitoso(parametros.getProperty("SERVER"),
parametros.getProperty("DATABASE"), parametros.getProperty("ID_MODULO"),
parametros.getProperty("OBJIDS"), parametros.getProperty("IDSEP"),
parametros.getProperty("NOMBRE"));
}
}
} else if (parametros.getProperty("ACTION").equals("DESCARGAR")) {
Double TOTBITES = new Double(Double.parseDouble(parametros.getProperty("TOTBITES")));
//System.out.println("COBRAR: " + COBRAR.booleanValue() + " SALDO: " + SALDO.doubleValue() + " COSTO: " + COSTO.doubleValue() + " TOTBITES: " + TOTBITES + " TOTMB: " + ((TOTBITES / 1024) / 1024) + " RES: " + (COSTO.doubleValue() * ((TOTBITES / 1024) / 1024)));
if (COBRAR.booleanValue()
&& SALDO.doubleValue() < (COSTO.doubleValue() * ((TOTBITES / 1024) / 1024))) {
System.out.println(
"El servicio S3 de descarga tiene un costo que no alcanza en el saldo");
ERROR = "El servicio S3 de descarga tiene un costo que no alcanza en el saldo";
codErr = "2";
ingresarRegistroFallido(request.getRemoteAddr(), request.getRemoteHost(),
parametros.getProperty("SERVER"), parametros.getProperty("USER"),
parametros.getProperty("PASSWORD"), request, ERROR, 2);
} else {
if (!descargarArchivo(response, msj, s3, S3BUKT.toString(), nombre,
parametros.getProperty("NOMBRE"))) {
System.out.println("No se permiti descargar el archivo del s3");
ERROR = msj.toString();
codErr = "3";
ingresarRegistroFallido(request.getRemoteAddr(), request.getRemoteHost(),
parametros.getProperty("SERVER"), parametros.getProperty("USER"),
parametros.getProperty("PASSWORD"), request, ERROR, 3);
} else
return;
}
}
if (ERROR == null) {
//Devuelve la respuesta al cliente
Element S3 = new Element("S3");
S3.setAttribute("Archivo", nombre);
S3.setAttribute("MsjError", "");
Document Reporte = new Document(S3);
Format format = Format.getPrettyFormat();
format.setEncoding("utf-8");
format.setTextMode(TextMode.NORMALIZE);
XMLOutputter xmlOutputter = new XMLOutputter(format);
ByteArrayOutputStream out = new ByteArrayOutputStream();
xmlOutputter.output(Reporte, out);
byte[] data = out.toByteArray();
ByteArrayInputStream istream = new ByteArrayInputStream(data);
String destino = "Archivo.xml";
JBajarArchivo fd = new JBajarArchivo();
fd.doDownload(response, getServletConfig().getServletContext(), istream, "text/xml",
data.length, destino);
}
}
}
} catch (Exception e) {
e.printStackTrace();
ERROR = "ERROR DE EXCEPCION EN SERVIDOR AWS S3: " + e.getMessage();
}
//Genera el archivo XML de error para ser devuelto al Servidor
if (ERROR != null) {
Element SIGN_ERROR = new Element("SIGN_ERROR");
SIGN_ERROR.setAttribute("CodError", codErr);
SIGN_ERROR.setAttribute("MsjError", ERROR);
Document Reporte = new Document(SIGN_ERROR);
Format format = Format.getPrettyFormat();
format.setEncoding("utf-8");
format.setTextMode(TextMode.NORMALIZE);
XMLOutputter xmlOutputter = new XMLOutputter(format);
ByteArrayOutputStream out = new ByteArrayOutputStream();
xmlOutputter.output(Reporte, out);
byte[] data = out.toByteArray();
ByteArrayInputStream istream = new ByteArrayInputStream(data);
String destino = "SIGN_ERROR.xml";
JBajarArchivo fd = new JBajarArchivo();
fd.doDownload(response, getServletConfig().getServletContext(), istream, "text/xml", data.length,
destino);
}
}
From source file:gobblin.aws.AWSSdkClient.java
License:Apache License
/*** * Creates a new Amazon S3 client to invoke service methods on Amazon S3 * * @return Amazon S3 client to invoke service methods on Amazon S3 *///from w ww . j a v a 2 s.c o m public AmazonS3 getS3Client() { if (lastCacheRefreshTime > 0 && lastCacheRefreshTime >= awsClusterSecurityManager.getLastRefreshTimeInMillis()) { return amazonS3; } synchronized (AWSSdkClient.class) { if (!(lastCacheRefreshTime > 0 && lastCacheRefreshTime >= awsClusterSecurityManager.getLastRefreshTimeInMillis())) { if (awsClusterSecurityManager.isAssumeRoleEnabled()) { amazonS3 = new AmazonS3Client(awsClusterSecurityManager.getBasicSessionCredentials()); } else { amazonS3 = new AmazonS3Client(awsClusterSecurityManager.getBasicAWSCredentials()); } amazonS3.setRegion(region); } } return amazonS3; }
From source file:gov.noaa.pfel.coastwatch.util.FileVisitorDNLS.java
License:Open Source License
/** * This is a convenience method for using this class. * <p>This works with Amazon AWS S3 bucket URLs. Internal /'s in the keys will be * treated as folder separators. If there aren't any /'s, all the keys will * be in the root directory.//from w w w .j a v a 2 s . c o m * * @param tDir The starting directory, with \\ or /, with or without trailing slash. * The resulting directoryPA will contain dirs with matching slashes and trailing slash. * @param tPathRegex a regex to constrain which subdirs to include. * This is ignored if recursive is false. * null or "" is treated as .* (i.e., match everything). * @param tDirectoriesToo if true, each directory name will get its own rows * in the results. * @return a table with columns with DIRECTORY, NAME, LASTMODIFIED, and SIZE columns. * LASTMODIFIED and SIZE are LongArrays -- For directories when the values * are otherwise unknown, the value will be Long.MAX_VALUE. * If directoriesToo=true, the original dir won't be included and any * directory's file NAME will be "". * @throws IOException if trouble */ public static Table oneStep(String tDir, String tFileNameRegex, boolean tRecursive, String tPathRegex, boolean tDirectoriesToo) throws IOException { long time = System.currentTimeMillis(); //is tDir an http URL? if (tDir.matches(FileVisitorDNLS.HTTP_REGEX)) { //Is it an S3 bucket with "files"? //If testing a "dir", url should have a trailing slash. Matcher matcher = AWS_S3_PATTERN.matcher(File2.addSlash(tDir)); //force trailing slash if (matcher.matches()) { //http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGET.html //If files have file-system-like names, e.g., // http://bucketname.s3.amazonaws.com/dir1/dir2/fileName.ext) // http://nasanex.s3.amazonaws.com/NEX-DCP30/BCSD/rcp26/mon/atmos/tasmin/r1i1p1/v1.0/CONUS/tasmin_amon_BCSD_rcp26_r1i1p1_CONUS_NorESM1-M_209601-209912.nc // you still can't request just dir2 info because they aren't directories. // They are just object keys with internal slashes. //So specify prefix in request. Table table = makeEmptyTable(); StringArray directoryPA = (StringArray) table.getColumn(DIRECTORY); StringArray namePA = (StringArray) table.getColumn(NAME); LongArray lastModifiedPA = (LongArray) table.getColumn(LASTMODIFIED); LongArray sizePA = (LongArray) table.getColumn(SIZE); String bucketName = matcher.group(1); String prefix = matcher.group(2); String baseURL = tDir.substring(0, matcher.start(2)); AmazonS3 s3client = new AmazonS3Client(new ProfileCredentialsProvider()); try { if (verbose) String2.log("FileVisitorDNLS.oneStep getting info from AWS S3 at" + "\nURL=" + tDir); //"\nbucket=" + bucketName + " prefix=" + prefix); //I wanted to generate lastMod for dir based on lastMod of files //but it would be inconsistent for different requests (recursive, fileNameRegex). //so just a set of dir names. HashSet<String> dirHashSet = new HashSet(); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName) .withPrefix(prefix); ObjectListing objectListing; do { objectListing = s3client.listObjects(listObjectsRequest); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { String keyFullName = objectSummary.getKey(); String keyDir = File2.getDirectory(baseURL + keyFullName); String keyName = File2.getNameAndExtension(keyFullName); if (debugMode) String2.log( "keyFullName=" + keyFullName + "\nkeyDir=" + keyDir + "\n tDir=" + tDir); if (keyDir.startsWith(tDir) && //it should (tRecursive || keyDir.length() == tDir.length())) { //store this dir if (tDirectoriesToo) { //S3 only returns object keys. I must infer/collect directories. //Store this dir and parents back to tDir. String choppedKeyDir = keyDir; while (choppedKeyDir.length() >= tDir.length()) { if (!dirHashSet.add(choppedKeyDir)) break; //hash set already had this, so will already have parents //chop off last subdirectory choppedKeyDir = File2.getDirectory( choppedKeyDir.substring(0, choppedKeyDir.length() - 1)); //remove trailing / } } //store this file's information //Sometimes directories appear as files are named "" with size=0. //I don't store those as files. if (debugMode) String2.log("keyName=" + keyFullName + "\n tFileNameRegex=" + tFileNameRegex + " matches=" + keyName.matches(tFileNameRegex)); if (keyName.length() > 0 && keyName.matches(tFileNameRegex)) { directoryPA.add(keyDir); namePA.add(keyName); lastModifiedPA.add(objectSummary.getLastModified().getTime()); //epoch millis sizePA.add(objectSummary.getSize()); //long } } } listObjectsRequest.setMarker(objectListing.getNextMarker()); } while (objectListing.isTruncated()); //add directories to the table if (tDirectoriesToo) { Iterator<String> it = dirHashSet.iterator(); while (it.hasNext()) { directoryPA.add(it.next()); namePA.add(""); lastModifiedPA.add(Long.MAX_VALUE); sizePA.add(Long.MAX_VALUE); } } table.leftToRightSortIgnoreCase(2); return table; } catch (AmazonServiceException ase) { throw new IOException("AmazonServiceException: " + ase.getErrorType() + " ERROR, HTTP Code=" + ase.getStatusCode() + ": " + ase.getMessage(), ase); } catch (AmazonClientException ace) { throw new IOException(ace.getMessage(), ace); } } //HYRAX before THREDDS //http://dods.jpl.nasa.gov/opendap/ocean_wind/ccmp/L3.5a/data/flk/1988/ matcher = HYRAX_PATTERN.matcher(tDir); if (matcher.matches()) { try { if (verbose) String2.log("FileVisitorDNLS.oneStep getting info from Hyrax at" + "\nURL=" + tDir); Table table = makeEmptyTable(); StringArray directoryPA = (StringArray) table.getColumn(DIRECTORY); StringArray namePA = (StringArray) table.getColumn(NAME); LongArray lastModifiedPA = (LongArray) table.getColumn(LASTMODIFIED); LongArray sizePA = (LongArray) table.getColumn(SIZE); DoubleArray lastModDA = new DoubleArray(); addToHyraxUrlList(tDir, tFileNameRegex, tRecursive, tPathRegex, tDirectoriesToo, namePA, lastModDA, sizePA); lastModifiedPA.append(lastModDA); int n = namePA.size(); for (int i = 0; i < n; i++) { String fn = namePA.get(i); directoryPA.add(File2.getDirectory(fn)); namePA.set(i, File2.getNameAndExtension(fn)); } table.leftToRightSortIgnoreCase(2); return table; } catch (Throwable t) { throw new IOException(t.getMessage(), t); } } //THREDDS matcher = THREDDS_PATTERN.matcher(tDir); if (matcher.matches()) { try { if (verbose) String2.log("FileVisitorDNLS.oneStep getting info from THREDDS at" + "\nURL=" + tDir); Table table = makeEmptyTable(); StringArray directoryPA = (StringArray) table.getColumn(DIRECTORY); StringArray namePA = (StringArray) table.getColumn(NAME); LongArray lastModifiedPA = (LongArray) table.getColumn(LASTMODIFIED); LongArray sizePA = (LongArray) table.getColumn(SIZE); DoubleArray lastModDA = new DoubleArray(); addToThreddsUrlList(tDir, tFileNameRegex, tRecursive, tPathRegex, tDirectoriesToo, namePA, lastModDA, sizePA); lastModifiedPA.append(lastModDA); int n = namePA.size(); for (int i = 0; i < n; i++) { String fn = namePA.get(i); directoryPA.add(File2.getDirectory(fn)); namePA.set(i, File2.getNameAndExtension(fn)); } table.leftToRightSortIgnoreCase(2); return table; } catch (Throwable t) { throw new IOException(t.getMessage(), t); } } //default: Apache-style WAF try { if (verbose) String2.log("FileVisitorDNLS.oneStep getting info from Apache-style WAF at" + "\nURL=" + tDir); Table table = makeEmptyTable(); StringArray directorySA = (StringArray) table.getColumn(DIRECTORY); StringArray nameSA = (StringArray) table.getColumn(NAME); LongArray lastModLA = (LongArray) table.getColumn(LASTMODIFIED); LongArray sizeLA = (LongArray) table.getColumn(SIZE); addToWAFUrlList(tDir, tFileNameRegex, tRecursive, tPathRegex, tDirectoriesToo, directorySA, nameSA, lastModLA, sizeLA); table.leftToRightSortIgnoreCase(2); return table; } catch (Throwable t) { throw new IOException(t.getMessage(), t); } } //local files //follow symbolic links: https://docs.oracle.com/javase/7/docs/api/java/nio/file/FileVisitor.html //But this doesn't follow Windows symbolic link .lnk's: // http://bugs.java.com/bugdatabase/view_bug.do?bug_id=4237760 FileVisitorDNLS fv = new FileVisitorDNLS(tDir, tFileNameRegex, tRecursive, tPathRegex, tDirectoriesToo); EnumSet<FileVisitOption> opts = EnumSet.of(FileVisitOption.FOLLOW_LINKS); Files.walkFileTree(FileSystems.getDefault().getPath(tDir), opts, //follow symbolic links Integer.MAX_VALUE, //maxDepth fv); fv.table.leftToRightSortIgnoreCase(2); if (verbose) String2.log("FileVisitorDNLS.oneStep(local) finished successfully. n=" + fv.directoryPA.size() + " time=" + (System.currentTimeMillis() - time) + "ms"); return fv.table; }