List of usage examples for com.amazonaws.services.s3 AmazonS3 getObject
public S3Object getObject(GetObjectRequest getObjectRequest) throws SdkClientException, AmazonServiceException;
Gets the object stored in Amazon S3 under the specified bucket and key.
From source file:gobblin.aws.AWSSdkClient.java
License:Apache License
/*** * Download a S3 object to local directory * * @param s3ObjectSummary S3 object summary for the object to download * @param targetDirectory Local target directory to download the object to * @throws IOException If any errors were encountered in downloading the object */// w w w . j ava 2 s . c o m public void downloadS3Object(S3ObjectSummary s3ObjectSummary, String targetDirectory) throws IOException { final AmazonS3 amazonS3 = getS3Client(); final GetObjectRequest getObjectRequest = new GetObjectRequest(s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey()); final S3Object s3Object = amazonS3.getObject(getObjectRequest); final String targetFile = StringUtils.removeEnd(targetDirectory, File.separator) + File.separator + s3Object.getKey(); FileUtils.copyInputStreamToFile(s3Object.getObjectContent(), new File(targetFile)); LOGGER.info("S3 object downloaded to file: " + targetFile); }
From source file:gov.usgs.cida.iplover.util.ImageStorage.java
public static byte[] get(String uuid) throws IOException { AmazonS3 s3 = prepS3Client(); String imageKey = KEY_BASE + "/" + uuid + ".jpg"; S3Object object = s3.getObject(new GetObjectRequest(BUCKET_NAME, imageKey)); return IOUtils.toByteArray(object.getObjectContent()); }
From source file:hydrograph.engine.spark.datasource.utils.AWSS3Util.java
License:Apache License
public void download(RunFileTransferEntity runFileTransferEntity) { log.debug("Start AWSS3Util download"); File filecheck = new File(runFileTransferEntity.getLocalPath()); if (runFileTransferEntity.getFailOnError()) if (!(filecheck.exists() && filecheck.isDirectory()) && !(runFileTransferEntity.getLocalPath().contains("hdfs://"))) { throw new AWSUtilException("Invalid local path"); }//from w ww. j av a2s . c o m boolean fail_if_exist = false; int retryAttempt = 0; int i; String amazonFileUploadLocationOriginal = null; String keyName = null; if (runFileTransferEntity.getRetryAttempt() == 0) retryAttempt = 1; else retryAttempt = runFileTransferEntity.getRetryAttempt(); for (i = 0; i < retryAttempt; i++) { log.info("connection attempt: " + (i + 1)); try { AmazonS3 s3Client = null; ClientConfiguration clientConf = new ClientConfiguration(); clientConf.setProtocol(Protocol.HTTPS); if (runFileTransferEntity.getCrediationalPropertiesFile() == null) { BasicAWSCredentials creds = new BasicAWSCredentials(runFileTransferEntity.getAccessKeyID(), runFileTransferEntity.getSecretAccessKey()); s3Client = AmazonS3ClientBuilder.standard().withClientConfiguration(clientConf) .withRegion(runFileTransferEntity.getRegion()) .withCredentials(new AWSStaticCredentialsProvider(creds)).build(); } else { File securityFile = new File(runFileTransferEntity.getCrediationalPropertiesFile()); PropertiesCredentials creds = new PropertiesCredentials(securityFile); s3Client = AmazonS3ClientBuilder.standard().withClientConfiguration(clientConf) .withRegion(runFileTransferEntity.getRegion()) .withCredentials(new AWSStaticCredentialsProvider(creds)).build(); } String s3folderName = null; String filepath = runFileTransferEntity.getFolder_name_in_bucket(); if (filepath.lastIndexOf("/") != -1) { s3folderName = filepath.substring(0, filepath.lastIndexOf("/")); keyName = filepath.substring(filepath.lastIndexOf("/") + 1); } else { keyName = filepath; } log.debug("keyName is: " + keyName); log.debug("bucket name is:" + runFileTransferEntity.getBucketName()); log.debug("Folder Name is" + runFileTransferEntity.getFolder_name_in_bucket()); if (s3folderName != null) { amazonFileUploadLocationOriginal = runFileTransferEntity.getBucketName() + "/" + s3folderName; } else { amazonFileUploadLocationOriginal = runFileTransferEntity.getBucketName(); } if (runFileTransferEntity.getLocalPath().contains("hdfs://")) { String outputPath = runFileTransferEntity.getLocalPath(); String s1 = outputPath.substring(7, outputPath.length()); String s2 = s1.substring(0, s1.indexOf("/")); File f = new File("/tmp"); if (!f.exists()) f.mkdir(); GetObjectRequest request = new GetObjectRequest(amazonFileUploadLocationOriginal, keyName); S3Object object = s3Client.getObject(request); if (runFileTransferEntity.getEncoding() != null) object.getObjectMetadata().setContentEncoding(runFileTransferEntity.getEncoding()); File fexist = new File(runFileTransferEntity.getLocalPath() + File.separatorChar + keyName); if (runFileTransferEntity.getOverwrite().trim().equalsIgnoreCase("Overwrite If Exists")) { S3ObjectInputStream objectContent = object.getObjectContent(); IOUtils.copyLarge(objectContent, new FileOutputStream("/tmp/" + keyName)); } else { if (!(fexist.exists() && !fexist.isDirectory())) { S3ObjectInputStream objectContent = object.getObjectContent(); IOUtils.copyLarge(objectContent, new FileOutputStream( runFileTransferEntity.getLocalPath() + File.separatorChar + keyName)); } else { fail_if_exist = true; Log.error("File already exists"); throw new AWSUtilException("File already exists"); } } Configuration conf = new Configuration(); conf.set("fs.defaultFS", "hdfs://" + s2); FileSystem hdfsFileSystem = FileSystem.get(conf); String s = outputPath.substring(7, outputPath.length()); String hdfspath = s.substring(s.indexOf("/"), s.length()); Path local = new Path("/tmp/" + keyName); Path hdfs = new Path(hdfspath); hdfsFileSystem.copyFromLocalFile(local, hdfs); } else { GetObjectRequest request = new GetObjectRequest(amazonFileUploadLocationOriginal, keyName); S3Object object = s3Client.getObject(request); if (runFileTransferEntity.getEncoding() != null) object.getObjectMetadata().setContentEncoding(runFileTransferEntity.getEncoding()); File fexist = new File(runFileTransferEntity.getLocalPath() + File.separatorChar + keyName); if (runFileTransferEntity.getOverwrite().trim().equalsIgnoreCase("Overwrite If Exists")) { S3ObjectInputStream objectContent = object.getObjectContent(); IOUtils.copyLarge(objectContent, new FileOutputStream( runFileTransferEntity.getLocalPath() + File.separatorChar + keyName)); } else { if (!(fexist.exists() && !fexist.isDirectory())) { S3ObjectInputStream objectContent = object.getObjectContent(); IOUtils.copyLarge(objectContent, new FileOutputStream( runFileTransferEntity.getLocalPath() + File.separatorChar + keyName)); } else { fail_if_exist = true; Log.error("File already exists"); throw new AWSUtilException("File already exists"); } } } } catch (AmazonServiceException e) { log.error("Amazon Service Exception", e); if (e.getStatusCode() == 403 || e.getStatusCode() == 404) { if (runFileTransferEntity.getFailOnError()) { Log.error("Incorrect details provided.Please provide correct details", e); throw new AWSUtilException("Incorrect details provided"); } else { Log.error("Unknown amezon exception occured", e); } } { try { Thread.sleep(runFileTransferEntity.getRetryAfterDuration()); } catch (Exception e1) { Log.error("Exception occured while sleeping the thread"); } continue; } } catch (Error e) { Log.error("Error occured while sleeping the thread"); throw new AWSUtilException(e); } catch (Exception e) { log.error("error while transfering file", e); try { Thread.sleep(runFileTransferEntity.getRetryAfterDuration()); } catch (Exception e1) { } catch (Error err) { Log.error("Error occured while downloading"); throw new AWSUtilException(err); } continue; } done = true; break; } if (runFileTransferEntity.getFailOnError() && !done) { log.error("File transfer failed"); throw new AWSUtilException("File transfer failed"); } else if (!done) { log.error("File transfer failed but mentioned fail on error as false"); } if (i == runFileTransferEntity.getRetryAttempt()) { if (runFileTransferEntity.getFailOnError()) { throw new AWSUtilException("File transfer failed"); } } log.debug("Finished AWSS3Util download"); }
From source file:ics.uci.edu.amazons3.S3Sample.java
License:Open Source License
public static void main(String[] args) throws IOException { /*//from w w w.j av a2 s .c o m * This credentials provider implementation loads your AWS credentials * from a properties file at the root of your classpath. * * Important: Be sure to fill in your AWS access credentials in the * AwsCredentials.properties file before you try to run this * sample. * http://aws.amazon.com/security-credentials */ final AmazonS3 s3 = new AmazonS3Client( new BasicAWSCredentials("AKIAJTW5BOY6EXOGV2YQ", "PDcnFYIf9Hdo9GsKTEjLXretZ3yEg4mRCDQKjxu6")); String bucketName = "my-first-s3-bucket-" + UUID.randomUUID(); String key = "MyObjectKey"; System.out.println("==========================================="); System.out.println("Getting Started with Amazon S3"); System.out.println("===========================================\n"); try { /* * Create a new S3 bucket - Amazon S3 bucket names are globally unique, * so once a bucket name has been taken by any user, you can't create * another bucket with that same name. * * You can optionally specify a location for your bucket if you want to * keep your data closer to your applications or users. */ System.out.println("Creating bucket " + bucketName + "\n"); s3.createBucket(bucketName); /* * List the buckets in your account */ System.out.println("Listing buckets"); for (Bucket bucket : s3.listBuckets()) { System.out.println(" - " + bucket.getName()); } System.out.println(); /* * Upload an object to your bucket - You can easily upload a file to * S3, or upload directly an InputStream if you know the length of * the data in the stream. You can also specify your own metadata * when uploading to S3, which allows you set a variety of options * like content-type and content-encoding, plus additional metadata * specific to your applications. */ System.out.println("Uploading a new object to S3 from a file\n"); s3.putObject(new PutObjectRequest(bucketName, key, createSampleFile())); /* * Download an object - When you download an object, you get all of * the object's metadata and a stream from which to read the contents. * It's important to read the contents of the stream as quickly as * possibly since the data is streamed directly from Amazon S3 and your * network connection will remain open until you read all the data or * close the input stream. * * GetObjectRequest also supports several other options, including * conditional downloading of objects based on modification times, * ETags, and selectively downloading a range of an object. */ System.out.println("Downloading an object"); S3Object object = s3.getObject(new GetObjectRequest(bucketName, key)); System.out.println("Content-Type: " + object.getObjectMetadata().getContentType()); displayTextInputStream(object.getObjectContent()); /* * List objects in your bucket by prefix - There are many options for * listing the objects in your bucket. Keep in mind that buckets with * many objects might truncate their results when listing their objects, * so be sure to check if the returned object listing is truncated, and * use the AmazonS3.listNextBatchOfObjects(...) operation to retrieve * additional results. */ System.out.println("Listing objects"); ObjectListing objectListing = s3 .listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix("My")); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { System.out.println( " - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")"); } System.out.println(); /* * Delete an object - Unless versioning has been turned on for your bucket, * there is no way to undelete an object, so use caution when deleting objects. */ System.out.println("Deleting an object\n"); s3.deleteObject(bucketName, key); /* * Delete a bucket - A bucket must be completely empty before it can be * deleted, so remember to delete any objects from your buckets before * you try to delete them. */ System.out.println("Deleting bucket " + bucketName + "\n"); s3.deleteBucket(bucketName); } catch (AmazonServiceException ase) { System.out.println("Caught an AmazonServiceException, which means your request made it " + "to Amazon S3, but was rejected with an error response for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { System.out.println("Caught an AmazonClientException, which means the client encountered " + "a serious internal problem while trying to communicate with S3, " + "such as not being able to access the network."); System.out.println("Error Message: " + ace.getMessage()); } }
From source file:io.dockstore.common.FileProvisioning.java
License:Apache License
private void downloadFromS3(String path, String targetFilePath) { AmazonS3 s3Client = getAmazonS3Client(config); String trimmedPath = path.replace("s3://", ""); List<String> splitPathList = Lists.newArrayList(trimmedPath.split("/")); String bucketName = splitPathList.remove(0); S3Object object = s3Client.getObject(new GetObjectRequest(bucketName, Joiner.on("/").join(splitPathList))); try {/*from ww w . j a v a2 s . c o m*/ FileOutputStream outputStream = new FileOutputStream(new File(targetFilePath)); S3ObjectInputStream inputStream = object.getObjectContent(); long inputSize = object.getObjectMetadata().getContentLength(); copyFromInputStreamToOutputStream(inputStream, inputSize, outputStream); } catch (IOException e) { LOG.error(e.getMessage()); throw new RuntimeException("Could not provision input files from S3", e); } }
From source file:jp.sanix.analysis.java
License:Open Source License
public static void main(String[] args) throws IOException, SQLException, AmazonServiceException, JSONException, NullPointerException, ParseException { if (args.length < 1) { System.out.println("Please specify at least an arguments that is pvs_serial_id."); System.out.println("analysis A9990004 // get today's data"); System.out.println("analysis A9990004 2015/05/30 // get from 2015/05/30 to today's data"); System.out.println("analysis A9990004 2015/05/30 2015/06/30"); System.exit(-1);/*w w w . ja v a2 s .c o m*/ } String id = args[0]; SimpleDateFormat format = new SimpleDateFormat("yyyy/MM/dd"); format.setTimeZone(TimeZone.getTimeZone("JST")); SimpleDateFormat normalformat = new SimpleDateFormat("yyyy/MM/dd'T'HH:mm:ss'Z'"); normalformat.setTimeZone(TimeZone.getTimeZone("JST")); SimpleDateFormat pgformat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); pgformat.setTimeZone(TimeZone.getTimeZone("UTC")); Calendar cal = Calendar.getInstance(); Calendar end = Calendar.getInstance(); String today = toDate(cal); System.out.println(id); try { cal.setTime(format.parse(args[1])); } catch (ParseException e) { } catch (ArrayIndexOutOfBoundsException e) { } try { end.setTime(format.parse(args[2])); } catch (ParseException e) { } catch (ArrayIndexOutOfBoundsException e) { } end.add(Calendar.DAY_OF_MONTH, 1); AmazonS3 s3 = new AmazonS3Client(); s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1)); Connection db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS); Statement st = db.createStatement(); ResultSet rs = st.executeQuery( "SELECT data, pvs_unique_code FROM data WHERE pvs_serial_id='" + id + "' OFFSET 0 LIMIT 1;"); rs.next(); String json = rs.getString(1); String key = rs.getString(2); rs.close(); db.close(); Date recent = new Date(); while (cal.before(end)) { xlsSheet xls = new xlsSheet(json); System.out.println("Getting data of " + toDate(cal)); /* AWS S3????? */ String bucketName = "pvdata-storage-production"; // String bucketName = "pvdata-storage-staging"; System.out.println("Get s3 data by key='" + bucketName + "/data/" + key + "/" + toDate(cal) + "/'"); ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName) .withPrefix("data/" + key + "/" + toDate(cal) + "/")); /* get data from s3 */ do { for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { String keyname = objectSummary.getKey(); S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname)); BufferedReader reader = new BufferedReader( new InputStreamReader(new GZIPInputStream(object.getObjectContent()))); while (reader.ready()) { String line = reader.readLine(); try { json = line.substring(line.indexOf("{")); recent = parseDate(line.substring(0, 25)); xls.putData(json); } catch (NullPointerException e) { } } reader.close(); object.close(); } objectListing = s3.listNextBatchOfObjects(objectListing); } while (objectListing.getMarker() != null); /* if today, read postgres to get recent data */ if (toDate(cal).equals(today)) { System.out.println("Get recent data from postgres"); try { db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS); st = db.createStatement(); String sql = "SELECT data FROM data WHERE pvs_unique_code='" + key + "' AND created_at > '" + pgformat.format(recent) + "';"; System.out.println(sql); rs = st.executeQuery(sql); while (rs.next()) { json = rs.getString(1); xls.putData(json); } rs.close(); db.close(); } catch (PSQLException e) { } catch (ParseException e) { } } System.out.println("Write Buffer"); xls.writeBuffer(); File file = File.createTempFile("temp", ".xlsx"); file.deleteOnExit(); xls.putFile(new FileOutputStream(file)); System.out.println("Put S3"); s3.putObject(new PutObjectRequest("sanix-data-analysis", FOLDER + id + "-" + toDate(cal).replace("/", "-") + ".xlsx", file)); System.out.println("Finished: " + toDate(cal)); cal.add(Calendar.DAY_OF_MONTH, 1); } File file = File.createTempFile("temp", ".html"); file.deleteOnExit(); BufferedWriter bw = new BufferedWriter(new FileWriter(file)); ObjectListing objectListing = s3 .listObjects(new ListObjectsRequest().withBucketName("sanix-data-analysis").withPrefix(FOLDER)); bw.write("<html><head></head><body><ul>\n"); do { for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { String keyname = objectSummary.getKey(); if (Pattern.compile("\\.xlsx$|\\.html$").matcher(keyname).find()) { bw.write("<li><a href=\"https://s3-ap-northeast-1.amazonaws.com/sanix-data-analysis/" + keyname + "\">" + keyname.replaceAll("^[^\\/]*\\/", "") + "</a></li>\n"); } } objectListing = s3.listNextBatchOfObjects(objectListing); } while (objectListing.getMarker() != null); bw.write("</ul></body></html>\n"); bw.flush(); bw.close(); s3.putObject(new PutObjectRequest("sanix-data-analysis", FOLDER + "index.html", file)); }
From source file:jp.sanix.analysisLocal.java
License:Open Source License
public static void main(String[] args) throws IOException, SQLException, AmazonServiceException, JSONException, NullPointerException, ParseException { BufferedReader br = new BufferedReader( new InputStreamReader(new FileInputStream(new File("canceled.csv")), "SJIS")); br.readLine();/*from www . j a v a2 s. co m*/ String l; while ((l = br.readLine()) != null) { String[] col = l.split(","); String id = col[0]; String datefrom = col[1]; String dateto = col[1]; SimpleDateFormat format = new SimpleDateFormat("yyyy/MM/dd"); format.setTimeZone(TimeZone.getTimeZone("JST")); SimpleDateFormat pgformat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); pgformat.setTimeZone(TimeZone.getTimeZone("UTC")); Calendar cal = Calendar.getInstance(); Calendar end = Calendar.getInstance(); String today = toDate(cal); try { cal.setTime(format.parse(datefrom)); } catch (ParseException e) { } try { end.setTime(format.parse(dateto)); end.add(Calendar.DAY_OF_MONTH, 1); } catch (ParseException e) { } AmazonS3 s3 = new AmazonS3Client(); s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1)); Connection db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS); Statement st = db.createStatement(); ResultSet rs = st.executeQuery( "SELECT data, pvs_unique_code FROM data WHERE pvs_serial_id='" + id + "' OFFSET 0 LIMIT 1;"); rs.next(); String json = rs.getString(1); String key = rs.getString(2); rs.close(); db.close(); Date recent = new Date(); while (cal.before(end)) { xlsSheet xls = new xlsSheet(json); System.out.println("Getting data of " + toDate(cal)); /* AWS S3????? */ String bucketName = "pvdata-storage-production"; System.out.println("Get s3 data by key='" + bucketName + "/data/" + key + "/" + toDate(cal) + "/'"); ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName) .withPrefix("data/" + key + "/" + toDate(cal) + "/")); /* get data from s3 */ do { for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { String keyname = objectSummary.getKey(); S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname)); BufferedReader reader = new BufferedReader( new InputStreamReader(new GZIPInputStream(object.getObjectContent()))); while (reader.ready()) { String line = reader.readLine(); try { json = line.substring(line.indexOf("{")); recent = parseDate(line.substring(0, 25)); xls.putData(json); } catch (NullPointerException e) { } } reader.close(); object.close(); } objectListing = s3.listNextBatchOfObjects(objectListing); } while (objectListing.getMarker() != null); /* if today, read postgres to get recent data */ if (toDate(cal).equals(today)) { System.out.println("Get recent data from postgres"); try { db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS); st = db.createStatement(); String sql = "SELECT data FROM data WHERE pvs_unique_code='" + key + "' AND created_at > '" + pgformat.format(recent) + "';"; System.out.println(sql); rs = st.executeQuery(sql); while (rs.next()) { json = rs.getString(1); xls.putData(json); } rs.close(); db.close(); } catch (PSQLException e) { } catch (ParseException e) { } } System.out.println("Write Buffer"); xls.writeBuffer(); File file = new File( "C:\\Users\\SANIX_CORERD\\Desktop\\" + id + "_" + toDate(cal).replace("/", "-") + ".xlsx"); xls.putFile(new FileOutputStream(file)); System.out.println("Finished: " + toDate(cal)); cal.add(Calendar.DAY_OF_MONTH, 1); } } br.close(); }
From source file:jp.sanix.analysisStaging.java
License:Open Source License
public static void main(String[] args) throws IOException, SQLException, AmazonServiceException, JSONException, NullPointerException, ParseException { // if (args.length < 1) { // System.out.println("Please specify at least an arguments that is pvs_serial_id."); // System.out.println("analysis A9990004 // get today's data"); // System.out.println("analysis A9990004 2015/05/30 // get from 2015/05/30 to today's data"); // System.out.println("analysis A9990004 2015/05/30 2015/06/30"); // System.exit(-1); // }//from w ww. ja va 2 s .co m // String id = args[0]; String uniq = "763ff280b16c61f524ae31b33401bcd6"; SimpleDateFormat format = new SimpleDateFormat("yyyy/MM/dd"); format.setTimeZone(TimeZone.getTimeZone("JST")); SimpleDateFormat normalformat = new SimpleDateFormat("yyyy/MM/dd'T'HH:mm:ss'Z'"); normalformat.setTimeZone(TimeZone.getTimeZone("JST")); SimpleDateFormat pgformat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); pgformat.setTimeZone(TimeZone.getTimeZone("UTC")); Calendar cal = Calendar.getInstance(); Calendar end = Calendar.getInstance(); cal.setTime(format.parse("2015/11/23")); end.setTime(format.parse("2015/11/23")); System.out.println(uniq); try { cal.setTime(format.parse(args[1])); } catch (ParseException e) { } catch (ArrayIndexOutOfBoundsException e) { } try { end.setTime(format.parse(args[2])); } catch (ParseException e) { } catch (ArrayIndexOutOfBoundsException e) { } end.add(Calendar.DAY_OF_MONTH, 1); AmazonS3 s3 = new AmazonS3Client(); s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1)); String bucketName = "pvdata-storage-staging"; while (cal.before(end)) { System.out.println("Getting data of " + toDate(cal)); /* AWS S3????? */ // String bucketName = "pvdata-storage-production"; System.out.println("Get s3 data by key='" + bucketName + "/data/" + uniq + "/" + toDate(cal) + "/'"); ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName) .withPrefix("data/" + uniq + "/" + toDate(cal) + "/")); /* get data from s3 */ xlsSheet xls = null; do { Boolean first = true; for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { String keyname = objectSummary.getKey(); S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname)); BufferedReader reader = new BufferedReader( new InputStreamReader(new GZIPInputStream(object.getObjectContent()))); while (reader.ready()) { String line = reader.readLine(); try { String json = line.substring(line.indexOf("{")); if (first) { xls = new xlsSheet(json); first = false; } else { xls.putData(json); } } catch (NullPointerException e) { } } reader.close(); object.close(); } objectListing = s3.listNextBatchOfObjects(objectListing); } while (objectListing.getMarker() != null); System.out.println("Write Buffer"); xls.writeBuffer(); File file = File.createTempFile("temp", ".xlsx"); file.deleteOnExit(); xls.putFile(new FileOutputStream(file)); System.out.println("Put S3"); s3.putObject(new PutObjectRequest("sanix-data-analysis", FOLDER + uniq + "-" + toDate(cal).replace("/", "-") + ".xlsx", file)); System.out.println("Finished: " + toDate(cal)); cal.add(Calendar.DAY_OF_MONTH, 1); } }
From source file:jp.sanix.mukaino.java
License:Open Source License
public static void main(String[] args) throws IOException, SQLException, AmazonServiceException, JSONException, NullPointerException, ParseException { AmazonS3 s3 = new AmazonS3Client(); s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1)); String[] ids = { "A0000089", "A0002570" }; Connection db;/*from w ww . j ava 2 s. c o m*/ if (System.getProperty("user.name").toString().equals("ec2-user")) { db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS); } else { db = DriverManager.getConnection(PG_CON_LOCAL, PG_USER, PG_PASS); } for (String id : ids) { pvSensor.getInstance().getById(id); Statement st = db.createStatement(); ResultSet rs = st.executeQuery("SELECT data, pvs_unique_code FROM data WHERE pvs_serial_id='" + id + "' ORDER BY created_at DESC OFFSET 0 LIMIT 1;"); rs.next(); String json = rs.getString(1); String key = rs.getString(2); rs.close(); st.close(); Calendar cal = Calendar.getInstance(); cal.add(Calendar.DAY_OF_MONTH, -1); // 1?? Calendar end = Calendar.getInstance(); cal.add(Calendar.DAY_OF_MONTH, -1); // 1??? do { String mDate = String.format("%04d-%02d-%02d", cal.get(Calendar.YEAR), cal.get(Calendar.MONTH) + 1, cal.get(Calendar.DAY_OF_MONTH)); xlsSheetMukaino xls = new xlsSheetMukaino(json); /* AWS S3????? */ String bucketName = "pvdata-storage-production"; System.out.print("Get s3 data by key='" + bucketName + String.format("data/%s/%04d/%02d/%02d/", key, cal.get(Calendar.YEAR), cal.get(Calendar.MONTH) + 1, cal.get(Calendar.DAY_OF_MONTH))); ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName) .withPrefix("data/" + key + "/" + mDate.replace("-", "/"))); /* get data from s3 */ do { for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { String keyname = objectSummary.getKey(); S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname)); BufferedReader reader = new BufferedReader( new InputStreamReader(new GZIPInputStream(object.getObjectContent()))); while (reader.ready()) { String line = reader.readLine(); try { json = line.substring(line.indexOf("{")); xls.putData(json); } catch (NullPointerException e) { } } reader.close(); object.close(); System.out.print("."); } objectListing = s3.listNextBatchOfObjects(objectListing); } while (objectListing.getMarker() != null); System.out.println(); xls.writeBuffer(); File file = File.createTempFile("temp", ".xlsx"); file.deleteOnExit(); xls.putFile(new FileOutputStream(file)); s3.putObject(new PutObjectRequest("sanix-data-analysis", "Rw6WHQLqGcvs3wpWwALh80IHAiZqWYzw/" + id + "-" + mDate + ".xlsx", file)); System.out.println("Finished: " + mDate); cal.add(Calendar.DAY_OF_MONTH, 1); } while (cal.before(end)); } db.close(); File file = File.createTempFile("temp", ".html"); file.deleteOnExit(); BufferedWriter bw = new BufferedWriter(new FileWriter(file)); ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName("sanix-data-analysis") .withPrefix("Rw6WHQLqGcvs3wpWwALh80IHAiZqWYzw/")); bw.write("<html><head></head><body><ul>\n"); do { for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { String keyname = objectSummary.getKey(); if (Pattern.compile("\\.xlsx$|\\.html$").matcher(keyname).find()) { bw.write("<li><a href=\"https://s3-ap-northeast-1.amazonaws.com/sanix-data-analysis/" + keyname + "\">" + keyname.replaceAll("^[^\\/]*\\/", "") + "</a></li>\n"); } } objectListing = s3.listNextBatchOfObjects(objectListing); } while (objectListing.getMarker() != null); bw.write("</ul></body></html>\n"); bw.flush(); bw.close(); s3.putObject( new PutObjectRequest("sanix-data-analysis", "Rw6WHQLqGcvs3wpWwALh80IHAiZqWYzw/index.html", file)); }
From source file:jp.sanix.Rawdata.java
License:Open Source License
public TreeMap<Date, JSONObject> get() throws IOException, SQLException, ParseException { TreeMap<Date, JSONObject> data = new TreeMap<Date, JSONObject>(); AmazonS3 s3 = new AmazonS3Client(); s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1)); SimpleDateFormat fmt = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); fmt.setTimeZone(TimeZone.getTimeZone("UTC")); Calendar cal = Calendar.getInstance(); cal.setTime(this.getDate()); /* AWS S3????? */ String bucketName = "pvdata-storage-production"; ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName) .withPrefix("data/" + this.getUniqueCode() + "/" + toDate(cal) + "/")); /* get data from s3 */ do {/*from ww w. j ava2 s . co m*/ for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { String keyname = objectSummary.getKey(); S3Object object = s3.getObject(new GetObjectRequest(BUCKET, keyname)); BufferedReader reader = new BufferedReader( new InputStreamReader(new GZIPInputStream(object.getObjectContent()))); while (reader.ready()) { String line = reader.readLine(); int pos; try { pos = line.indexOf("{"); } catch (NullPointerException e) { continue; } try { String jsons = line.substring(pos); JSONObject json = new JSONObject(jsons); JSONArray arr = json.getJSONObject("data").getJSONArray("sample"); for (int i = 0; i < arr.length(); i++) { JSONObject obj = arr.getJSONObject(i); Date date = fmt.parse(obj.getString("time")); for (int j = 0; j < obj.getJSONArray("powcon").length(); j++) { if (obj.getJSONArray("powcon").getJSONObject(j).get("genKwh") != JSONObject.NULL) { Double genkwh; genkwh = obj.getJSONArray("powcon").getJSONObject(j).getDouble("genKwh"); JSONObject jobj; if (data.containsKey(date)) { jobj = data.get(date); } else { jobj = new JSONObject(); } if (genkwh != null) { jobj.put(String.valueOf(j), genkwh); } if (jobj.length() > 0) { data.put(date, jobj); } } } } } catch (JSONException e) { e.printStackTrace(); } catch (StringIndexOutOfBoundsException e) { e.printStackTrace(); } } reader.close(); object.close(); } objectListing = s3.listNextBatchOfObjects(objectListing); } while (objectListing.getMarker() != null); Calendar today = Calendar.getInstance(); today.setTimeZone(TimeZone.getTimeZone("JST")); if (toDate(cal).equals(toDate(today))) { SimpleDateFormat pgfmt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); pgfmt.setTimeZone(TimeZone.getTimeZone("UTC")); Calendar recent = Calendar.getInstance(); recent.setTimeZone(TimeZone.getTimeZone("UST")); recent.add(Calendar.HOUR, -2); Connection db; if (System.getProperty("user.name").toString().equals("ec2-user")) { db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS); } else { db = DriverManager.getConnection(PG_CON_LOCAL, PG_USER, PG_PASS); } Statement st = db.createStatement(); String sql = "SELECT data FROM data WHERE pvs_unique_code='" + this.getUniqueCode() + "' AND created_at > '" + pgfmt.format(recent.getTime()) + "';"; ResultSet rs = st.executeQuery(sql); while (rs.next()) { String jsons = rs.getString(1); try { JSONObject json = new JSONObject(jsons); JSONArray arr = json.getJSONObject("data").getJSONArray("sample"); for (int i = 0; i < arr.length(); i++) { JSONObject obj = arr.getJSONObject(i); Date date = fmt.parse(obj.getString("time")); for (int j = 0; j < obj.getJSONArray("powcon").length(); j++) { if (obj.getJSONArray("powcon").getJSONObject(j).get("genKwh") != JSONObject.NULL) { Double genkwh; genkwh = obj.getJSONArray("powcon").getJSONObject(j).getDouble("genKwh"); JSONObject jobj; if (data.containsKey(date)) { jobj = data.get(date); } else { jobj = new JSONObject(); } if (genkwh != null) { jobj.put(String.valueOf(j), genkwh); } if (jobj.length() > 0) { data.put(date, jobj); } } } } } catch (JSONException e) { e.printStackTrace(); } } rs.close(); db.close(); } return data; }