List of usage examples for com.amazonaws.services.s3 AmazonS3Client AmazonS3Client
@Deprecated
public AmazonS3Client()
From source file:jp.sanix.mukaino.java
License:Open Source License
public static void main(String[] args) throws IOException, SQLException, AmazonServiceException, JSONException, NullPointerException, ParseException { AmazonS3 s3 = new AmazonS3Client(); s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1)); String[] ids = { "A0000089", "A0002570" }; Connection db;//from w w w.j a va 2 s .c o m if (System.getProperty("user.name").toString().equals("ec2-user")) { db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS); } else { db = DriverManager.getConnection(PG_CON_LOCAL, PG_USER, PG_PASS); } for (String id : ids) { pvSensor.getInstance().getById(id); Statement st = db.createStatement(); ResultSet rs = st.executeQuery("SELECT data, pvs_unique_code FROM data WHERE pvs_serial_id='" + id + "' ORDER BY created_at DESC OFFSET 0 LIMIT 1;"); rs.next(); String json = rs.getString(1); String key = rs.getString(2); rs.close(); st.close(); Calendar cal = Calendar.getInstance(); cal.add(Calendar.DAY_OF_MONTH, -1); // 1?? Calendar end = Calendar.getInstance(); cal.add(Calendar.DAY_OF_MONTH, -1); // 1??? do { String mDate = String.format("%04d-%02d-%02d", cal.get(Calendar.YEAR), cal.get(Calendar.MONTH) + 1, cal.get(Calendar.DAY_OF_MONTH)); xlsSheetMukaino xls = new xlsSheetMukaino(json); /* AWS S3????? */ String bucketName = "pvdata-storage-production"; System.out.print("Get s3 data by key='" + bucketName + String.format("data/%s/%04d/%02d/%02d/", key, cal.get(Calendar.YEAR), cal.get(Calendar.MONTH) + 1, cal.get(Calendar.DAY_OF_MONTH))); ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName) .withPrefix("data/" + key + "/" + mDate.replace("-", "/"))); /* get data from s3 */ do { for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { String keyname = objectSummary.getKey(); S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname)); BufferedReader reader = new BufferedReader( new InputStreamReader(new GZIPInputStream(object.getObjectContent()))); while (reader.ready()) { String line = reader.readLine(); try { json = line.substring(line.indexOf("{")); xls.putData(json); } catch (NullPointerException e) { } } reader.close(); object.close(); System.out.print("."); } objectListing = s3.listNextBatchOfObjects(objectListing); } while (objectListing.getMarker() != null); System.out.println(); xls.writeBuffer(); File file = File.createTempFile("temp", ".xlsx"); file.deleteOnExit(); xls.putFile(new FileOutputStream(file)); s3.putObject(new PutObjectRequest("sanix-data-analysis", "Rw6WHQLqGcvs3wpWwALh80IHAiZqWYzw/" + id + "-" + mDate + ".xlsx", file)); System.out.println("Finished: " + mDate); cal.add(Calendar.DAY_OF_MONTH, 1); } while (cal.before(end)); } db.close(); File file = File.createTempFile("temp", ".html"); file.deleteOnExit(); BufferedWriter bw = new BufferedWriter(new FileWriter(file)); ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName("sanix-data-analysis") .withPrefix("Rw6WHQLqGcvs3wpWwALh80IHAiZqWYzw/")); bw.write("<html><head></head><body><ul>\n"); do { for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { String keyname = objectSummary.getKey(); if (Pattern.compile("\\.xlsx$|\\.html$").matcher(keyname).find()) { bw.write("<li><a href=\"https://s3-ap-northeast-1.amazonaws.com/sanix-data-analysis/" + keyname + "\">" + keyname.replaceAll("^[^\\/]*\\/", "") + "</a></li>\n"); } } objectListing = s3.listNextBatchOfObjects(objectListing); } while (objectListing.getMarker() != null); bw.write("</ul></body></html>\n"); bw.flush(); bw.close(); s3.putObject( new PutObjectRequest("sanix-data-analysis", "Rw6WHQLqGcvs3wpWwALh80IHAiZqWYzw/index.html", file)); }
From source file:jp.sanix.Rawdata.java
License:Open Source License
public TreeMap<Date, JSONObject> get() throws IOException, SQLException, ParseException { TreeMap<Date, JSONObject> data = new TreeMap<Date, JSONObject>(); AmazonS3 s3 = new AmazonS3Client(); s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1)); SimpleDateFormat fmt = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); fmt.setTimeZone(TimeZone.getTimeZone("UTC")); Calendar cal = Calendar.getInstance(); cal.setTime(this.getDate()); /* AWS S3????? */ String bucketName = "pvdata-storage-production"; ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName) .withPrefix("data/" + this.getUniqueCode() + "/" + toDate(cal) + "/")); /* get data from s3 */ do {//from ww w . j av a2 s . co m for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { String keyname = objectSummary.getKey(); S3Object object = s3.getObject(new GetObjectRequest(BUCKET, keyname)); BufferedReader reader = new BufferedReader( new InputStreamReader(new GZIPInputStream(object.getObjectContent()))); while (reader.ready()) { String line = reader.readLine(); int pos; try { pos = line.indexOf("{"); } catch (NullPointerException e) { continue; } try { String jsons = line.substring(pos); JSONObject json = new JSONObject(jsons); JSONArray arr = json.getJSONObject("data").getJSONArray("sample"); for (int i = 0; i < arr.length(); i++) { JSONObject obj = arr.getJSONObject(i); Date date = fmt.parse(obj.getString("time")); for (int j = 0; j < obj.getJSONArray("powcon").length(); j++) { if (obj.getJSONArray("powcon").getJSONObject(j).get("genKwh") != JSONObject.NULL) { Double genkwh; genkwh = obj.getJSONArray("powcon").getJSONObject(j).getDouble("genKwh"); JSONObject jobj; if (data.containsKey(date)) { jobj = data.get(date); } else { jobj = new JSONObject(); } if (genkwh != null) { jobj.put(String.valueOf(j), genkwh); } if (jobj.length() > 0) { data.put(date, jobj); } } } } } catch (JSONException e) { e.printStackTrace(); } catch (StringIndexOutOfBoundsException e) { e.printStackTrace(); } } reader.close(); object.close(); } objectListing = s3.listNextBatchOfObjects(objectListing); } while (objectListing.getMarker() != null); Calendar today = Calendar.getInstance(); today.setTimeZone(TimeZone.getTimeZone("JST")); if (toDate(cal).equals(toDate(today))) { SimpleDateFormat pgfmt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); pgfmt.setTimeZone(TimeZone.getTimeZone("UTC")); Calendar recent = Calendar.getInstance(); recent.setTimeZone(TimeZone.getTimeZone("UST")); recent.add(Calendar.HOUR, -2); Connection db; if (System.getProperty("user.name").toString().equals("ec2-user")) { db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS); } else { db = DriverManager.getConnection(PG_CON_LOCAL, PG_USER, PG_PASS); } Statement st = db.createStatement(); String sql = "SELECT data FROM data WHERE pvs_unique_code='" + this.getUniqueCode() + "' AND created_at > '" + pgfmt.format(recent.getTime()) + "';"; ResultSet rs = st.executeQuery(sql); while (rs.next()) { String jsons = rs.getString(1); try { JSONObject json = new JSONObject(jsons); JSONArray arr = json.getJSONObject("data").getJSONArray("sample"); for (int i = 0; i < arr.length(); i++) { JSONObject obj = arr.getJSONObject(i); Date date = fmt.parse(obj.getString("time")); for (int j = 0; j < obj.getJSONArray("powcon").length(); j++) { if (obj.getJSONArray("powcon").getJSONObject(j).get("genKwh") != JSONObject.NULL) { Double genkwh; genkwh = obj.getJSONArray("powcon").getJSONObject(j).getDouble("genKwh"); JSONObject jobj; if (data.containsKey(date)) { jobj = data.get(date); } else { jobj = new JSONObject(); } if (genkwh != null) { jobj.put(String.valueOf(j), genkwh); } if (jobj.length() > 0) { data.put(date, jobj); } } } } } catch (JSONException e) { e.printStackTrace(); } } rs.close(); db.close(); } return data; }
From source file:jp.sanix.weatherData.java
License:Open Source License
public static void main(String[] args) throws IOException, SQLException, AmazonServiceException, JSONException, NullPointerException, ParseException { AmazonS3 s3 = new AmazonS3Client(); s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1)); BufferedReader br = new BufferedReader(new InputStreamReader( new FileInputStream(new File("src/main/web/?.csv")), "SJIS")); String line = null;//from w w w .jav a 2 s . c o m br.readLine(); // 1??? while ((line = br.readLine()) != null) { String[] col = line.split(","); /* AWS S3????? */ String bucketName = "weather-forecast"; ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName) .withPrefix("day_insolation/" + col[0] + "/")); File file = File.createTempFile("temp", ".csv"); file.deleteOnExit(); BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file), "SJIS")); bw.write( ",?,?,???,,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23\n"); System.out.println(col[0] + ":" + col[1] + col[2]); /* get data from s3 */ int i = 0; do { for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { String keyname = objectSummary.getKey(); S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname)); BufferedReader reader = new BufferedReader(new InputStreamReader(object.getObjectContent())); StringBuilder sb = new StringBuilder(); String line2 = null; System.out.print(String.valueOf(i++) + "\r"); while ((line2 = reader.readLine()) != null) { sb.append(line2); } reader.close(); object.close(); try { JSONObject json = new JSONObject(sb.toString()); bw.write(String.join(",", col) + "," + json.get("date").toString().replace("-", "/") + ","); JSONArray jarr = json.getJSONArray("hour_data"); bw.write(jarr.join(",") + "\n"); // System.out.println(String.join(",", col) + "," + json.get("date").toString().replace("-", "/") + "," + jarr.join(",")); } catch (JSONException e) { // System.exit(1); } } objectListing = s3.listNextBatchOfObjects(objectListing); } while (objectListing.getMarker() != null); bw.flush(); bw.close(); if (i > 0) { s3.putObject(new PutObjectRequest("sanix-data-analysis", STORE_PATH + col[1] + col[2] + "_insolation.csv", file)); } } br.close(); br = new BufferedReader(new InputStreamReader( new FileInputStream(new File("src/main/web/?.csv")), "SJIS")); br.readLine(); // 1??? while ((line = br.readLine()) != null) { String[] col = line.split(","); /* AWS S3????? */ String bucketName = "weather-forecast"; ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName) .withPrefix("day_temperature/" + col[0] + "/")); File file = File.createTempFile("temp", ".csv"); file.deleteOnExit(); BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file), "SJIS")); bw.write( ",?,?,???,,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23\n"); System.out.println(col[0] + ":" + col[1] + col[2]); /* get data from s3 */ int i = 0; do { for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { String keyname = objectSummary.getKey(); S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname)); BufferedReader reader = new BufferedReader(new InputStreamReader(object.getObjectContent())); StringBuilder sb = new StringBuilder(); String line2 = null; System.out.print(String.valueOf(i++) + "\r"); while ((line2 = reader.readLine()) != null) { sb.append(line2); } reader.close(); object.close(); try { JSONObject json = new JSONObject(sb.toString()); bw.write(String.join(",", col) + "," + json.get("date").toString().replace("-", "/") + ","); JSONArray jarr = json.getJSONArray("hour_data"); bw.write(jarr.join(",") + "\n"); // System.out.println(String.join(",", col) + "," + json.get("date").toString().replace("-", "/") + "," + jarr.join(",")); } catch (JSONException e) { // System.exit(1); } } objectListing = s3.listNextBatchOfObjects(objectListing); } while (objectListing.getMarker() != null); bw.flush(); bw.close(); if (i > 0) { s3.putObject(new PutObjectRequest("sanix-data-analysis", STORE_PATH + col[1] + col[2] + "_temperture.csv", file)); } } br.close(); }
From source file:jp.sanix.yokusei.java
License:Open Source License
public static void main(String[] args) throws IOException, SQLException, AmazonServiceException, JSONException, NullPointerException, ParseException { String id = "A0002441"; String datefrom = "2015/10/01"; String dateto = "2015/10/12"; SimpleDateFormat format = new SimpleDateFormat("yyyy/MM/dd"); format.setTimeZone(TimeZone.getTimeZone("JST")); SimpleDateFormat pgformat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); pgformat.setTimeZone(TimeZone.getTimeZone("UTC")); Calendar cal = Calendar.getInstance(); Calendar end = Calendar.getInstance(); String today = toDate(cal);//ww w. j a v a 2s .c o m try { cal.setTime(format.parse(datefrom)); } catch (ParseException e) { } try { end.setTime(format.parse(dateto)); end.add(Calendar.DAY_OF_MONTH, 1); } catch (ParseException e) { } AmazonS3 s3 = new AmazonS3Client(); s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1)); Connection db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS); Statement st = db.createStatement(); ResultSet rs = st.executeQuery( "SELECT data, pvs_unique_code FROM data WHERE pvs_serial_id='" + id + "' OFFSET 0 LIMIT 1;"); rs.next(); String json = rs.getString(1); String key = rs.getString(2); rs.close(); db.close(); Date recent = new Date(); xlsSheetYokusei xls = new xlsSheetYokusei(json); while (cal.before(end)) { System.out.println("Getting data of " + toDate(cal)); /* AWS S3????? */ String bucketName = "pvdata-storage-production"; System.out.println("Get s3 data by key='" + bucketName + "/data/" + key + "/" + toDate(cal) + "/'"); ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName) .withPrefix("data/" + key + "/" + toDate(cal) + "/")); /* get data from s3 */ do { for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { String keyname = objectSummary.getKey(); S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname)); BufferedReader reader = new BufferedReader( new InputStreamReader(new GZIPInputStream(object.getObjectContent()))); String line; while ((line = reader.readLine()) != null) { try { json = line.substring(line.indexOf("{")); xls.putData(json); } catch (NullPointerException e) { } } reader.close(); object.close(); } objectListing = s3.listNextBatchOfObjects(objectListing); } while (objectListing.getMarker() != null); /* if today, read postgres to get recent data */ if (toDate(cal).equals(today)) { System.out.println("Get recent data from postgres"); try { db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS); st = db.createStatement(); String sql = "SELECT data FROM data WHERE pvs_unique_code='" + key + "' AND created_at > '" + pgformat.format(recent) + "';"; System.out.println(sql); rs = st.executeQuery(sql); while (rs.next()) { json = rs.getString(1); xls.putData(json); } rs.close(); db.close(); } catch (PSQLException e) { } catch (ParseException e) { } } System.out.println("Write Buffer"); xls.writeBuffer(); cal.add(Calendar.DAY_OF_MONTH, 1); } File file = new File( "C:\\Users\\SANIX_CORERD\\Desktop\\" + id + "-Diamond" + toDate(cal).replace("/", "-") + ".xlsx"); xls.putFile(new FileOutputStream(file)); System.out.println("Finished: " + toDate(cal)); }
From source file:model.data.location.FileAccessFactory.java
License:Apache License
/** * Gets the input stream for an S3 file store. This will stream the bytes from S3. Null, or exception will be thrown * if an error occurs during acquisition. * /*from w ww . j ava 2 s . c o m*/ * The S3 Credentials MUST be populated using the setCredentials() method before executing this call, or a * Credentials exception is likely to be thrown by S3. */ @JsonIgnore public InputStream getS3File(FileLocation fileLocation, String accessKey, String privateKey, String s3EncryptKey) { // Get the file from S3. Connect to S3 Bucket. Only apply credentials if they are present. final AmazonS3Client s3Client; S3FileStore fileStore = (S3FileStore) fileLocation; if (accessKey.isEmpty() || privateKey.isEmpty()) { s3Client = new AmazonS3Client(); } else { // If an encryption key was provided, use the encrypted client BasicAWSCredentials credentials = new BasicAWSCredentials(accessKey, privateKey); if (s3EncryptKey != null) { KMSEncryptionMaterialsProvider materialProvider = new KMSEncryptionMaterialsProvider(s3EncryptKey); s3Client = new AmazonS3EncryptionClient(credentials, materialProvider, new CryptoConfiguration().withKmsRegion(Regions.US_EAST_1)) .withRegion(Region.getRegion(Regions.US_EAST_1)); } else { s3Client = new AmazonS3Client(credentials); } } S3Object s3Object = s3Client.getObject(fileStore.getBucketName(), fileStore.getFileName()); return s3Object.getObjectContent(); }