List of usage examples for com.amazonaws.regions Region getRegion
public static Region getRegion(Regions region)
From source file:jp.co.hde.mail.ses.AmazonSESSample.java
License:Apache License
public static void main(String[] args) throws IOException { // Construct an object to contain the recipient address. Destination destination = new Destination().withToAddresses(new String[] { TO }); // Create the subject and body of the message. Content subject = new Content().withData(SUBJECT); Content textBody = new Content().withData(TEXT_BODY); Content htmlBody = new Content().withData(HTML_BODY); Body body = new Body().withHtml(htmlBody); // Create a message with the specified subject and body. Message message = new Message().withSubject(subject).withBody(body); com.amazonaws.services.simpleemail.model. // Assemble the email. SendEmailRequest request = new SendEmailRequest().withSource(FROM).withDestination(destination) .withMessage(message); try {//from w w w . j a v a2s . co m System.out.println("Attempting to send an email through Amazon SES by using the AWS SDK for Java..."); // Instantiate an Amazon SES client, which will make the service call. The service call requires your AWS credentials. // Because we're not providing an argument when instantiating the client, the SDK will attempt to find your AWS credentials // using the default credential provider chain. The first place the chain looks for the credentials is in environment variables // AWS_ACCESS_KEY_ID and AWS_SECRET_KEY. // For more information, see http://docs.aws.amazon.com/AWSSdkDocsJava/latest/DeveloperGuide/credentials.html AmazonSimpleEmailServiceClient client = new AmazonSimpleEmailServiceClient(); // Choose the AWS region of the Amazon SES endpoint you want to connect to. Note that your sandbox // status, sending limits, and Amazon SES identity-related settings are specific to a given AWS // region, so be sure to select an AWS region in which you set up Amazon SES. Here, we are using // the region. Examples of other regions that Amazon SES supports are US_EAST_1 // and EU_WEST_1. For a complete list, see http://docs.aws.amazon.com/ses/latest/DeveloperGuide/regions.html Region REGION = Region.getRegion(Regions.US_EAST_1); client.setRegion(REGION); // Send the email. client.sendEmail(request); System.out.println("Email sent!"); } catch (Exception ex) { System.out.println("The email was not sent."); System.out.println("Error message: " + ex.getMessage()); } }
From source file:jp.sanix.analysis.java
License:Open Source License
public static void main(String[] args) throws IOException, SQLException, AmazonServiceException, JSONException, NullPointerException, ParseException { if (args.length < 1) { System.out.println("Please specify at least an arguments that is pvs_serial_id."); System.out.println("analysis A9990004 // get today's data"); System.out.println("analysis A9990004 2015/05/30 // get from 2015/05/30 to today's data"); System.out.println("analysis A9990004 2015/05/30 2015/06/30"); System.exit(-1);/*from w w w . j av a2 s. co m*/ } String id = args[0]; SimpleDateFormat format = new SimpleDateFormat("yyyy/MM/dd"); format.setTimeZone(TimeZone.getTimeZone("JST")); SimpleDateFormat normalformat = new SimpleDateFormat("yyyy/MM/dd'T'HH:mm:ss'Z'"); normalformat.setTimeZone(TimeZone.getTimeZone("JST")); SimpleDateFormat pgformat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); pgformat.setTimeZone(TimeZone.getTimeZone("UTC")); Calendar cal = Calendar.getInstance(); Calendar end = Calendar.getInstance(); String today = toDate(cal); System.out.println(id); try { cal.setTime(format.parse(args[1])); } catch (ParseException e) { } catch (ArrayIndexOutOfBoundsException e) { } try { end.setTime(format.parse(args[2])); } catch (ParseException e) { } catch (ArrayIndexOutOfBoundsException e) { } end.add(Calendar.DAY_OF_MONTH, 1); AmazonS3 s3 = new AmazonS3Client(); s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1)); Connection db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS); Statement st = db.createStatement(); ResultSet rs = st.executeQuery( "SELECT data, pvs_unique_code FROM data WHERE pvs_serial_id='" + id + "' OFFSET 0 LIMIT 1;"); rs.next(); String json = rs.getString(1); String key = rs.getString(2); rs.close(); db.close(); Date recent = new Date(); while (cal.before(end)) { xlsSheet xls = new xlsSheet(json); System.out.println("Getting data of " + toDate(cal)); /* AWS S3????? */ String bucketName = "pvdata-storage-production"; // String bucketName = "pvdata-storage-staging"; System.out.println("Get s3 data by key='" + bucketName + "/data/" + key + "/" + toDate(cal) + "/'"); ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName) .withPrefix("data/" + key + "/" + toDate(cal) + "/")); /* get data from s3 */ do { for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { String keyname = objectSummary.getKey(); S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname)); BufferedReader reader = new BufferedReader( new InputStreamReader(new GZIPInputStream(object.getObjectContent()))); while (reader.ready()) { String line = reader.readLine(); try { json = line.substring(line.indexOf("{")); recent = parseDate(line.substring(0, 25)); xls.putData(json); } catch (NullPointerException e) { } } reader.close(); object.close(); } objectListing = s3.listNextBatchOfObjects(objectListing); } while (objectListing.getMarker() != null); /* if today, read postgres to get recent data */ if (toDate(cal).equals(today)) { System.out.println("Get recent data from postgres"); try { db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS); st = db.createStatement(); String sql = "SELECT data FROM data WHERE pvs_unique_code='" + key + "' AND created_at > '" + pgformat.format(recent) + "';"; System.out.println(sql); rs = st.executeQuery(sql); while (rs.next()) { json = rs.getString(1); xls.putData(json); } rs.close(); db.close(); } catch (PSQLException e) { } catch (ParseException e) { } } System.out.println("Write Buffer"); xls.writeBuffer(); File file = File.createTempFile("temp", ".xlsx"); file.deleteOnExit(); xls.putFile(new FileOutputStream(file)); System.out.println("Put S3"); s3.putObject(new PutObjectRequest("sanix-data-analysis", FOLDER + id + "-" + toDate(cal).replace("/", "-") + ".xlsx", file)); System.out.println("Finished: " + toDate(cal)); cal.add(Calendar.DAY_OF_MONTH, 1); } File file = File.createTempFile("temp", ".html"); file.deleteOnExit(); BufferedWriter bw = new BufferedWriter(new FileWriter(file)); ObjectListing objectListing = s3 .listObjects(new ListObjectsRequest().withBucketName("sanix-data-analysis").withPrefix(FOLDER)); bw.write("<html><head></head><body><ul>\n"); do { for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { String keyname = objectSummary.getKey(); if (Pattern.compile("\\.xlsx$|\\.html$").matcher(keyname).find()) { bw.write("<li><a href=\"https://s3-ap-northeast-1.amazonaws.com/sanix-data-analysis/" + keyname + "\">" + keyname.replaceAll("^[^\\/]*\\/", "") + "</a></li>\n"); } } objectListing = s3.listNextBatchOfObjects(objectListing); } while (objectListing.getMarker() != null); bw.write("</ul></body></html>\n"); bw.flush(); bw.close(); s3.putObject(new PutObjectRequest("sanix-data-analysis", FOLDER + "index.html", file)); }
From source file:jp.sanix.analysisLocal.java
License:Open Source License
public static void main(String[] args) throws IOException, SQLException, AmazonServiceException, JSONException, NullPointerException, ParseException { BufferedReader br = new BufferedReader( new InputStreamReader(new FileInputStream(new File("canceled.csv")), "SJIS")); br.readLine();/* www .java 2 s . c o m*/ String l; while ((l = br.readLine()) != null) { String[] col = l.split(","); String id = col[0]; String datefrom = col[1]; String dateto = col[1]; SimpleDateFormat format = new SimpleDateFormat("yyyy/MM/dd"); format.setTimeZone(TimeZone.getTimeZone("JST")); SimpleDateFormat pgformat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); pgformat.setTimeZone(TimeZone.getTimeZone("UTC")); Calendar cal = Calendar.getInstance(); Calendar end = Calendar.getInstance(); String today = toDate(cal); try { cal.setTime(format.parse(datefrom)); } catch (ParseException e) { } try { end.setTime(format.parse(dateto)); end.add(Calendar.DAY_OF_MONTH, 1); } catch (ParseException e) { } AmazonS3 s3 = new AmazonS3Client(); s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1)); Connection db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS); Statement st = db.createStatement(); ResultSet rs = st.executeQuery( "SELECT data, pvs_unique_code FROM data WHERE pvs_serial_id='" + id + "' OFFSET 0 LIMIT 1;"); rs.next(); String json = rs.getString(1); String key = rs.getString(2); rs.close(); db.close(); Date recent = new Date(); while (cal.before(end)) { xlsSheet xls = new xlsSheet(json); System.out.println("Getting data of " + toDate(cal)); /* AWS S3????? */ String bucketName = "pvdata-storage-production"; System.out.println("Get s3 data by key='" + bucketName + "/data/" + key + "/" + toDate(cal) + "/'"); ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName) .withPrefix("data/" + key + "/" + toDate(cal) + "/")); /* get data from s3 */ do { for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { String keyname = objectSummary.getKey(); S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname)); BufferedReader reader = new BufferedReader( new InputStreamReader(new GZIPInputStream(object.getObjectContent()))); while (reader.ready()) { String line = reader.readLine(); try { json = line.substring(line.indexOf("{")); recent = parseDate(line.substring(0, 25)); xls.putData(json); } catch (NullPointerException e) { } } reader.close(); object.close(); } objectListing = s3.listNextBatchOfObjects(objectListing); } while (objectListing.getMarker() != null); /* if today, read postgres to get recent data */ if (toDate(cal).equals(today)) { System.out.println("Get recent data from postgres"); try { db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS); st = db.createStatement(); String sql = "SELECT data FROM data WHERE pvs_unique_code='" + key + "' AND created_at > '" + pgformat.format(recent) + "';"; System.out.println(sql); rs = st.executeQuery(sql); while (rs.next()) { json = rs.getString(1); xls.putData(json); } rs.close(); db.close(); } catch (PSQLException e) { } catch (ParseException e) { } } System.out.println("Write Buffer"); xls.writeBuffer(); File file = new File( "C:\\Users\\SANIX_CORERD\\Desktop\\" + id + "_" + toDate(cal).replace("/", "-") + ".xlsx"); xls.putFile(new FileOutputStream(file)); System.out.println("Finished: " + toDate(cal)); cal.add(Calendar.DAY_OF_MONTH, 1); } } br.close(); }
From source file:jp.sanix.analysisStaging.java
License:Open Source License
public static void main(String[] args) throws IOException, SQLException, AmazonServiceException, JSONException, NullPointerException, ParseException { // if (args.length < 1) { // System.out.println("Please specify at least an arguments that is pvs_serial_id."); // System.out.println("analysis A9990004 // get today's data"); // System.out.println("analysis A9990004 2015/05/30 // get from 2015/05/30 to today's data"); // System.out.println("analysis A9990004 2015/05/30 2015/06/30"); // System.exit(-1); // }//from w w w .j a v a 2 s .co m // String id = args[0]; String uniq = "763ff280b16c61f524ae31b33401bcd6"; SimpleDateFormat format = new SimpleDateFormat("yyyy/MM/dd"); format.setTimeZone(TimeZone.getTimeZone("JST")); SimpleDateFormat normalformat = new SimpleDateFormat("yyyy/MM/dd'T'HH:mm:ss'Z'"); normalformat.setTimeZone(TimeZone.getTimeZone("JST")); SimpleDateFormat pgformat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); pgformat.setTimeZone(TimeZone.getTimeZone("UTC")); Calendar cal = Calendar.getInstance(); Calendar end = Calendar.getInstance(); cal.setTime(format.parse("2015/11/23")); end.setTime(format.parse("2015/11/23")); System.out.println(uniq); try { cal.setTime(format.parse(args[1])); } catch (ParseException e) { } catch (ArrayIndexOutOfBoundsException e) { } try { end.setTime(format.parse(args[2])); } catch (ParseException e) { } catch (ArrayIndexOutOfBoundsException e) { } end.add(Calendar.DAY_OF_MONTH, 1); AmazonS3 s3 = new AmazonS3Client(); s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1)); String bucketName = "pvdata-storage-staging"; while (cal.before(end)) { System.out.println("Getting data of " + toDate(cal)); /* AWS S3????? */ // String bucketName = "pvdata-storage-production"; System.out.println("Get s3 data by key='" + bucketName + "/data/" + uniq + "/" + toDate(cal) + "/'"); ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName) .withPrefix("data/" + uniq + "/" + toDate(cal) + "/")); /* get data from s3 */ xlsSheet xls = null; do { Boolean first = true; for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { String keyname = objectSummary.getKey(); S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname)); BufferedReader reader = new BufferedReader( new InputStreamReader(new GZIPInputStream(object.getObjectContent()))); while (reader.ready()) { String line = reader.readLine(); try { String json = line.substring(line.indexOf("{")); if (first) { xls = new xlsSheet(json); first = false; } else { xls.putData(json); } } catch (NullPointerException e) { } } reader.close(); object.close(); } objectListing = s3.listNextBatchOfObjects(objectListing); } while (objectListing.getMarker() != null); System.out.println("Write Buffer"); xls.writeBuffer(); File file = File.createTempFile("temp", ".xlsx"); file.deleteOnExit(); xls.putFile(new FileOutputStream(file)); System.out.println("Put S3"); s3.putObject(new PutObjectRequest("sanix-data-analysis", FOLDER + uniq + "-" + toDate(cal).replace("/", "-") + ".xlsx", file)); System.out.println("Finished: " + toDate(cal)); cal.add(Calendar.DAY_OF_MONTH, 1); } }
From source file:jp.sanix.mukaino.java
License:Open Source License
public static void main(String[] args) throws IOException, SQLException, AmazonServiceException, JSONException, NullPointerException, ParseException { AmazonS3 s3 = new AmazonS3Client(); s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1)); String[] ids = { "A0000089", "A0002570" }; Connection db;/*from w w w .j av a 2 s . c om*/ if (System.getProperty("user.name").toString().equals("ec2-user")) { db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS); } else { db = DriverManager.getConnection(PG_CON_LOCAL, PG_USER, PG_PASS); } for (String id : ids) { pvSensor.getInstance().getById(id); Statement st = db.createStatement(); ResultSet rs = st.executeQuery("SELECT data, pvs_unique_code FROM data WHERE pvs_serial_id='" + id + "' ORDER BY created_at DESC OFFSET 0 LIMIT 1;"); rs.next(); String json = rs.getString(1); String key = rs.getString(2); rs.close(); st.close(); Calendar cal = Calendar.getInstance(); cal.add(Calendar.DAY_OF_MONTH, -1); // 1?? Calendar end = Calendar.getInstance(); cal.add(Calendar.DAY_OF_MONTH, -1); // 1??? do { String mDate = String.format("%04d-%02d-%02d", cal.get(Calendar.YEAR), cal.get(Calendar.MONTH) + 1, cal.get(Calendar.DAY_OF_MONTH)); xlsSheetMukaino xls = new xlsSheetMukaino(json); /* AWS S3????? */ String bucketName = "pvdata-storage-production"; System.out.print("Get s3 data by key='" + bucketName + String.format("data/%s/%04d/%02d/%02d/", key, cal.get(Calendar.YEAR), cal.get(Calendar.MONTH) + 1, cal.get(Calendar.DAY_OF_MONTH))); ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName) .withPrefix("data/" + key + "/" + mDate.replace("-", "/"))); /* get data from s3 */ do { for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { String keyname = objectSummary.getKey(); S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname)); BufferedReader reader = new BufferedReader( new InputStreamReader(new GZIPInputStream(object.getObjectContent()))); while (reader.ready()) { String line = reader.readLine(); try { json = line.substring(line.indexOf("{")); xls.putData(json); } catch (NullPointerException e) { } } reader.close(); object.close(); System.out.print("."); } objectListing = s3.listNextBatchOfObjects(objectListing); } while (objectListing.getMarker() != null); System.out.println(); xls.writeBuffer(); File file = File.createTempFile("temp", ".xlsx"); file.deleteOnExit(); xls.putFile(new FileOutputStream(file)); s3.putObject(new PutObjectRequest("sanix-data-analysis", "Rw6WHQLqGcvs3wpWwALh80IHAiZqWYzw/" + id + "-" + mDate + ".xlsx", file)); System.out.println("Finished: " + mDate); cal.add(Calendar.DAY_OF_MONTH, 1); } while (cal.before(end)); } db.close(); File file = File.createTempFile("temp", ".html"); file.deleteOnExit(); BufferedWriter bw = new BufferedWriter(new FileWriter(file)); ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName("sanix-data-analysis") .withPrefix("Rw6WHQLqGcvs3wpWwALh80IHAiZqWYzw/")); bw.write("<html><head></head><body><ul>\n"); do { for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { String keyname = objectSummary.getKey(); if (Pattern.compile("\\.xlsx$|\\.html$").matcher(keyname).find()) { bw.write("<li><a href=\"https://s3-ap-northeast-1.amazonaws.com/sanix-data-analysis/" + keyname + "\">" + keyname.replaceAll("^[^\\/]*\\/", "") + "</a></li>\n"); } } objectListing = s3.listNextBatchOfObjects(objectListing); } while (objectListing.getMarker() != null); bw.write("</ul></body></html>\n"); bw.flush(); bw.close(); s3.putObject( new PutObjectRequest("sanix-data-analysis", "Rw6WHQLqGcvs3wpWwALh80IHAiZqWYzw/index.html", file)); }
From source file:jp.sanix.Rawdata.java
License:Open Source License
public TreeMap<Date, JSONObject> get() throws IOException, SQLException, ParseException { TreeMap<Date, JSONObject> data = new TreeMap<Date, JSONObject>(); AmazonS3 s3 = new AmazonS3Client(); s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1)); SimpleDateFormat fmt = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); fmt.setTimeZone(TimeZone.getTimeZone("UTC")); Calendar cal = Calendar.getInstance(); cal.setTime(this.getDate()); /* AWS S3????? */ String bucketName = "pvdata-storage-production"; ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName) .withPrefix("data/" + this.getUniqueCode() + "/" + toDate(cal) + "/")); /* get data from s3 */ do {//from ww w .jav a2 s . co m for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { String keyname = objectSummary.getKey(); S3Object object = s3.getObject(new GetObjectRequest(BUCKET, keyname)); BufferedReader reader = new BufferedReader( new InputStreamReader(new GZIPInputStream(object.getObjectContent()))); while (reader.ready()) { String line = reader.readLine(); int pos; try { pos = line.indexOf("{"); } catch (NullPointerException e) { continue; } try { String jsons = line.substring(pos); JSONObject json = new JSONObject(jsons); JSONArray arr = json.getJSONObject("data").getJSONArray("sample"); for (int i = 0; i < arr.length(); i++) { JSONObject obj = arr.getJSONObject(i); Date date = fmt.parse(obj.getString("time")); for (int j = 0; j < obj.getJSONArray("powcon").length(); j++) { if (obj.getJSONArray("powcon").getJSONObject(j).get("genKwh") != JSONObject.NULL) { Double genkwh; genkwh = obj.getJSONArray("powcon").getJSONObject(j).getDouble("genKwh"); JSONObject jobj; if (data.containsKey(date)) { jobj = data.get(date); } else { jobj = new JSONObject(); } if (genkwh != null) { jobj.put(String.valueOf(j), genkwh); } if (jobj.length() > 0) { data.put(date, jobj); } } } } } catch (JSONException e) { e.printStackTrace(); } catch (StringIndexOutOfBoundsException e) { e.printStackTrace(); } } reader.close(); object.close(); } objectListing = s3.listNextBatchOfObjects(objectListing); } while (objectListing.getMarker() != null); Calendar today = Calendar.getInstance(); today.setTimeZone(TimeZone.getTimeZone("JST")); if (toDate(cal).equals(toDate(today))) { SimpleDateFormat pgfmt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); pgfmt.setTimeZone(TimeZone.getTimeZone("UTC")); Calendar recent = Calendar.getInstance(); recent.setTimeZone(TimeZone.getTimeZone("UST")); recent.add(Calendar.HOUR, -2); Connection db; if (System.getProperty("user.name").toString().equals("ec2-user")) { db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS); } else { db = DriverManager.getConnection(PG_CON_LOCAL, PG_USER, PG_PASS); } Statement st = db.createStatement(); String sql = "SELECT data FROM data WHERE pvs_unique_code='" + this.getUniqueCode() + "' AND created_at > '" + pgfmt.format(recent.getTime()) + "';"; ResultSet rs = st.executeQuery(sql); while (rs.next()) { String jsons = rs.getString(1); try { JSONObject json = new JSONObject(jsons); JSONArray arr = json.getJSONObject("data").getJSONArray("sample"); for (int i = 0; i < arr.length(); i++) { JSONObject obj = arr.getJSONObject(i); Date date = fmt.parse(obj.getString("time")); for (int j = 0; j < obj.getJSONArray("powcon").length(); j++) { if (obj.getJSONArray("powcon").getJSONObject(j).get("genKwh") != JSONObject.NULL) { Double genkwh; genkwh = obj.getJSONArray("powcon").getJSONObject(j).getDouble("genKwh"); JSONObject jobj; if (data.containsKey(date)) { jobj = data.get(date); } else { jobj = new JSONObject(); } if (genkwh != null) { jobj.put(String.valueOf(j), genkwh); } if (jobj.length() > 0) { data.put(date, jobj); } } } } } catch (JSONException e) { e.printStackTrace(); } } rs.close(); db.close(); } return data; }
From source file:jp.sanix.weatherData.java
License:Open Source License
public static void main(String[] args) throws IOException, SQLException, AmazonServiceException, JSONException, NullPointerException, ParseException { AmazonS3 s3 = new AmazonS3Client(); s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1)); BufferedReader br = new BufferedReader(new InputStreamReader( new FileInputStream(new File("src/main/web/?.csv")), "SJIS")); String line = null;// w w w . java2s .com br.readLine(); // 1??? while ((line = br.readLine()) != null) { String[] col = line.split(","); /* AWS S3????? */ String bucketName = "weather-forecast"; ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName) .withPrefix("day_insolation/" + col[0] + "/")); File file = File.createTempFile("temp", ".csv"); file.deleteOnExit(); BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file), "SJIS")); bw.write( ",?,?,???,,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23\n"); System.out.println(col[0] + ":" + col[1] + col[2]); /* get data from s3 */ int i = 0; do { for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { String keyname = objectSummary.getKey(); S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname)); BufferedReader reader = new BufferedReader(new InputStreamReader(object.getObjectContent())); StringBuilder sb = new StringBuilder(); String line2 = null; System.out.print(String.valueOf(i++) + "\r"); while ((line2 = reader.readLine()) != null) { sb.append(line2); } reader.close(); object.close(); try { JSONObject json = new JSONObject(sb.toString()); bw.write(String.join(",", col) + "," + json.get("date").toString().replace("-", "/") + ","); JSONArray jarr = json.getJSONArray("hour_data"); bw.write(jarr.join(",") + "\n"); // System.out.println(String.join(",", col) + "," + json.get("date").toString().replace("-", "/") + "," + jarr.join(",")); } catch (JSONException e) { // System.exit(1); } } objectListing = s3.listNextBatchOfObjects(objectListing); } while (objectListing.getMarker() != null); bw.flush(); bw.close(); if (i > 0) { s3.putObject(new PutObjectRequest("sanix-data-analysis", STORE_PATH + col[1] + col[2] + "_insolation.csv", file)); } } br.close(); br = new BufferedReader(new InputStreamReader( new FileInputStream(new File("src/main/web/?.csv")), "SJIS")); br.readLine(); // 1??? while ((line = br.readLine()) != null) { String[] col = line.split(","); /* AWS S3????? */ String bucketName = "weather-forecast"; ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName) .withPrefix("day_temperature/" + col[0] + "/")); File file = File.createTempFile("temp", ".csv"); file.deleteOnExit(); BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file), "SJIS")); bw.write( ",?,?,???,,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23\n"); System.out.println(col[0] + ":" + col[1] + col[2]); /* get data from s3 */ int i = 0; do { for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { String keyname = objectSummary.getKey(); S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname)); BufferedReader reader = new BufferedReader(new InputStreamReader(object.getObjectContent())); StringBuilder sb = new StringBuilder(); String line2 = null; System.out.print(String.valueOf(i++) + "\r"); while ((line2 = reader.readLine()) != null) { sb.append(line2); } reader.close(); object.close(); try { JSONObject json = new JSONObject(sb.toString()); bw.write(String.join(",", col) + "," + json.get("date").toString().replace("-", "/") + ","); JSONArray jarr = json.getJSONArray("hour_data"); bw.write(jarr.join(",") + "\n"); // System.out.println(String.join(",", col) + "," + json.get("date").toString().replace("-", "/") + "," + jarr.join(",")); } catch (JSONException e) { // System.exit(1); } } objectListing = s3.listNextBatchOfObjects(objectListing); } while (objectListing.getMarker() != null); bw.flush(); bw.close(); if (i > 0) { s3.putObject(new PutObjectRequest("sanix-data-analysis", STORE_PATH + col[1] + col[2] + "_temperture.csv", file)); } } br.close(); }
From source file:jp.sanix.yokusei.java
License:Open Source License
public static void main(String[] args) throws IOException, SQLException, AmazonServiceException, JSONException, NullPointerException, ParseException { String id = "A0002441"; String datefrom = "2015/10/01"; String dateto = "2015/10/12"; SimpleDateFormat format = new SimpleDateFormat("yyyy/MM/dd"); format.setTimeZone(TimeZone.getTimeZone("JST")); SimpleDateFormat pgformat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); pgformat.setTimeZone(TimeZone.getTimeZone("UTC")); Calendar cal = Calendar.getInstance(); Calendar end = Calendar.getInstance(); String today = toDate(cal);//w ww . j a v a2 s .c o m try { cal.setTime(format.parse(datefrom)); } catch (ParseException e) { } try { end.setTime(format.parse(dateto)); end.add(Calendar.DAY_OF_MONTH, 1); } catch (ParseException e) { } AmazonS3 s3 = new AmazonS3Client(); s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1)); Connection db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS); Statement st = db.createStatement(); ResultSet rs = st.executeQuery( "SELECT data, pvs_unique_code FROM data WHERE pvs_serial_id='" + id + "' OFFSET 0 LIMIT 1;"); rs.next(); String json = rs.getString(1); String key = rs.getString(2); rs.close(); db.close(); Date recent = new Date(); xlsSheetYokusei xls = new xlsSheetYokusei(json); while (cal.before(end)) { System.out.println("Getting data of " + toDate(cal)); /* AWS S3????? */ String bucketName = "pvdata-storage-production"; System.out.println("Get s3 data by key='" + bucketName + "/data/" + key + "/" + toDate(cal) + "/'"); ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName) .withPrefix("data/" + key + "/" + toDate(cal) + "/")); /* get data from s3 */ do { for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { String keyname = objectSummary.getKey(); S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname)); BufferedReader reader = new BufferedReader( new InputStreamReader(new GZIPInputStream(object.getObjectContent()))); String line; while ((line = reader.readLine()) != null) { try { json = line.substring(line.indexOf("{")); xls.putData(json); } catch (NullPointerException e) { } } reader.close(); object.close(); } objectListing = s3.listNextBatchOfObjects(objectListing); } while (objectListing.getMarker() != null); /* if today, read postgres to get recent data */ if (toDate(cal).equals(today)) { System.out.println("Get recent data from postgres"); try { db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS); st = db.createStatement(); String sql = "SELECT data FROM data WHERE pvs_unique_code='" + key + "' AND created_at > '" + pgformat.format(recent) + "';"; System.out.println(sql); rs = st.executeQuery(sql); while (rs.next()) { json = rs.getString(1); xls.putData(json); } rs.close(); db.close(); } catch (PSQLException e) { } catch (ParseException e) { } } System.out.println("Write Buffer"); xls.writeBuffer(); cal.add(Calendar.DAY_OF_MONTH, 1); } File file = new File( "C:\\Users\\SANIX_CORERD\\Desktop\\" + id + "-Diamond" + toDate(cal).replace("/", "-") + ".xlsx"); xls.putFile(new FileOutputStream(file)); System.out.println("Finished: " + toDate(cal)); }
From source file:kinesisadaptersample.StreamsAdapterDemo.java
License:Open Source License
/** * @param args/*from w w w.ja v a 2 s. c om*/ */ public static void main(String[] args) throws Exception { System.out.println("Starting demo..."); String srcTable = tablePrefix + "-src"; String destTable = tablePrefix + "-dest"; streamsCredentials = new ProfileCredentialsProvider(); dynamoDBCredentials = new ProfileCredentialsProvider(); recordProcessorFactory = new StreamsRecordProcessorFactory(destTable); adapterClient = new AmazonDynamoDBStreamsAdapterClient(streamsCredentials); adapterClient.setRegion(Region.getRegion(Regions.EU_WEST_1)); dynamoDBClient = AmazonDynamoDBClientBuilder.standard().withRegion(Regions.EU_WEST_1).build(); cloudWatchClient = AmazonCloudWatchClientBuilder.standard().withRegion(Regions.EU_WEST_1).build(); setUpTables(); workerConfig = new KinesisClientLibConfiguration("streams-adapter-demo", streamArn, streamsCredentials, "streams-demo-worker").withMaxRecords(1000).withIdleTimeBetweenReadsInMillis(500) .withInitialPositionInStream(InitialPositionInStream.TRIM_HORIZON); System.out.println("Creating worker for stream: " + streamArn); worker = new Worker(recordProcessorFactory, workerConfig, adapterClient, dynamoDBClient, cloudWatchClient); System.out.println("Starting worker..."); Thread t = new Thread(worker); t.start(); Thread.sleep(25000); worker.shutdown(); t.join(); if (StreamsAdapterDemoHelper.scanTable(dynamoDBClient, srcTable).getItems() .equals(StreamsAdapterDemoHelper.scanTable(dynamoDBClient, destTable).getItems())) { System.out.println("Scan result is equal."); } else { System.out.println("Tables are different!"); } System.out.println("Done."); cleanupAndExit(0); }
From source file:lumbermill.internal.aws.KinesisClientFactory.java
License:Apache License
private AmazonKinesisAsync createClient(ClientConfiguration config, MapWrap configuration) { AmazonKinesisAsync kinesisClient = new AmazonKinesisAsyncClient( getAwsCredentialsProvider(configuration, config), config/*, Executors.newFixedThreadPool(config.getMaxConnections(), new ThreadFactoryBuilder().setNameFormat("lumbermill-async-kinesis-%d").build())*/); Regions region = Regions.fromName(configuration.asString("region", "eu-west-1")); kinesisClient.setRegion(Region.getRegion(region)); if (configuration.exists("endpoint")) { kinesisClient.setEndpoint(configuration.asString("endpoint")); }/*w ww .jav a2 s . c om*/ return kinesisClient; }