List of usage examples for com.amazonaws.services.s3 AmazonS3 setRegion
public void setRegion(com.amazonaws.regions.Region region) throws IllegalArgumentException;
From source file:fsi_admin.JAwsS3Conn.java
License:Open Source License
@SuppressWarnings({ "rawtypes", "unchecked" })
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
String ERROR = null, codErr = null;
try {/* ww w . j av a 2 s. c o m*/
Properties parametros = new Properties();
Vector archivos = new Vector();
DiskFileUpload fu = new DiskFileUpload();
List items = fu.parseRequest(request);
Iterator iter = items.iterator();
while (iter.hasNext()) {
FileItem item = (FileItem) iter.next();
if (item.isFormField())
parametros.put(item.getFieldName(), item.getString());
else
archivos.addElement(item);
}
if (parametros.getProperty("SERVER") == null || parametros.getProperty("DATABASE") == null
|| parametros.getProperty("USER") == null || parametros.getProperty("PASSWORD") == null
|| parametros.getProperty("ACTION") == null) {
System.out.println("No recibi parametros de conexin antes del archivo");
ERROR = "ERROR: El servidor no recibi todos los parametros de conexion (SERVER,DATABASE,USER,PASSWORD,ACTION) antes del archivo";
codErr = "3";
ingresarRegistroFallido(request.getRemoteAddr(), request.getRemoteHost(),
parametros.getProperty("SERVER"), parametros.getProperty("USER"),
parametros.getProperty("PASSWORD"), request, ERROR, 3);
}
//Hasta aqui se han enviado todos los parametros ninguno nulo
if (ERROR == null) {
StringBuffer msj = new StringBuffer(), S3BUKT = new StringBuffer(), S3USR = new StringBuffer(),
S3PASS = new StringBuffer();
MutableBoolean COBRAR = new MutableBoolean(false);
MutableDouble COSTO = new MutableDouble(0.0), SALDO = new MutableDouble(0.0);
// Primero obtiene info del S3
if (!obtenInfoAWSS3(request.getRemoteAddr(), request.getRemoteHost(),
parametros.getProperty("SERVER"), parametros.getProperty("DATABASE"),
parametros.getProperty("USER"), parametros.getProperty("PASSWORD"), S3BUKT, S3USR, S3PASS,
msj, COSTO, SALDO, COBRAR)) {
System.out.println("El usuario y contrasea de servicio estan mal");
ERROR = msj.toString();
codErr = "2";
ingresarRegistroFallido(request.getRemoteAddr(), request.getRemoteHost(),
parametros.getProperty("SERVER"), parametros.getProperty("USER"),
parametros.getProperty("PASSWORD"), request, ERROR, 2);
} else {
AWSCredentials credentials = new BasicAWSCredentials(S3USR.toString(), S3PASS.toString());
AmazonS3 s3 = new AmazonS3Client(credentials);
Region usWest2 = Region.getRegion(Regions.US_WEST_2);
s3.setRegion(usWest2);
//System.out.println("AwsConn:" + parametros.getProperty("NOMBRE") + ":parametros.getProperty(NOMBRE)");
String nombre = parametros.getProperty("SERVER") + parametros.getProperty("DATABASE")
+ parametros.getProperty("ID_MODULO") + parametros.getProperty("OBJIDS")
+ parametros.getProperty("IDSEP") + parametros.getProperty("NOMBRE");
//System.out.println("AwsConn_Nombre:" + nombre + ":nombre");
if (parametros.getProperty("ACTION").equals("SUBIR")) {
Double TOTBITES = new Double(Double.parseDouble(parametros.getProperty("TOTBITES")));
Double TAMBITES = new Double(Double.parseDouble(parametros.getProperty("TAMBITES")));
if (COBRAR.booleanValue() && SALDO
.doubleValue() < (COSTO.doubleValue() * (((TOTBITES + TAMBITES) / 1024) / 1024))) {
System.out
.println("El servicio S3 de subida tiene un costo que no alcanza en el saldo");
ERROR = "El servicio S3 de subida tiene un costo que no alcanza en el saldo";
codErr = "2";
ingresarRegistroFallido(request.getRemoteAddr(), request.getRemoteHost(),
parametros.getProperty("SERVER"), parametros.getProperty("USER"),
parametros.getProperty("PASSWORD"), request, ERROR, 2);
} else {
if (!subirArchivo(msj, s3, S3BUKT.toString(), nombre, archivos)) {
System.out.println("No se permiti subir el archivo al s3");
ERROR = msj.toString();
codErr = "3";
ingresarRegistroFallido(request.getRemoteAddr(), request.getRemoteHost(),
parametros.getProperty("SERVER"), parametros.getProperty("USER"),
parametros.getProperty("PASSWORD"), request, ERROR, 3);
} else {
ingresarRegistroExitoso(parametros.getProperty("SERVER"),
parametros.getProperty("DATABASE"), parametros.getProperty("ID_MODULO"),
parametros.getProperty("OBJIDS"), parametros.getProperty("IDSEP"),
parametros.getProperty("NOMBRE"), parametros.getProperty("TAMBITES"));
}
}
} else if (parametros.getProperty("ACTION").equals("ELIMINAR")) {
Double TOTBITES = new Double(Double.parseDouble(parametros.getProperty("TOTBITES")));
if (COBRAR.booleanValue()
&& SALDO.doubleValue() < (COSTO.doubleValue() * ((TOTBITES / 1024) / 1024))) {
System.out
.println("El servicio S3 de borrado tiene un costo que no alcanza en el saldo");
ERROR = "El servicio S3 de borrado tiene un costo que no alcanza en el saldo";
codErr = "2";
ingresarRegistroFallido(request.getRemoteAddr(), request.getRemoteHost(),
parametros.getProperty("SERVER"), parametros.getProperty("USER"),
parametros.getProperty("PASSWORD"), request, ERROR, 2);
} else {
if (!eliminarArchivo(msj, s3, S3BUKT.toString(), nombre)) {
System.out.println("No se permiti eliminar el archivo del s3");
ERROR = msj.toString();
codErr = "3";
ingresarRegistroFallido(request.getRemoteAddr(), request.getRemoteHost(),
parametros.getProperty("SERVER"), parametros.getProperty("USER"),
parametros.getProperty("PASSWORD"), request, ERROR, 3);
} else {
eliminarRegistroExitoso(parametros.getProperty("SERVER"),
parametros.getProperty("DATABASE"), parametros.getProperty("ID_MODULO"),
parametros.getProperty("OBJIDS"), parametros.getProperty("IDSEP"),
parametros.getProperty("NOMBRE"));
}
}
} else if (parametros.getProperty("ACTION").equals("DESCARGAR")) {
Double TOTBITES = new Double(Double.parseDouble(parametros.getProperty("TOTBITES")));
//System.out.println("COBRAR: " + COBRAR.booleanValue() + " SALDO: " + SALDO.doubleValue() + " COSTO: " + COSTO.doubleValue() + " TOTBITES: " + TOTBITES + " TOTMB: " + ((TOTBITES / 1024) / 1024) + " RES: " + (COSTO.doubleValue() * ((TOTBITES / 1024) / 1024)));
if (COBRAR.booleanValue()
&& SALDO.doubleValue() < (COSTO.doubleValue() * ((TOTBITES / 1024) / 1024))) {
System.out.println(
"El servicio S3 de descarga tiene un costo que no alcanza en el saldo");
ERROR = "El servicio S3 de descarga tiene un costo que no alcanza en el saldo";
codErr = "2";
ingresarRegistroFallido(request.getRemoteAddr(), request.getRemoteHost(),
parametros.getProperty("SERVER"), parametros.getProperty("USER"),
parametros.getProperty("PASSWORD"), request, ERROR, 2);
} else {
if (!descargarArchivo(response, msj, s3, S3BUKT.toString(), nombre,
parametros.getProperty("NOMBRE"))) {
System.out.println("No se permiti descargar el archivo del s3");
ERROR = msj.toString();
codErr = "3";
ingresarRegistroFallido(request.getRemoteAddr(), request.getRemoteHost(),
parametros.getProperty("SERVER"), parametros.getProperty("USER"),
parametros.getProperty("PASSWORD"), request, ERROR, 3);
} else
return;
}
}
if (ERROR == null) {
//Devuelve la respuesta al cliente
Element S3 = new Element("S3");
S3.setAttribute("Archivo", nombre);
S3.setAttribute("MsjError", "");
Document Reporte = new Document(S3);
Format format = Format.getPrettyFormat();
format.setEncoding("utf-8");
format.setTextMode(TextMode.NORMALIZE);
XMLOutputter xmlOutputter = new XMLOutputter(format);
ByteArrayOutputStream out = new ByteArrayOutputStream();
xmlOutputter.output(Reporte, out);
byte[] data = out.toByteArray();
ByteArrayInputStream istream = new ByteArrayInputStream(data);
String destino = "Archivo.xml";
JBajarArchivo fd = new JBajarArchivo();
fd.doDownload(response, getServletConfig().getServletContext(), istream, "text/xml",
data.length, destino);
}
}
}
} catch (Exception e) {
e.printStackTrace();
ERROR = "ERROR DE EXCEPCION EN SERVIDOR AWS S3: " + e.getMessage();
}
//Genera el archivo XML de error para ser devuelto al Servidor
if (ERROR != null) {
Element SIGN_ERROR = new Element("SIGN_ERROR");
SIGN_ERROR.setAttribute("CodError", codErr);
SIGN_ERROR.setAttribute("MsjError", ERROR);
Document Reporte = new Document(SIGN_ERROR);
Format format = Format.getPrettyFormat();
format.setEncoding("utf-8");
format.setTextMode(TextMode.NORMALIZE);
XMLOutputter xmlOutputter = new XMLOutputter(format);
ByteArrayOutputStream out = new ByteArrayOutputStream();
xmlOutputter.output(Reporte, out);
byte[] data = out.toByteArray();
ByteArrayInputStream istream = new ByteArrayInputStream(data);
String destino = "SIGN_ERROR.xml";
JBajarArchivo fd = new JBajarArchivo();
fd.doDownload(response, getServletConfig().getServletContext(), istream, "text/xml", data.length,
destino);
}
}
From source file:gov.usgs.cida.iplover.util.ImageStorage.java
public static AmazonS3 prepS3Client() { AWSCredentials credentials = null;//www . j ava2 s . c o m try { credentials = new ProfileCredentialsProvider().getCredentials(); } catch (Exception e) { throw new AmazonClientException("Cannot load the credentials from the credential profiles file. " + "Please make sure that your credentials file is at the correct " + "location (~/.aws/credentials), and is in valid format.", e); } AmazonS3 s3 = new AmazonS3Client(credentials); Region usWest2 = Region.getRegion(Regions.US_WEST_2); s3.setRegion(usWest2); return s3; }
From source file:Java21.S3Files.java
License:Open Source License
public static void main(String[] args) throws IOException { /*// w w w. j ava2 s .co m * The ProfileCredentialsProvider will return your [default] * credential profile by reading from the credentials file located at * (~/.aws/credentials). */ AWSCredentials credentials = null; try { credentials = new ProfileCredentialsProvider().getCredentials(); } catch (Exception e) { throw new AmazonClientException("Cannot load the credentials from the credential profiles file. " + "Please make sure that your credentials file is at the correct " + "location (~/.aws/credentials), and is in valid format.", e); } AmazonS3 s3 = new AmazonS3Client(credentials); Region usWest2 = Region.getRegion(Regions.US_WEST_2); s3.setRegion(usWest2); String bucketName = "msm-gb-env-etl-iq/dev1-dwh/" + UUID.randomUUID(); //String key = ""; System.out.println("==========================================="); System.out.println("Getting Started with Amazon S3"); System.out.println("===========================================\n"); /* * List objects in your bucket by prefix - There are many options for * listing the objects in your bucket. Keep in mind that buckets with * many objects might truncate their results when listing their objects, * so be sure to check if the returned object listing is truncated, and * use the AmazonS3.listNextBatchOfObjects(...) operation to retrieve * additional results. */ System.out.println("Listing objects"); ObjectListing objectListing = s3 .listObjects(new ListObjectsRequest().withBucketName(bucketName).withPrefix("My")); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { System.out.println(" - " + objectSummary.getKey() + " " + "(size = " + objectSummary.getSize() + ")"); } System.out.println(); }
From source file:javafxawss3sample.FXMLDocumentController.java
@Override public void initialize(URL url, ResourceBundle rb) { AmazonS3 s3 = new AmazonS3Client(); Region usWest2 = Region.getRegion(Regions.US_WEST_2); s3.setRegion(usWest2); System.out.println("==========================================="); System.out.println("Getting Started with Amazon S3"); System.out.println("==========================================="); s3.listBuckets().forEach(bucket -> { // creating tab for each bucket Tab tab = new Tab(); String bucketNm = bucket.getName(); tab.setText(bucketNm);/* w w w. jav a 2 s .c om*/ tabPane.getTabs().add(tab); VBox vbox = new VBox(10); vbox.setPadding(new Insets(10)); vbox.setStyle("-fx-background-color: #ffc9ae"); // Owner HBox hbox1 = new HBox(10); Label ownerLabel = new Label("Owner:"); String owner = bucket.getOwner().getDisplayName(); Text ownerText = new Text(owner); hbox1.getChildren().addAll(ownerLabel, ownerText); // Create Date HBox hbox2 = new HBox(10); Label dateLabel = new Label("Create Date:"); String createDate = bucket.getCreationDate().toString(); Text dateText = new Text(createDate); hbox2.getChildren().addAll(dateLabel, dateText); tab.setContent(vbox); tab.setStyle("-fx-background-color: #ffc9ae;"); // ListView for file path ObservableList<String> items = FXCollections.observableArrayList("file path"); ListView view = new ListView(items); ObjectListing objList = s3 .listObjects(new ListObjectsRequest().withBucketName(bucketNm).withPrefix("")); objList.getObjectSummaries().forEach(obj -> { String filePath = obj.getKey(); items.add(filePath); }); vbox.getChildren().addAll(hbox1, hbox2, view); }); }
From source file:jp.sanix.analysis.java
License:Open Source License
public static void main(String[] args) throws IOException, SQLException, AmazonServiceException, JSONException, NullPointerException, ParseException { if (args.length < 1) { System.out.println("Please specify at least an arguments that is pvs_serial_id."); System.out.println("analysis A9990004 // get today's data"); System.out.println("analysis A9990004 2015/05/30 // get from 2015/05/30 to today's data"); System.out.println("analysis A9990004 2015/05/30 2015/06/30"); System.exit(-1);/*from w w w . ja v a 2 s. c o m*/ } String id = args[0]; SimpleDateFormat format = new SimpleDateFormat("yyyy/MM/dd"); format.setTimeZone(TimeZone.getTimeZone("JST")); SimpleDateFormat normalformat = new SimpleDateFormat("yyyy/MM/dd'T'HH:mm:ss'Z'"); normalformat.setTimeZone(TimeZone.getTimeZone("JST")); SimpleDateFormat pgformat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); pgformat.setTimeZone(TimeZone.getTimeZone("UTC")); Calendar cal = Calendar.getInstance(); Calendar end = Calendar.getInstance(); String today = toDate(cal); System.out.println(id); try { cal.setTime(format.parse(args[1])); } catch (ParseException e) { } catch (ArrayIndexOutOfBoundsException e) { } try { end.setTime(format.parse(args[2])); } catch (ParseException e) { } catch (ArrayIndexOutOfBoundsException e) { } end.add(Calendar.DAY_OF_MONTH, 1); AmazonS3 s3 = new AmazonS3Client(); s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1)); Connection db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS); Statement st = db.createStatement(); ResultSet rs = st.executeQuery( "SELECT data, pvs_unique_code FROM data WHERE pvs_serial_id='" + id + "' OFFSET 0 LIMIT 1;"); rs.next(); String json = rs.getString(1); String key = rs.getString(2); rs.close(); db.close(); Date recent = new Date(); while (cal.before(end)) { xlsSheet xls = new xlsSheet(json); System.out.println("Getting data of " + toDate(cal)); /* AWS S3????? */ String bucketName = "pvdata-storage-production"; // String bucketName = "pvdata-storage-staging"; System.out.println("Get s3 data by key='" + bucketName + "/data/" + key + "/" + toDate(cal) + "/'"); ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName) .withPrefix("data/" + key + "/" + toDate(cal) + "/")); /* get data from s3 */ do { for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { String keyname = objectSummary.getKey(); S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname)); BufferedReader reader = new BufferedReader( new InputStreamReader(new GZIPInputStream(object.getObjectContent()))); while (reader.ready()) { String line = reader.readLine(); try { json = line.substring(line.indexOf("{")); recent = parseDate(line.substring(0, 25)); xls.putData(json); } catch (NullPointerException e) { } } reader.close(); object.close(); } objectListing = s3.listNextBatchOfObjects(objectListing); } while (objectListing.getMarker() != null); /* if today, read postgres to get recent data */ if (toDate(cal).equals(today)) { System.out.println("Get recent data from postgres"); try { db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS); st = db.createStatement(); String sql = "SELECT data FROM data WHERE pvs_unique_code='" + key + "' AND created_at > '" + pgformat.format(recent) + "';"; System.out.println(sql); rs = st.executeQuery(sql); while (rs.next()) { json = rs.getString(1); xls.putData(json); } rs.close(); db.close(); } catch (PSQLException e) { } catch (ParseException e) { } } System.out.println("Write Buffer"); xls.writeBuffer(); File file = File.createTempFile("temp", ".xlsx"); file.deleteOnExit(); xls.putFile(new FileOutputStream(file)); System.out.println("Put S3"); s3.putObject(new PutObjectRequest("sanix-data-analysis", FOLDER + id + "-" + toDate(cal).replace("/", "-") + ".xlsx", file)); System.out.println("Finished: " + toDate(cal)); cal.add(Calendar.DAY_OF_MONTH, 1); } File file = File.createTempFile("temp", ".html"); file.deleteOnExit(); BufferedWriter bw = new BufferedWriter(new FileWriter(file)); ObjectListing objectListing = s3 .listObjects(new ListObjectsRequest().withBucketName("sanix-data-analysis").withPrefix(FOLDER)); bw.write("<html><head></head><body><ul>\n"); do { for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { String keyname = objectSummary.getKey(); if (Pattern.compile("\\.xlsx$|\\.html$").matcher(keyname).find()) { bw.write("<li><a href=\"https://s3-ap-northeast-1.amazonaws.com/sanix-data-analysis/" + keyname + "\">" + keyname.replaceAll("^[^\\/]*\\/", "") + "</a></li>\n"); } } objectListing = s3.listNextBatchOfObjects(objectListing); } while (objectListing.getMarker() != null); bw.write("</ul></body></html>\n"); bw.flush(); bw.close(); s3.putObject(new PutObjectRequest("sanix-data-analysis", FOLDER + "index.html", file)); }
From source file:jp.sanix.analysisLocal.java
License:Open Source License
public static void main(String[] args) throws IOException, SQLException, AmazonServiceException, JSONException, NullPointerException, ParseException { BufferedReader br = new BufferedReader( new InputStreamReader(new FileInputStream(new File("canceled.csv")), "SJIS")); br.readLine();/* w w w . jav a 2 s. c o m*/ String l; while ((l = br.readLine()) != null) { String[] col = l.split(","); String id = col[0]; String datefrom = col[1]; String dateto = col[1]; SimpleDateFormat format = new SimpleDateFormat("yyyy/MM/dd"); format.setTimeZone(TimeZone.getTimeZone("JST")); SimpleDateFormat pgformat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); pgformat.setTimeZone(TimeZone.getTimeZone("UTC")); Calendar cal = Calendar.getInstance(); Calendar end = Calendar.getInstance(); String today = toDate(cal); try { cal.setTime(format.parse(datefrom)); } catch (ParseException e) { } try { end.setTime(format.parse(dateto)); end.add(Calendar.DAY_OF_MONTH, 1); } catch (ParseException e) { } AmazonS3 s3 = new AmazonS3Client(); s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1)); Connection db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS); Statement st = db.createStatement(); ResultSet rs = st.executeQuery( "SELECT data, pvs_unique_code FROM data WHERE pvs_serial_id='" + id + "' OFFSET 0 LIMIT 1;"); rs.next(); String json = rs.getString(1); String key = rs.getString(2); rs.close(); db.close(); Date recent = new Date(); while (cal.before(end)) { xlsSheet xls = new xlsSheet(json); System.out.println("Getting data of " + toDate(cal)); /* AWS S3????? */ String bucketName = "pvdata-storage-production"; System.out.println("Get s3 data by key='" + bucketName + "/data/" + key + "/" + toDate(cal) + "/'"); ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName) .withPrefix("data/" + key + "/" + toDate(cal) + "/")); /* get data from s3 */ do { for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { String keyname = objectSummary.getKey(); S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname)); BufferedReader reader = new BufferedReader( new InputStreamReader(new GZIPInputStream(object.getObjectContent()))); while (reader.ready()) { String line = reader.readLine(); try { json = line.substring(line.indexOf("{")); recent = parseDate(line.substring(0, 25)); xls.putData(json); } catch (NullPointerException e) { } } reader.close(); object.close(); } objectListing = s3.listNextBatchOfObjects(objectListing); } while (objectListing.getMarker() != null); /* if today, read postgres to get recent data */ if (toDate(cal).equals(today)) { System.out.println("Get recent data from postgres"); try { db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS); st = db.createStatement(); String sql = "SELECT data FROM data WHERE pvs_unique_code='" + key + "' AND created_at > '" + pgformat.format(recent) + "';"; System.out.println(sql); rs = st.executeQuery(sql); while (rs.next()) { json = rs.getString(1); xls.putData(json); } rs.close(); db.close(); } catch (PSQLException e) { } catch (ParseException e) { } } System.out.println("Write Buffer"); xls.writeBuffer(); File file = new File( "C:\\Users\\SANIX_CORERD\\Desktop\\" + id + "_" + toDate(cal).replace("/", "-") + ".xlsx"); xls.putFile(new FileOutputStream(file)); System.out.println("Finished: " + toDate(cal)); cal.add(Calendar.DAY_OF_MONTH, 1); } } br.close(); }
From source file:jp.sanix.analysisStaging.java
License:Open Source License
public static void main(String[] args) throws IOException, SQLException, AmazonServiceException, JSONException, NullPointerException, ParseException { // if (args.length < 1) { // System.out.println("Please specify at least an arguments that is pvs_serial_id."); // System.out.println("analysis A9990004 // get today's data"); // System.out.println("analysis A9990004 2015/05/30 // get from 2015/05/30 to today's data"); // System.out.println("analysis A9990004 2015/05/30 2015/06/30"); // System.exit(-1); // }//from www . jav a 2s . co m // String id = args[0]; String uniq = "763ff280b16c61f524ae31b33401bcd6"; SimpleDateFormat format = new SimpleDateFormat("yyyy/MM/dd"); format.setTimeZone(TimeZone.getTimeZone("JST")); SimpleDateFormat normalformat = new SimpleDateFormat("yyyy/MM/dd'T'HH:mm:ss'Z'"); normalformat.setTimeZone(TimeZone.getTimeZone("JST")); SimpleDateFormat pgformat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); pgformat.setTimeZone(TimeZone.getTimeZone("UTC")); Calendar cal = Calendar.getInstance(); Calendar end = Calendar.getInstance(); cal.setTime(format.parse("2015/11/23")); end.setTime(format.parse("2015/11/23")); System.out.println(uniq); try { cal.setTime(format.parse(args[1])); } catch (ParseException e) { } catch (ArrayIndexOutOfBoundsException e) { } try { end.setTime(format.parse(args[2])); } catch (ParseException e) { } catch (ArrayIndexOutOfBoundsException e) { } end.add(Calendar.DAY_OF_MONTH, 1); AmazonS3 s3 = new AmazonS3Client(); s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1)); String bucketName = "pvdata-storage-staging"; while (cal.before(end)) { System.out.println("Getting data of " + toDate(cal)); /* AWS S3????? */ // String bucketName = "pvdata-storage-production"; System.out.println("Get s3 data by key='" + bucketName + "/data/" + uniq + "/" + toDate(cal) + "/'"); ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName) .withPrefix("data/" + uniq + "/" + toDate(cal) + "/")); /* get data from s3 */ xlsSheet xls = null; do { Boolean first = true; for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { String keyname = objectSummary.getKey(); S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname)); BufferedReader reader = new BufferedReader( new InputStreamReader(new GZIPInputStream(object.getObjectContent()))); while (reader.ready()) { String line = reader.readLine(); try { String json = line.substring(line.indexOf("{")); if (first) { xls = new xlsSheet(json); first = false; } else { xls.putData(json); } } catch (NullPointerException e) { } } reader.close(); object.close(); } objectListing = s3.listNextBatchOfObjects(objectListing); } while (objectListing.getMarker() != null); System.out.println("Write Buffer"); xls.writeBuffer(); File file = File.createTempFile("temp", ".xlsx"); file.deleteOnExit(); xls.putFile(new FileOutputStream(file)); System.out.println("Put S3"); s3.putObject(new PutObjectRequest("sanix-data-analysis", FOLDER + uniq + "-" + toDate(cal).replace("/", "-") + ".xlsx", file)); System.out.println("Finished: " + toDate(cal)); cal.add(Calendar.DAY_OF_MONTH, 1); } }
From source file:jp.sanix.mukaino.java
License:Open Source License
public static void main(String[] args) throws IOException, SQLException, AmazonServiceException, JSONException, NullPointerException, ParseException { AmazonS3 s3 = new AmazonS3Client(); s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1)); String[] ids = { "A0000089", "A0002570" }; Connection db;/* w ww .j ava 2 s . c o m*/ if (System.getProperty("user.name").toString().equals("ec2-user")) { db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS); } else { db = DriverManager.getConnection(PG_CON_LOCAL, PG_USER, PG_PASS); } for (String id : ids) { pvSensor.getInstance().getById(id); Statement st = db.createStatement(); ResultSet rs = st.executeQuery("SELECT data, pvs_unique_code FROM data WHERE pvs_serial_id='" + id + "' ORDER BY created_at DESC OFFSET 0 LIMIT 1;"); rs.next(); String json = rs.getString(1); String key = rs.getString(2); rs.close(); st.close(); Calendar cal = Calendar.getInstance(); cal.add(Calendar.DAY_OF_MONTH, -1); // 1?? Calendar end = Calendar.getInstance(); cal.add(Calendar.DAY_OF_MONTH, -1); // 1??? do { String mDate = String.format("%04d-%02d-%02d", cal.get(Calendar.YEAR), cal.get(Calendar.MONTH) + 1, cal.get(Calendar.DAY_OF_MONTH)); xlsSheetMukaino xls = new xlsSheetMukaino(json); /* AWS S3????? */ String bucketName = "pvdata-storage-production"; System.out.print("Get s3 data by key='" + bucketName + String.format("data/%s/%04d/%02d/%02d/", key, cal.get(Calendar.YEAR), cal.get(Calendar.MONTH) + 1, cal.get(Calendar.DAY_OF_MONTH))); ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName) .withPrefix("data/" + key + "/" + mDate.replace("-", "/"))); /* get data from s3 */ do { for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { String keyname = objectSummary.getKey(); S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname)); BufferedReader reader = new BufferedReader( new InputStreamReader(new GZIPInputStream(object.getObjectContent()))); while (reader.ready()) { String line = reader.readLine(); try { json = line.substring(line.indexOf("{")); xls.putData(json); } catch (NullPointerException e) { } } reader.close(); object.close(); System.out.print("."); } objectListing = s3.listNextBatchOfObjects(objectListing); } while (objectListing.getMarker() != null); System.out.println(); xls.writeBuffer(); File file = File.createTempFile("temp", ".xlsx"); file.deleteOnExit(); xls.putFile(new FileOutputStream(file)); s3.putObject(new PutObjectRequest("sanix-data-analysis", "Rw6WHQLqGcvs3wpWwALh80IHAiZqWYzw/" + id + "-" + mDate + ".xlsx", file)); System.out.println("Finished: " + mDate); cal.add(Calendar.DAY_OF_MONTH, 1); } while (cal.before(end)); } db.close(); File file = File.createTempFile("temp", ".html"); file.deleteOnExit(); BufferedWriter bw = new BufferedWriter(new FileWriter(file)); ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName("sanix-data-analysis") .withPrefix("Rw6WHQLqGcvs3wpWwALh80IHAiZqWYzw/")); bw.write("<html><head></head><body><ul>\n"); do { for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { String keyname = objectSummary.getKey(); if (Pattern.compile("\\.xlsx$|\\.html$").matcher(keyname).find()) { bw.write("<li><a href=\"https://s3-ap-northeast-1.amazonaws.com/sanix-data-analysis/" + keyname + "\">" + keyname.replaceAll("^[^\\/]*\\/", "") + "</a></li>\n"); } } objectListing = s3.listNextBatchOfObjects(objectListing); } while (objectListing.getMarker() != null); bw.write("</ul></body></html>\n"); bw.flush(); bw.close(); s3.putObject( new PutObjectRequest("sanix-data-analysis", "Rw6WHQLqGcvs3wpWwALh80IHAiZqWYzw/index.html", file)); }
From source file:jp.sanix.Rawdata.java
License:Open Source License
public TreeMap<Date, JSONObject> get() throws IOException, SQLException, ParseException { TreeMap<Date, JSONObject> data = new TreeMap<Date, JSONObject>(); AmazonS3 s3 = new AmazonS3Client(); s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1)); SimpleDateFormat fmt = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); fmt.setTimeZone(TimeZone.getTimeZone("UTC")); Calendar cal = Calendar.getInstance(); cal.setTime(this.getDate()); /* AWS S3????? */ String bucketName = "pvdata-storage-production"; ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName) .withPrefix("data/" + this.getUniqueCode() + "/" + toDate(cal) + "/")); /* get data from s3 */ do {//www. j a va2s. com for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { String keyname = objectSummary.getKey(); S3Object object = s3.getObject(new GetObjectRequest(BUCKET, keyname)); BufferedReader reader = new BufferedReader( new InputStreamReader(new GZIPInputStream(object.getObjectContent()))); while (reader.ready()) { String line = reader.readLine(); int pos; try { pos = line.indexOf("{"); } catch (NullPointerException e) { continue; } try { String jsons = line.substring(pos); JSONObject json = new JSONObject(jsons); JSONArray arr = json.getJSONObject("data").getJSONArray("sample"); for (int i = 0; i < arr.length(); i++) { JSONObject obj = arr.getJSONObject(i); Date date = fmt.parse(obj.getString("time")); for (int j = 0; j < obj.getJSONArray("powcon").length(); j++) { if (obj.getJSONArray("powcon").getJSONObject(j).get("genKwh") != JSONObject.NULL) { Double genkwh; genkwh = obj.getJSONArray("powcon").getJSONObject(j).getDouble("genKwh"); JSONObject jobj; if (data.containsKey(date)) { jobj = data.get(date); } else { jobj = new JSONObject(); } if (genkwh != null) { jobj.put(String.valueOf(j), genkwh); } if (jobj.length() > 0) { data.put(date, jobj); } } } } } catch (JSONException e) { e.printStackTrace(); } catch (StringIndexOutOfBoundsException e) { e.printStackTrace(); } } reader.close(); object.close(); } objectListing = s3.listNextBatchOfObjects(objectListing); } while (objectListing.getMarker() != null); Calendar today = Calendar.getInstance(); today.setTimeZone(TimeZone.getTimeZone("JST")); if (toDate(cal).equals(toDate(today))) { SimpleDateFormat pgfmt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); pgfmt.setTimeZone(TimeZone.getTimeZone("UTC")); Calendar recent = Calendar.getInstance(); recent.setTimeZone(TimeZone.getTimeZone("UST")); recent.add(Calendar.HOUR, -2); Connection db; if (System.getProperty("user.name").toString().equals("ec2-user")) { db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS); } else { db = DriverManager.getConnection(PG_CON_LOCAL, PG_USER, PG_PASS); } Statement st = db.createStatement(); String sql = "SELECT data FROM data WHERE pvs_unique_code='" + this.getUniqueCode() + "' AND created_at > '" + pgfmt.format(recent.getTime()) + "';"; ResultSet rs = st.executeQuery(sql); while (rs.next()) { String jsons = rs.getString(1); try { JSONObject json = new JSONObject(jsons); JSONArray arr = json.getJSONObject("data").getJSONArray("sample"); for (int i = 0; i < arr.length(); i++) { JSONObject obj = arr.getJSONObject(i); Date date = fmt.parse(obj.getString("time")); for (int j = 0; j < obj.getJSONArray("powcon").length(); j++) { if (obj.getJSONArray("powcon").getJSONObject(j).get("genKwh") != JSONObject.NULL) { Double genkwh; genkwh = obj.getJSONArray("powcon").getJSONObject(j).getDouble("genKwh"); JSONObject jobj; if (data.containsKey(date)) { jobj = data.get(date); } else { jobj = new JSONObject(); } if (genkwh != null) { jobj.put(String.valueOf(j), genkwh); } if (jobj.length() > 0) { data.put(date, jobj); } } } } } catch (JSONException e) { e.printStackTrace(); } } rs.close(); db.close(); } return data; }
From source file:jp.sanix.weatherData.java
License:Open Source License
public static void main(String[] args) throws IOException, SQLException, AmazonServiceException, JSONException, NullPointerException, ParseException { AmazonS3 s3 = new AmazonS3Client(); s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1)); BufferedReader br = new BufferedReader(new InputStreamReader( new FileInputStream(new File("src/main/web/?.csv")), "SJIS")); String line = null;/*w w w.j ava 2s. co m*/ br.readLine(); // 1??? while ((line = br.readLine()) != null) { String[] col = line.split(","); /* AWS S3????? */ String bucketName = "weather-forecast"; ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName) .withPrefix("day_insolation/" + col[0] + "/")); File file = File.createTempFile("temp", ".csv"); file.deleteOnExit(); BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file), "SJIS")); bw.write( ",?,?,???,,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23\n"); System.out.println(col[0] + ":" + col[1] + col[2]); /* get data from s3 */ int i = 0; do { for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { String keyname = objectSummary.getKey(); S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname)); BufferedReader reader = new BufferedReader(new InputStreamReader(object.getObjectContent())); StringBuilder sb = new StringBuilder(); String line2 = null; System.out.print(String.valueOf(i++) + "\r"); while ((line2 = reader.readLine()) != null) { sb.append(line2); } reader.close(); object.close(); try { JSONObject json = new JSONObject(sb.toString()); bw.write(String.join(",", col) + "," + json.get("date").toString().replace("-", "/") + ","); JSONArray jarr = json.getJSONArray("hour_data"); bw.write(jarr.join(",") + "\n"); // System.out.println(String.join(",", col) + "," + json.get("date").toString().replace("-", "/") + "," + jarr.join(",")); } catch (JSONException e) { // System.exit(1); } } objectListing = s3.listNextBatchOfObjects(objectListing); } while (objectListing.getMarker() != null); bw.flush(); bw.close(); if (i > 0) { s3.putObject(new PutObjectRequest("sanix-data-analysis", STORE_PATH + col[1] + col[2] + "_insolation.csv", file)); } } br.close(); br = new BufferedReader(new InputStreamReader( new FileInputStream(new File("src/main/web/?.csv")), "SJIS")); br.readLine(); // 1??? while ((line = br.readLine()) != null) { String[] col = line.split(","); /* AWS S3????? */ String bucketName = "weather-forecast"; ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName) .withPrefix("day_temperature/" + col[0] + "/")); File file = File.createTempFile("temp", ".csv"); file.deleteOnExit(); BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file), "SJIS")); bw.write( ",?,?,???,,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23\n"); System.out.println(col[0] + ":" + col[1] + col[2]); /* get data from s3 */ int i = 0; do { for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { String keyname = objectSummary.getKey(); S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname)); BufferedReader reader = new BufferedReader(new InputStreamReader(object.getObjectContent())); StringBuilder sb = new StringBuilder(); String line2 = null; System.out.print(String.valueOf(i++) + "\r"); while ((line2 = reader.readLine()) != null) { sb.append(line2); } reader.close(); object.close(); try { JSONObject json = new JSONObject(sb.toString()); bw.write(String.join(",", col) + "," + json.get("date").toString().replace("-", "/") + ","); JSONArray jarr = json.getJSONArray("hour_data"); bw.write(jarr.join(",") + "\n"); // System.out.println(String.join(",", col) + "," + json.get("date").toString().replace("-", "/") + "," + jarr.join(",")); } catch (JSONException e) { // System.exit(1); } } objectListing = s3.listNextBatchOfObjects(objectListing); } while (objectListing.getMarker() != null); bw.flush(); bw.close(); if (i > 0) { s3.putObject(new PutObjectRequest("sanix-data-analysis", STORE_PATH + col[1] + col[2] + "_temperture.csv", file)); } } br.close(); }