Example usage for com.amazonaws.services.s3.model S3ObjectSummary getKey

List of usage examples for com.amazonaws.services.s3.model S3ObjectSummary getKey

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.services.s3.model.*
  5. S3ObjectSummary
  6. getKey

Introduction

In this page you can find the example usage for com.amazonaws.services.s3.model S3ObjectSummary getKey.

Prototype

public String getKey()

Source Link

Document

Gets the key under which this object is stored in Amazon S3.

Usage

From source file:jp.sanix.analysisLocal.java

License:Open Source License

public static void main(String[] args) throws IOException, SQLException, AmazonServiceException, JSONException,
        NullPointerException, ParseException {

    BufferedReader br = new BufferedReader(
            new InputStreamReader(new FileInputStream(new File("canceled.csv")), "SJIS"));
    br.readLine();/*from w  w w  . j  ava 2 s.  co m*/

    String l;
    while ((l = br.readLine()) != null) {
        String[] col = l.split(",");
        String id = col[0];
        String datefrom = col[1];
        String dateto = col[1];

        SimpleDateFormat format = new SimpleDateFormat("yyyy/MM/dd");
        format.setTimeZone(TimeZone.getTimeZone("JST"));
        SimpleDateFormat pgformat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        pgformat.setTimeZone(TimeZone.getTimeZone("UTC"));

        Calendar cal = Calendar.getInstance();
        Calendar end = Calendar.getInstance();
        String today = toDate(cal);

        try {
            cal.setTime(format.parse(datefrom));
        } catch (ParseException e) {
        }
        try {
            end.setTime(format.parse(dateto));
            end.add(Calendar.DAY_OF_MONTH, 1);
        } catch (ParseException e) {
        }

        AmazonS3 s3 = new AmazonS3Client();
        s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1));

        Connection db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS);
        Statement st = db.createStatement();
        ResultSet rs = st.executeQuery(
                "SELECT data, pvs_unique_code FROM data WHERE pvs_serial_id='" + id + "' OFFSET 0 LIMIT 1;");
        rs.next();
        String json = rs.getString(1);
        String key = rs.getString(2);
        rs.close();
        db.close();
        Date recent = new Date();

        while (cal.before(end)) {
            xlsSheet xls = new xlsSheet(json);
            System.out.println("Getting data of " + toDate(cal));

            /* AWS S3????? */
            String bucketName = "pvdata-storage-production";
            System.out.println("Get s3 data by key='" + bucketName + "/data/" + key + "/" + toDate(cal) + "/'");
            ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName)
                    .withPrefix("data/" + key + "/" + toDate(cal) + "/"));

            /* get data from s3 */
            do {
                for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                    String keyname = objectSummary.getKey();
                    S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname));
                    BufferedReader reader = new BufferedReader(
                            new InputStreamReader(new GZIPInputStream(object.getObjectContent())));
                    while (reader.ready()) {
                        String line = reader.readLine();
                        try {
                            json = line.substring(line.indexOf("{"));
                            recent = parseDate(line.substring(0, 25));
                            xls.putData(json);
                        } catch (NullPointerException e) {
                        }
                    }
                    reader.close();
                    object.close();
                }
                objectListing = s3.listNextBatchOfObjects(objectListing);
            } while (objectListing.getMarker() != null);

            /* if today, read postgres to get recent data */
            if (toDate(cal).equals(today)) {
                System.out.println("Get recent data from postgres");
                try {
                    db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS);
                    st = db.createStatement();
                    String sql = "SELECT data FROM data WHERE pvs_unique_code='" + key + "' AND created_at > '"
                            + pgformat.format(recent) + "';";
                    System.out.println(sql);
                    rs = st.executeQuery(sql);
                    while (rs.next()) {
                        json = rs.getString(1);
                        xls.putData(json);
                    }
                    rs.close();
                    db.close();
                } catch (PSQLException e) {
                } catch (ParseException e) {
                }
            }
            System.out.println("Write Buffer");
            xls.writeBuffer();

            File file = new File(
                    "C:\\Users\\SANIX_CORERD\\Desktop\\" + id + "_" + toDate(cal).replace("/", "-") + ".xlsx");
            xls.putFile(new FileOutputStream(file));
            System.out.println("Finished: " + toDate(cal));

            cal.add(Calendar.DAY_OF_MONTH, 1);
        }
    }
    br.close();
}

From source file:jp.sanix.analysisStaging.java

License:Open Source License

public static void main(String[] args) throws IOException, SQLException, AmazonServiceException, JSONException,
        NullPointerException, ParseException {

    //      if (args.length < 1) {
    //         System.out.println("Please specify at least an arguments that is pvs_serial_id.");
    //         System.out.println("analysis A9990004 // get today's data");
    //         System.out.println("analysis A9990004 2015/05/30 // get from 2015/05/30 to today's data");
    //         System.out.println("analysis A9990004 2015/05/30 2015/06/30");
    //         System.exit(-1);
    //      }/* w  w  w  .j  a  v a  2  s  .com*/

    //      String id = args[0];
    String uniq = "763ff280b16c61f524ae31b33401bcd6";

    SimpleDateFormat format = new SimpleDateFormat("yyyy/MM/dd");
    format.setTimeZone(TimeZone.getTimeZone("JST"));
    SimpleDateFormat normalformat = new SimpleDateFormat("yyyy/MM/dd'T'HH:mm:ss'Z'");
    normalformat.setTimeZone(TimeZone.getTimeZone("JST"));
    SimpleDateFormat pgformat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
    pgformat.setTimeZone(TimeZone.getTimeZone("UTC"));

    Calendar cal = Calendar.getInstance();
    Calendar end = Calendar.getInstance();
    cal.setTime(format.parse("2015/11/23"));
    end.setTime(format.parse("2015/11/23"));

    System.out.println(uniq);
    try {
        cal.setTime(format.parse(args[1]));
    } catch (ParseException e) {
    } catch (ArrayIndexOutOfBoundsException e) {
    }
    try {
        end.setTime(format.parse(args[2]));
    } catch (ParseException e) {
    } catch (ArrayIndexOutOfBoundsException e) {
    }
    end.add(Calendar.DAY_OF_MONTH, 1);

    AmazonS3 s3 = new AmazonS3Client();

    s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1));

    String bucketName = "pvdata-storage-staging";

    while (cal.before(end)) {
        System.out.println("Getting data of " + toDate(cal));

        /* AWS S3????? */
        //         String bucketName = "pvdata-storage-production";
        System.out.println("Get s3 data by key='" + bucketName + "/data/" + uniq + "/" + toDate(cal) + "/'");
        ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName)
                .withPrefix("data/" + uniq + "/" + toDate(cal) + "/"));

        /* get data from s3 */
        xlsSheet xls = null;
        do {
            Boolean first = true;
            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                String keyname = objectSummary.getKey();
                S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname));
                BufferedReader reader = new BufferedReader(
                        new InputStreamReader(new GZIPInputStream(object.getObjectContent())));
                while (reader.ready()) {
                    String line = reader.readLine();
                    try {
                        String json = line.substring(line.indexOf("{"));
                        if (first) {
                            xls = new xlsSheet(json);
                            first = false;
                        } else {
                            xls.putData(json);
                        }
                    } catch (NullPointerException e) {
                    }
                }
                reader.close();
                object.close();
            }
            objectListing = s3.listNextBatchOfObjects(objectListing);
        } while (objectListing.getMarker() != null);

        System.out.println("Write Buffer");
        xls.writeBuffer();

        File file = File.createTempFile("temp", ".xlsx");
        file.deleteOnExit();
        xls.putFile(new FileOutputStream(file));
        System.out.println("Put S3");
        s3.putObject(new PutObjectRequest("sanix-data-analysis",
                FOLDER + uniq + "-" + toDate(cal).replace("/", "-") + ".xlsx", file));
        System.out.println("Finished: " + toDate(cal));

        cal.add(Calendar.DAY_OF_MONTH, 1);
    }

}

From source file:jp.sanix.mukaino.java

License:Open Source License

public static void main(String[] args) throws IOException, SQLException, AmazonServiceException, JSONException,
        NullPointerException, ParseException {

    AmazonS3 s3 = new AmazonS3Client();
    s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1));

    String[] ids = { "A0000089", "A0002570" };

    Connection db;/*w  w  w. ja  v  a  2 s  .  c  o  m*/
    if (System.getProperty("user.name").toString().equals("ec2-user")) {
        db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS);
    } else {
        db = DriverManager.getConnection(PG_CON_LOCAL, PG_USER, PG_PASS);
    }

    for (String id : ids) {
        pvSensor.getInstance().getById(id);
        Statement st = db.createStatement();
        ResultSet rs = st.executeQuery("SELECT data, pvs_unique_code FROM data WHERE pvs_serial_id='" + id
                + "' ORDER BY created_at DESC OFFSET 0 LIMIT 1;");
        rs.next();
        String json = rs.getString(1);
        String key = rs.getString(2);
        rs.close();
        st.close();

        Calendar cal = Calendar.getInstance();
        cal.add(Calendar.DAY_OF_MONTH, -1); // 1??

        Calendar end = Calendar.getInstance();
        cal.add(Calendar.DAY_OF_MONTH, -1); // 1???

        do {
            String mDate = String.format("%04d-%02d-%02d", cal.get(Calendar.YEAR), cal.get(Calendar.MONTH) + 1,
                    cal.get(Calendar.DAY_OF_MONTH));
            xlsSheetMukaino xls = new xlsSheetMukaino(json);
            /* AWS S3????? */
            String bucketName = "pvdata-storage-production";
            System.out.print("Get s3 data by key='" + bucketName + String.format("data/%s/%04d/%02d/%02d/", key,
                    cal.get(Calendar.YEAR), cal.get(Calendar.MONTH) + 1, cal.get(Calendar.DAY_OF_MONTH)));
            ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName)
                    .withPrefix("data/" + key + "/" + mDate.replace("-", "/")));

            /* get data from s3 */
            do {
                for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                    String keyname = objectSummary.getKey();
                    S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname));
                    BufferedReader reader = new BufferedReader(
                            new InputStreamReader(new GZIPInputStream(object.getObjectContent())));
                    while (reader.ready()) {
                        String line = reader.readLine();
                        try {
                            json = line.substring(line.indexOf("{"));
                            xls.putData(json);
                        } catch (NullPointerException e) {
                        }
                    }
                    reader.close();
                    object.close();
                    System.out.print(".");
                }
                objectListing = s3.listNextBatchOfObjects(objectListing);
            } while (objectListing.getMarker() != null);
            System.out.println();

            xls.writeBuffer();
            File file = File.createTempFile("temp", ".xlsx");
            file.deleteOnExit();
            xls.putFile(new FileOutputStream(file));
            s3.putObject(new PutObjectRequest("sanix-data-analysis",
                    "Rw6WHQLqGcvs3wpWwALh80IHAiZqWYzw/" + id + "-" + mDate + ".xlsx", file));
            System.out.println("Finished: " + mDate);
            cal.add(Calendar.DAY_OF_MONTH, 1);
        } while (cal.before(end));

    }
    db.close();

    File file = File.createTempFile("temp", ".html");
    file.deleteOnExit();
    BufferedWriter bw = new BufferedWriter(new FileWriter(file));
    ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName("sanix-data-analysis")
            .withPrefix("Rw6WHQLqGcvs3wpWwALh80IHAiZqWYzw/"));
    bw.write("<html><head></head><body><ul>\n");
    do {
        for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
            String keyname = objectSummary.getKey();
            if (Pattern.compile("\\.xlsx$|\\.html$").matcher(keyname).find()) {
                bw.write("<li><a href=\"https://s3-ap-northeast-1.amazonaws.com/sanix-data-analysis/" + keyname
                        + "\">" + keyname.replaceAll("^[^\\/]*\\/", "") + "</a></li>\n");
            }
        }
        objectListing = s3.listNextBatchOfObjects(objectListing);
    } while (objectListing.getMarker() != null);
    bw.write("</ul></body></html>\n");
    bw.flush();
    bw.close();

    s3.putObject(
            new PutObjectRequest("sanix-data-analysis", "Rw6WHQLqGcvs3wpWwALh80IHAiZqWYzw/index.html", file));
}

From source file:jp.sanix.Rawdata.java

License:Open Source License

public TreeMap<Date, JSONObject> get() throws IOException, SQLException, ParseException {

    TreeMap<Date, JSONObject> data = new TreeMap<Date, JSONObject>();
    AmazonS3 s3 = new AmazonS3Client();
    s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1));

    SimpleDateFormat fmt = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
    fmt.setTimeZone(TimeZone.getTimeZone("UTC"));
    Calendar cal = Calendar.getInstance();
    cal.setTime(this.getDate());

    /* AWS S3????? */
    String bucketName = "pvdata-storage-production";
    ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName)
            .withPrefix("data/" + this.getUniqueCode() + "/" + toDate(cal) + "/"));

    /* get data from s3 */
    do {/*  w w  w .j  ava 2 s.  c o m*/
        for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
            String keyname = objectSummary.getKey();
            S3Object object = s3.getObject(new GetObjectRequest(BUCKET, keyname));
            BufferedReader reader = new BufferedReader(
                    new InputStreamReader(new GZIPInputStream(object.getObjectContent())));
            while (reader.ready()) {
                String line = reader.readLine();
                int pos;
                try {
                    pos = line.indexOf("{");
                } catch (NullPointerException e) {
                    continue;
                }
                try {
                    String jsons = line.substring(pos);
                    JSONObject json = new JSONObject(jsons);
                    JSONArray arr = json.getJSONObject("data").getJSONArray("sample");
                    for (int i = 0; i < arr.length(); i++) {
                        JSONObject obj = arr.getJSONObject(i);
                        Date date = fmt.parse(obj.getString("time"));
                        for (int j = 0; j < obj.getJSONArray("powcon").length(); j++) {
                            if (obj.getJSONArray("powcon").getJSONObject(j).get("genKwh") != JSONObject.NULL) {
                                Double genkwh;
                                genkwh = obj.getJSONArray("powcon").getJSONObject(j).getDouble("genKwh");
                                JSONObject jobj;
                                if (data.containsKey(date)) {
                                    jobj = data.get(date);
                                } else {
                                    jobj = new JSONObject();
                                }
                                if (genkwh != null) {
                                    jobj.put(String.valueOf(j), genkwh);
                                }
                                if (jobj.length() > 0) {
                                    data.put(date, jobj);
                                }
                            }
                        }
                    }
                } catch (JSONException e) {
                    e.printStackTrace();
                } catch (StringIndexOutOfBoundsException e) {
                    e.printStackTrace();
                }
            }
            reader.close();
            object.close();
        }
        objectListing = s3.listNextBatchOfObjects(objectListing);
    } while (objectListing.getMarker() != null);

    Calendar today = Calendar.getInstance();
    today.setTimeZone(TimeZone.getTimeZone("JST"));
    if (toDate(cal).equals(toDate(today))) {
        SimpleDateFormat pgfmt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        pgfmt.setTimeZone(TimeZone.getTimeZone("UTC"));
        Calendar recent = Calendar.getInstance();
        recent.setTimeZone(TimeZone.getTimeZone("UST"));
        recent.add(Calendar.HOUR, -2);
        Connection db;
        if (System.getProperty("user.name").toString().equals("ec2-user")) {
            db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS);
        } else {
            db = DriverManager.getConnection(PG_CON_LOCAL, PG_USER, PG_PASS);
        }
        Statement st = db.createStatement();
        String sql = "SELECT data FROM data WHERE pvs_unique_code='" + this.getUniqueCode()
                + "' AND created_at > '" + pgfmt.format(recent.getTime()) + "';";
        ResultSet rs = st.executeQuery(sql);
        while (rs.next()) {
            String jsons = rs.getString(1);
            try {
                JSONObject json = new JSONObject(jsons);
                JSONArray arr = json.getJSONObject("data").getJSONArray("sample");
                for (int i = 0; i < arr.length(); i++) {
                    JSONObject obj = arr.getJSONObject(i);
                    Date date = fmt.parse(obj.getString("time"));
                    for (int j = 0; j < obj.getJSONArray("powcon").length(); j++) {
                        if (obj.getJSONArray("powcon").getJSONObject(j).get("genKwh") != JSONObject.NULL) {
                            Double genkwh;
                            genkwh = obj.getJSONArray("powcon").getJSONObject(j).getDouble("genKwh");
                            JSONObject jobj;
                            if (data.containsKey(date)) {
                                jobj = data.get(date);
                            } else {
                                jobj = new JSONObject();
                            }
                            if (genkwh != null) {
                                jobj.put(String.valueOf(j), genkwh);
                            }
                            if (jobj.length() > 0) {
                                data.put(date, jobj);
                            }
                        }
                    }
                }
            } catch (JSONException e) {
                e.printStackTrace();
            }
        }
        rs.close();
        db.close();
    }
    return data;
}

From source file:jp.sanix.weatherData.java

License:Open Source License

public static void main(String[] args) throws IOException, SQLException, AmazonServiceException, JSONException,
        NullPointerException, ParseException {

    AmazonS3 s3 = new AmazonS3Client();
    s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1));

    BufferedReader br = new BufferedReader(new InputStreamReader(
            new FileInputStream(new File("src/main/web/?.csv")), "SJIS"));

    String line = null;//from  w  w w.  j a va 2  s .  co m
    br.readLine(); // 1???
    while ((line = br.readLine()) != null) {

        String[] col = line.split(",");

        /* AWS S3????? */
        String bucketName = "weather-forecast";
        ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName)
                .withPrefix("day_insolation/" + col[0] + "/"));

        File file = File.createTempFile("temp", ".csv");
        file.deleteOnExit();
        BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file), "SJIS"));
        bw.write(
                ",?,?,???,,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23\n");

        System.out.println(col[0] + ":" + col[1] + col[2]);
        /* get data from s3 */
        int i = 0;
        do {
            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                String keyname = objectSummary.getKey();
                S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname));
                BufferedReader reader = new BufferedReader(new InputStreamReader(object.getObjectContent()));
                StringBuilder sb = new StringBuilder();
                String line2 = null;
                System.out.print(String.valueOf(i++) + "\r");
                while ((line2 = reader.readLine()) != null) {
                    sb.append(line2);
                }
                reader.close();
                object.close();
                try {
                    JSONObject json = new JSONObject(sb.toString());
                    bw.write(String.join(",", col) + "," + json.get("date").toString().replace("-", "/") + ",");
                    JSONArray jarr = json.getJSONArray("hour_data");
                    bw.write(jarr.join(",") + "\n");
                    //                  System.out.println(String.join(",", col) + "," + json.get("date").toString().replace("-", "/") + "," + jarr.join(","));
                } catch (JSONException e) {
                    //                  System.exit(1);
                }
            }
            objectListing = s3.listNextBatchOfObjects(objectListing);
        } while (objectListing.getMarker() != null);

        bw.flush();
        bw.close();
        if (i > 0) {
            s3.putObject(new PutObjectRequest("sanix-data-analysis",
                    STORE_PATH + col[1] + col[2] + "_insolation.csv", file));
        }
    }
    br.close();

    br = new BufferedReader(new InputStreamReader(
            new FileInputStream(new File("src/main/web/?.csv")), "SJIS"));
    br.readLine(); // 1???
    while ((line = br.readLine()) != null) {

        String[] col = line.split(",");

        /* AWS S3????? */
        String bucketName = "weather-forecast";
        ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName)
                .withPrefix("day_temperature/" + col[0] + "/"));

        File file = File.createTempFile("temp", ".csv");
        file.deleteOnExit();
        BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file), "SJIS"));
        bw.write(
                ",?,?,???,,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23\n");

        System.out.println(col[0] + ":" + col[1] + col[2]);
        /* get data from s3 */
        int i = 0;
        do {
            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                String keyname = objectSummary.getKey();
                S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname));
                BufferedReader reader = new BufferedReader(new InputStreamReader(object.getObjectContent()));
                StringBuilder sb = new StringBuilder();
                String line2 = null;
                System.out.print(String.valueOf(i++) + "\r");
                while ((line2 = reader.readLine()) != null) {
                    sb.append(line2);
                }
                reader.close();
                object.close();
                try {
                    JSONObject json = new JSONObject(sb.toString());
                    bw.write(String.join(",", col) + "," + json.get("date").toString().replace("-", "/") + ",");
                    JSONArray jarr = json.getJSONArray("hour_data");
                    bw.write(jarr.join(",") + "\n");
                    //                  System.out.println(String.join(",", col) + "," + json.get("date").toString().replace("-", "/") + "," + jarr.join(","));
                } catch (JSONException e) {
                    //                  System.exit(1);
                }
            }
            objectListing = s3.listNextBatchOfObjects(objectListing);
        } while (objectListing.getMarker() != null);

        bw.flush();
        bw.close();
        if (i > 0) {
            s3.putObject(new PutObjectRequest("sanix-data-analysis",
                    STORE_PATH + col[1] + col[2] + "_temperture.csv", file));
        }
    }
    br.close();
}

From source file:jp.sanix.yokusei.java

License:Open Source License

public static void main(String[] args) throws IOException, SQLException, AmazonServiceException, JSONException,
        NullPointerException, ParseException {

    String id = "A0002441";
    String datefrom = "2015/10/01";
    String dateto = "2015/10/12";

    SimpleDateFormat format = new SimpleDateFormat("yyyy/MM/dd");
    format.setTimeZone(TimeZone.getTimeZone("JST"));
    SimpleDateFormat pgformat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
    pgformat.setTimeZone(TimeZone.getTimeZone("UTC"));

    Calendar cal = Calendar.getInstance();
    Calendar end = Calendar.getInstance();
    String today = toDate(cal);// w  w  w. j a v a  2  s  . c o  m

    try {
        cal.setTime(format.parse(datefrom));
    } catch (ParseException e) {
    }
    try {
        end.setTime(format.parse(dateto));
        end.add(Calendar.DAY_OF_MONTH, 1);
    } catch (ParseException e) {
    }

    AmazonS3 s3 = new AmazonS3Client();
    s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1));

    Connection db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS);
    Statement st = db.createStatement();
    ResultSet rs = st.executeQuery(
            "SELECT data, pvs_unique_code FROM data WHERE pvs_serial_id='" + id + "' OFFSET 0 LIMIT 1;");
    rs.next();
    String json = rs.getString(1);
    String key = rs.getString(2);
    rs.close();
    db.close();
    Date recent = new Date();

    xlsSheetYokusei xls = new xlsSheetYokusei(json);
    while (cal.before(end)) {
        System.out.println("Getting data of " + toDate(cal));

        /* AWS S3????? */
        String bucketName = "pvdata-storage-production";
        System.out.println("Get s3 data by key='" + bucketName + "/data/" + key + "/" + toDate(cal) + "/'");
        ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName)
                .withPrefix("data/" + key + "/" + toDate(cal) + "/"));

        /* get data from s3 */
        do {
            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                String keyname = objectSummary.getKey();
                S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname));
                BufferedReader reader = new BufferedReader(
                        new InputStreamReader(new GZIPInputStream(object.getObjectContent())));
                String line;
                while ((line = reader.readLine()) != null) {
                    try {
                        json = line.substring(line.indexOf("{"));
                        xls.putData(json);
                    } catch (NullPointerException e) {
                    }
                }
                reader.close();
                object.close();
            }
            objectListing = s3.listNextBatchOfObjects(objectListing);
        } while (objectListing.getMarker() != null);

        /* if today, read postgres to get recent data */
        if (toDate(cal).equals(today)) {
            System.out.println("Get recent data from postgres");
            try {
                db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS);
                st = db.createStatement();
                String sql = "SELECT data FROM data WHERE pvs_unique_code='" + key + "' AND created_at > '"
                        + pgformat.format(recent) + "';";
                System.out.println(sql);
                rs = st.executeQuery(sql);
                while (rs.next()) {
                    json = rs.getString(1);
                    xls.putData(json);
                }
                rs.close();
                db.close();
            } catch (PSQLException e) {
            } catch (ParseException e) {
            }
        }
        System.out.println("Write Buffer");
        xls.writeBuffer();

        cal.add(Calendar.DAY_OF_MONTH, 1);
    }
    File file = new File(
            "C:\\Users\\SANIX_CORERD\\Desktop\\" + id + "-Diamond" + toDate(cal).replace("/", "-") + ".xlsx");
    xls.putFile(new FileOutputStream(file));
    System.out.println("Finished: " + toDate(cal));
}

From source file:mail.server.storage.AWSStorageDelete.java

License:GNU General Public License

protected void deleteBucketContents(AmazonS3 s3, String bucketName) throws Exception {
    while (true) {
        List<String> keys = new ArrayList<String>();

        log.debug("creating batch delete");

        ObjectListing listing = s3.listObjects(bucketName);
        for (S3ObjectSummary i : listing.getObjectSummaries()) {
            log.debug("key", i.getKey());
            keys.add(i.getKey());// w ww . j a  va  2  s .  c  o  m
        }

        if (keys.isEmpty())
            break;

        DeleteObjectsRequest req = new DeleteObjectsRequest(bucketName).withKeys(keys.toArray(new String[0]));
        log.debug("deleting");
        s3.deleteObjects(req);
    }
}

From source file:mx.iteso.desi.cloud.hw3.AWSFaceCompare.java

License:Apache License

public Face compare(ByteBuffer imageBuffer) {
    final ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(srcBucket).withPrefix("Faces/");
    ListObjectsV2Result result;//ww  w.j  a  v  a2s  .c o  m
    ArrayList<String> s3Files = new ArrayList<>();
    Face empty = new Face("", 0.0f);

    try {
        String sourceFile = "source.jpg";
        Path file = Paths.get(sourceFile);
        Files.write(file, imageBuffer.array(), StandardOpenOption.CREATE);
        upload(sourceFile);
    } catch (IOException ex) {
        Logger.getLogger(FaceAddFrame.class.getName()).log(Level.SEVERE, null, ex);
    }

    do {
        result = s3.listObjectsV2(req);

        for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
            s3Files.add(objectSummary.getKey());
        }
        req.setContinuationToken(result.getNextContinuationToken());
    } while (result.isTruncated() == true);

    for (String s3File : s3Files) {
        if (s3File.endsWith(".jpg")) {
            System.out.println("Checking " + s3File + "...");
            Face face = compare("Compare/source.jpg", s3File);
            if (face.getCofidence() > SIMILARITY_THRESHOLD) {
                return face;
            }
        }
    }
    return empty;
}

From source file:n3phele.agent.repohandlers.S3Large.java

License:Open Source License

@Override
public List<String> getFileList() {
    List<String> result = new ArrayList<String>();
    int starIndex = key.indexOf("*");
    int questionIndex = key.indexOf("?");
    int curlyIndex = key.indexOf("{");
    if (starIndex == -1 && questionIndex == -1 && curlyIndex == -1) {
        this.totalLength = null;
        return result; // not wildcard
    }/*from   ww w .j a va2 s  . c om*/

    if (starIndex == -1)
        starIndex = Integer.MAX_VALUE;
    if (questionIndex == -1)
        questionIndex = Integer.MAX_VALUE;
    if (curlyIndex == -1)
        curlyIndex = Integer.MAX_VALUE;

    int wildStart = Math.min(Math.min(starIndex, questionIndex), curlyIndex);

    base = key.substring(0, wildStart);
    base = base.substring(0, base.lastIndexOf("/") + 1);
    // String wild = key.substring(base.lastIndexOf("/")+1);
    boolean done = false;
    this.totalLength = 0L;
    Pattern pattern = Pattern.compile("^" + Helper.wildcardToRegex(key));
    ObjectListing listing = s3().listObjects(this.root, base);
    while (!done) {
        done = !listing.isTruncated();

        for (S3ObjectSummary f : listing.getObjectSummaries()) {
            if (pattern.matcher(f.getKey()).matches()) {
                String file = f.getKey().substring(base.length());
                result.add(file);
                this.totalLength += f.getSize();
                log.info("Adding " + file + " size " + f.getSize());
            }
        }
        if (!done)
            listing = s3().listNextBatchOfObjects(listing);
    }

    return result;
}

From source file:n3phele.storage.s3.CloudStorageImpl.java

License:Open Source License

public boolean deleteFolder(Repository repo, String filename) {
    boolean result = false;
    Credential credential = repo.getCredential().decrypt();
    int retry = 3;

    setPermissions(repo, filename, false);
    if (!filename.endsWith("/")) {
        filename += "/";
    }//from   w  w w  .ja v a  2s.  c  om

    AmazonS3Client s3 = new AmazonS3Client(
            new BasicAWSCredentials(credential.getAccount(), credential.getSecret()));
    s3.setEndpoint(repo.getTarget().toString());
    while (retry-- > 0) {
        try {
            ObjectListing objects = s3.listObjects(repo.getRoot(), filename);
            for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
                log.info("Delete " + repo.getRoot() + ":" + objectSummary.getKey());
                s3.deleteObject(repo.getRoot(), objectSummary.getKey());
            }
            if (objects.isTruncated()) {
                retry++;
                log.info("Doing next portion");
                continue;
            }
            result = true;
            break;
        } catch (AmazonServiceException e) {
            log.log(Level.WARNING, "Service Error processing " + repo, e);
        } catch (AmazonClientException e) {
            log.log(Level.SEVERE, "Client Error processing " + repo, e);
        }
    }
    return result;
}