Example usage for com.amazonaws.services.s3 AmazonS3 listObjects

List of usage examples for com.amazonaws.services.s3 AmazonS3 listObjects

  1. HOME
  2. Java
  3. com.amazonaws
  4. com.amazonaws.services.s3.*
  5. AmazonS3
  6. listObjects

Introduction

In this page you can find the example usage for com.amazonaws.services.s3 AmazonS3 listObjects.

Prototype

public ObjectListing listObjects(ListObjectsRequest listObjectsRequest)
        throws SdkClientException, AmazonServiceException;

Source Link

Document

Returns a list of summary information about the objects in the specified bucket.

Usage

From source file:jp.classmethod.aws.gradle.s3.DeleteBucketTask.java

License:Apache License

@TaskAction
public void deleteBucket() {
    // to enable conventionMappings feature
    String bucketName = getBucketName();
    boolean ifExists = isIfExists();

    if (bucketName == null) {
        throw new GradleException("bucketName is not specified");
    }/*from  www  .j  ava2s .c o m*/

    AmazonS3PluginExtension ext = getProject().getExtensions().getByType(AmazonS3PluginExtension.class);
    AmazonS3 s3 = ext.getClient();

    if (ifExists == false || exists(s3)) {
        if (deleteObjects) {
            getLogger().info("Delete all S3 objects in bucket [{}]", bucketName);
            ObjectListing objectListing = s3.listObjects(bucketName);
            while (objectListing.getObjectSummaries().isEmpty() == false) {
                objectListing.getObjectSummaries().forEach(summary -> {
                    getLogger().info(" => delete s3://{}/{}", bucketName, summary.getKey());
                    s3.deleteObject(bucketName, summary.getKey());
                });
                objectListing = s3.listNextBatchOfObjects(objectListing);
            }
        }
        s3.deleteBucket(bucketName);
        getLogger().info("S3 bucket {} is deleted", bucketName);
    } else {
        getLogger().debug("S3 bucket {} does not exist", bucketName);
    }
}

From source file:jp.sanix.analysis.java

License:Open Source License

public static void main(String[] args) throws IOException, SQLException, AmazonServiceException, JSONException,
        NullPointerException, ParseException {

    if (args.length < 1) {
        System.out.println("Please specify at least an arguments that is pvs_serial_id.");
        System.out.println("analysis A9990004 // get today's data");
        System.out.println("analysis A9990004 2015/05/30 // get from 2015/05/30 to today's data");
        System.out.println("analysis A9990004 2015/05/30 2015/06/30");
        System.exit(-1);// w  w w .  ja  v  a  2s  .com
    }

    String id = args[0];

    SimpleDateFormat format = new SimpleDateFormat("yyyy/MM/dd");
    format.setTimeZone(TimeZone.getTimeZone("JST"));
    SimpleDateFormat normalformat = new SimpleDateFormat("yyyy/MM/dd'T'HH:mm:ss'Z'");
    normalformat.setTimeZone(TimeZone.getTimeZone("JST"));
    SimpleDateFormat pgformat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
    pgformat.setTimeZone(TimeZone.getTimeZone("UTC"));

    Calendar cal = Calendar.getInstance();
    Calendar end = Calendar.getInstance();
    String today = toDate(cal);

    System.out.println(id);
    try {
        cal.setTime(format.parse(args[1]));
    } catch (ParseException e) {
    } catch (ArrayIndexOutOfBoundsException e) {
    }
    try {
        end.setTime(format.parse(args[2]));
    } catch (ParseException e) {
    } catch (ArrayIndexOutOfBoundsException e) {
    }
    end.add(Calendar.DAY_OF_MONTH, 1);

    AmazonS3 s3 = new AmazonS3Client();

    s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1));

    Connection db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS);
    Statement st = db.createStatement();
    ResultSet rs = st.executeQuery(
            "SELECT data, pvs_unique_code FROM data WHERE pvs_serial_id='" + id + "' OFFSET 0 LIMIT 1;");
    rs.next();
    String json = rs.getString(1);
    String key = rs.getString(2);
    rs.close();
    db.close();
    Date recent = new Date();

    while (cal.before(end)) {
        xlsSheet xls = new xlsSheet(json);
        System.out.println("Getting data of " + toDate(cal));

        /* AWS S3????? */
        String bucketName = "pvdata-storage-production";
        //         String bucketName = "pvdata-storage-staging";
        System.out.println("Get s3 data by key='" + bucketName + "/data/" + key + "/" + toDate(cal) + "/'");
        ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName)
                .withPrefix("data/" + key + "/" + toDate(cal) + "/"));

        /* get data from s3 */
        do {
            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                String keyname = objectSummary.getKey();
                S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname));
                BufferedReader reader = new BufferedReader(
                        new InputStreamReader(new GZIPInputStream(object.getObjectContent())));
                while (reader.ready()) {
                    String line = reader.readLine();
                    try {
                        json = line.substring(line.indexOf("{"));
                        recent = parseDate(line.substring(0, 25));
                        xls.putData(json);
                    } catch (NullPointerException e) {
                    }
                }
                reader.close();
                object.close();
            }
            objectListing = s3.listNextBatchOfObjects(objectListing);
        } while (objectListing.getMarker() != null);

        /* if today, read postgres to get recent data */
        if (toDate(cal).equals(today)) {
            System.out.println("Get recent data from postgres");
            try {
                db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS);
                st = db.createStatement();
                String sql = "SELECT data FROM data WHERE pvs_unique_code='" + key + "' AND created_at > '"
                        + pgformat.format(recent) + "';";
                System.out.println(sql);
                rs = st.executeQuery(sql);
                while (rs.next()) {
                    json = rs.getString(1);
                    xls.putData(json);
                }
                rs.close();
                db.close();
            } catch (PSQLException e) {
            } catch (ParseException e) {
            }
        }
        System.out.println("Write Buffer");
        xls.writeBuffer();

        File file = File.createTempFile("temp", ".xlsx");
        file.deleteOnExit();
        xls.putFile(new FileOutputStream(file));
        System.out.println("Put S3");
        s3.putObject(new PutObjectRequest("sanix-data-analysis",
                FOLDER + id + "-" + toDate(cal).replace("/", "-") + ".xlsx", file));
        System.out.println("Finished: " + toDate(cal));

        cal.add(Calendar.DAY_OF_MONTH, 1);
    }

    File file = File.createTempFile("temp", ".html");
    file.deleteOnExit();
    BufferedWriter bw = new BufferedWriter(new FileWriter(file));
    ObjectListing objectListing = s3
            .listObjects(new ListObjectsRequest().withBucketName("sanix-data-analysis").withPrefix(FOLDER));
    bw.write("<html><head></head><body><ul>\n");
    do {
        for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
            String keyname = objectSummary.getKey();
            if (Pattern.compile("\\.xlsx$|\\.html$").matcher(keyname).find()) {
                bw.write("<li><a href=\"https://s3-ap-northeast-1.amazonaws.com/sanix-data-analysis/" + keyname
                        + "\">" + keyname.replaceAll("^[^\\/]*\\/", "") + "</a></li>\n");
            }
        }
        objectListing = s3.listNextBatchOfObjects(objectListing);
    } while (objectListing.getMarker() != null);
    bw.write("</ul></body></html>\n");
    bw.flush();
    bw.close();

    s3.putObject(new PutObjectRequest("sanix-data-analysis", FOLDER + "index.html", file));
}

From source file:jp.sanix.analysisLocal.java

License:Open Source License

public static void main(String[] args) throws IOException, SQLException, AmazonServiceException, JSONException,
        NullPointerException, ParseException {

    BufferedReader br = new BufferedReader(
            new InputStreamReader(new FileInputStream(new File("canceled.csv")), "SJIS"));
    br.readLine();/*from  w  ww  .j a  va  2  s .  c o m*/

    String l;
    while ((l = br.readLine()) != null) {
        String[] col = l.split(",");
        String id = col[0];
        String datefrom = col[1];
        String dateto = col[1];

        SimpleDateFormat format = new SimpleDateFormat("yyyy/MM/dd");
        format.setTimeZone(TimeZone.getTimeZone("JST"));
        SimpleDateFormat pgformat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        pgformat.setTimeZone(TimeZone.getTimeZone("UTC"));

        Calendar cal = Calendar.getInstance();
        Calendar end = Calendar.getInstance();
        String today = toDate(cal);

        try {
            cal.setTime(format.parse(datefrom));
        } catch (ParseException e) {
        }
        try {
            end.setTime(format.parse(dateto));
            end.add(Calendar.DAY_OF_MONTH, 1);
        } catch (ParseException e) {
        }

        AmazonS3 s3 = new AmazonS3Client();
        s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1));

        Connection db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS);
        Statement st = db.createStatement();
        ResultSet rs = st.executeQuery(
                "SELECT data, pvs_unique_code FROM data WHERE pvs_serial_id='" + id + "' OFFSET 0 LIMIT 1;");
        rs.next();
        String json = rs.getString(1);
        String key = rs.getString(2);
        rs.close();
        db.close();
        Date recent = new Date();

        while (cal.before(end)) {
            xlsSheet xls = new xlsSheet(json);
            System.out.println("Getting data of " + toDate(cal));

            /* AWS S3????? */
            String bucketName = "pvdata-storage-production";
            System.out.println("Get s3 data by key='" + bucketName + "/data/" + key + "/" + toDate(cal) + "/'");
            ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName)
                    .withPrefix("data/" + key + "/" + toDate(cal) + "/"));

            /* get data from s3 */
            do {
                for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                    String keyname = objectSummary.getKey();
                    S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname));
                    BufferedReader reader = new BufferedReader(
                            new InputStreamReader(new GZIPInputStream(object.getObjectContent())));
                    while (reader.ready()) {
                        String line = reader.readLine();
                        try {
                            json = line.substring(line.indexOf("{"));
                            recent = parseDate(line.substring(0, 25));
                            xls.putData(json);
                        } catch (NullPointerException e) {
                        }
                    }
                    reader.close();
                    object.close();
                }
                objectListing = s3.listNextBatchOfObjects(objectListing);
            } while (objectListing.getMarker() != null);

            /* if today, read postgres to get recent data */
            if (toDate(cal).equals(today)) {
                System.out.println("Get recent data from postgres");
                try {
                    db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS);
                    st = db.createStatement();
                    String sql = "SELECT data FROM data WHERE pvs_unique_code='" + key + "' AND created_at > '"
                            + pgformat.format(recent) + "';";
                    System.out.println(sql);
                    rs = st.executeQuery(sql);
                    while (rs.next()) {
                        json = rs.getString(1);
                        xls.putData(json);
                    }
                    rs.close();
                    db.close();
                } catch (PSQLException e) {
                } catch (ParseException e) {
                }
            }
            System.out.println("Write Buffer");
            xls.writeBuffer();

            File file = new File(
                    "C:\\Users\\SANIX_CORERD\\Desktop\\" + id + "_" + toDate(cal).replace("/", "-") + ".xlsx");
            xls.putFile(new FileOutputStream(file));
            System.out.println("Finished: " + toDate(cal));

            cal.add(Calendar.DAY_OF_MONTH, 1);
        }
    }
    br.close();
}

From source file:jp.sanix.analysisStaging.java

License:Open Source License

public static void main(String[] args) throws IOException, SQLException, AmazonServiceException, JSONException,
        NullPointerException, ParseException {

    //      if (args.length < 1) {
    //         System.out.println("Please specify at least an arguments that is pvs_serial_id.");
    //         System.out.println("analysis A9990004 // get today's data");
    //         System.out.println("analysis A9990004 2015/05/30 // get from 2015/05/30 to today's data");
    //         System.out.println("analysis A9990004 2015/05/30 2015/06/30");
    //         System.exit(-1);
    //      }//www  .j a v  a 2 s . com

    //      String id = args[0];
    String uniq = "763ff280b16c61f524ae31b33401bcd6";

    SimpleDateFormat format = new SimpleDateFormat("yyyy/MM/dd");
    format.setTimeZone(TimeZone.getTimeZone("JST"));
    SimpleDateFormat normalformat = new SimpleDateFormat("yyyy/MM/dd'T'HH:mm:ss'Z'");
    normalformat.setTimeZone(TimeZone.getTimeZone("JST"));
    SimpleDateFormat pgformat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
    pgformat.setTimeZone(TimeZone.getTimeZone("UTC"));

    Calendar cal = Calendar.getInstance();
    Calendar end = Calendar.getInstance();
    cal.setTime(format.parse("2015/11/23"));
    end.setTime(format.parse("2015/11/23"));

    System.out.println(uniq);
    try {
        cal.setTime(format.parse(args[1]));
    } catch (ParseException e) {
    } catch (ArrayIndexOutOfBoundsException e) {
    }
    try {
        end.setTime(format.parse(args[2]));
    } catch (ParseException e) {
    } catch (ArrayIndexOutOfBoundsException e) {
    }
    end.add(Calendar.DAY_OF_MONTH, 1);

    AmazonS3 s3 = new AmazonS3Client();

    s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1));

    String bucketName = "pvdata-storage-staging";

    while (cal.before(end)) {
        System.out.println("Getting data of " + toDate(cal));

        /* AWS S3????? */
        //         String bucketName = "pvdata-storage-production";
        System.out.println("Get s3 data by key='" + bucketName + "/data/" + uniq + "/" + toDate(cal) + "/'");
        ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName)
                .withPrefix("data/" + uniq + "/" + toDate(cal) + "/"));

        /* get data from s3 */
        xlsSheet xls = null;
        do {
            Boolean first = true;
            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                String keyname = objectSummary.getKey();
                S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname));
                BufferedReader reader = new BufferedReader(
                        new InputStreamReader(new GZIPInputStream(object.getObjectContent())));
                while (reader.ready()) {
                    String line = reader.readLine();
                    try {
                        String json = line.substring(line.indexOf("{"));
                        if (first) {
                            xls = new xlsSheet(json);
                            first = false;
                        } else {
                            xls.putData(json);
                        }
                    } catch (NullPointerException e) {
                    }
                }
                reader.close();
                object.close();
            }
            objectListing = s3.listNextBatchOfObjects(objectListing);
        } while (objectListing.getMarker() != null);

        System.out.println("Write Buffer");
        xls.writeBuffer();

        File file = File.createTempFile("temp", ".xlsx");
        file.deleteOnExit();
        xls.putFile(new FileOutputStream(file));
        System.out.println("Put S3");
        s3.putObject(new PutObjectRequest("sanix-data-analysis",
                FOLDER + uniq + "-" + toDate(cal).replace("/", "-") + ".xlsx", file));
        System.out.println("Finished: " + toDate(cal));

        cal.add(Calendar.DAY_OF_MONTH, 1);
    }

}

From source file:jp.sanix.mukaino.java

License:Open Source License

public static void main(String[] args) throws IOException, SQLException, AmazonServiceException, JSONException,
        NullPointerException, ParseException {

    AmazonS3 s3 = new AmazonS3Client();
    s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1));

    String[] ids = { "A0000089", "A0002570" };

    Connection db;/*from  w  ww .  j a v  a2s. com*/
    if (System.getProperty("user.name").toString().equals("ec2-user")) {
        db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS);
    } else {
        db = DriverManager.getConnection(PG_CON_LOCAL, PG_USER, PG_PASS);
    }

    for (String id : ids) {
        pvSensor.getInstance().getById(id);
        Statement st = db.createStatement();
        ResultSet rs = st.executeQuery("SELECT data, pvs_unique_code FROM data WHERE pvs_serial_id='" + id
                + "' ORDER BY created_at DESC OFFSET 0 LIMIT 1;");
        rs.next();
        String json = rs.getString(1);
        String key = rs.getString(2);
        rs.close();
        st.close();

        Calendar cal = Calendar.getInstance();
        cal.add(Calendar.DAY_OF_MONTH, -1); // 1??

        Calendar end = Calendar.getInstance();
        cal.add(Calendar.DAY_OF_MONTH, -1); // 1???

        do {
            String mDate = String.format("%04d-%02d-%02d", cal.get(Calendar.YEAR), cal.get(Calendar.MONTH) + 1,
                    cal.get(Calendar.DAY_OF_MONTH));
            xlsSheetMukaino xls = new xlsSheetMukaino(json);
            /* AWS S3????? */
            String bucketName = "pvdata-storage-production";
            System.out.print("Get s3 data by key='" + bucketName + String.format("data/%s/%04d/%02d/%02d/", key,
                    cal.get(Calendar.YEAR), cal.get(Calendar.MONTH) + 1, cal.get(Calendar.DAY_OF_MONTH)));
            ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName)
                    .withPrefix("data/" + key + "/" + mDate.replace("-", "/")));

            /* get data from s3 */
            do {
                for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                    String keyname = objectSummary.getKey();
                    S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname));
                    BufferedReader reader = new BufferedReader(
                            new InputStreamReader(new GZIPInputStream(object.getObjectContent())));
                    while (reader.ready()) {
                        String line = reader.readLine();
                        try {
                            json = line.substring(line.indexOf("{"));
                            xls.putData(json);
                        } catch (NullPointerException e) {
                        }
                    }
                    reader.close();
                    object.close();
                    System.out.print(".");
                }
                objectListing = s3.listNextBatchOfObjects(objectListing);
            } while (objectListing.getMarker() != null);
            System.out.println();

            xls.writeBuffer();
            File file = File.createTempFile("temp", ".xlsx");
            file.deleteOnExit();
            xls.putFile(new FileOutputStream(file));
            s3.putObject(new PutObjectRequest("sanix-data-analysis",
                    "Rw6WHQLqGcvs3wpWwALh80IHAiZqWYzw/" + id + "-" + mDate + ".xlsx", file));
            System.out.println("Finished: " + mDate);
            cal.add(Calendar.DAY_OF_MONTH, 1);
        } while (cal.before(end));

    }
    db.close();

    File file = File.createTempFile("temp", ".html");
    file.deleteOnExit();
    BufferedWriter bw = new BufferedWriter(new FileWriter(file));
    ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName("sanix-data-analysis")
            .withPrefix("Rw6WHQLqGcvs3wpWwALh80IHAiZqWYzw/"));
    bw.write("<html><head></head><body><ul>\n");
    do {
        for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
            String keyname = objectSummary.getKey();
            if (Pattern.compile("\\.xlsx$|\\.html$").matcher(keyname).find()) {
                bw.write("<li><a href=\"https://s3-ap-northeast-1.amazonaws.com/sanix-data-analysis/" + keyname
                        + "\">" + keyname.replaceAll("^[^\\/]*\\/", "") + "</a></li>\n");
            }
        }
        objectListing = s3.listNextBatchOfObjects(objectListing);
    } while (objectListing.getMarker() != null);
    bw.write("</ul></body></html>\n");
    bw.flush();
    bw.close();

    s3.putObject(
            new PutObjectRequest("sanix-data-analysis", "Rw6WHQLqGcvs3wpWwALh80IHAiZqWYzw/index.html", file));
}

From source file:jp.sanix.Rawdata.java

License:Open Source License

public TreeMap<Date, JSONObject> get() throws IOException, SQLException, ParseException {

    TreeMap<Date, JSONObject> data = new TreeMap<Date, JSONObject>();
    AmazonS3 s3 = new AmazonS3Client();
    s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1));

    SimpleDateFormat fmt = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
    fmt.setTimeZone(TimeZone.getTimeZone("UTC"));
    Calendar cal = Calendar.getInstance();
    cal.setTime(this.getDate());

    /* AWS S3????? */
    String bucketName = "pvdata-storage-production";
    ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName)
            .withPrefix("data/" + this.getUniqueCode() + "/" + toDate(cal) + "/"));

    /* get data from s3 */
    do {/*from  w w  w .ja  v a2s  .  co  m*/
        for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
            String keyname = objectSummary.getKey();
            S3Object object = s3.getObject(new GetObjectRequest(BUCKET, keyname));
            BufferedReader reader = new BufferedReader(
                    new InputStreamReader(new GZIPInputStream(object.getObjectContent())));
            while (reader.ready()) {
                String line = reader.readLine();
                int pos;
                try {
                    pos = line.indexOf("{");
                } catch (NullPointerException e) {
                    continue;
                }
                try {
                    String jsons = line.substring(pos);
                    JSONObject json = new JSONObject(jsons);
                    JSONArray arr = json.getJSONObject("data").getJSONArray("sample");
                    for (int i = 0; i < arr.length(); i++) {
                        JSONObject obj = arr.getJSONObject(i);
                        Date date = fmt.parse(obj.getString("time"));
                        for (int j = 0; j < obj.getJSONArray("powcon").length(); j++) {
                            if (obj.getJSONArray("powcon").getJSONObject(j).get("genKwh") != JSONObject.NULL) {
                                Double genkwh;
                                genkwh = obj.getJSONArray("powcon").getJSONObject(j).getDouble("genKwh");
                                JSONObject jobj;
                                if (data.containsKey(date)) {
                                    jobj = data.get(date);
                                } else {
                                    jobj = new JSONObject();
                                }
                                if (genkwh != null) {
                                    jobj.put(String.valueOf(j), genkwh);
                                }
                                if (jobj.length() > 0) {
                                    data.put(date, jobj);
                                }
                            }
                        }
                    }
                } catch (JSONException e) {
                    e.printStackTrace();
                } catch (StringIndexOutOfBoundsException e) {
                    e.printStackTrace();
                }
            }
            reader.close();
            object.close();
        }
        objectListing = s3.listNextBatchOfObjects(objectListing);
    } while (objectListing.getMarker() != null);

    Calendar today = Calendar.getInstance();
    today.setTimeZone(TimeZone.getTimeZone("JST"));
    if (toDate(cal).equals(toDate(today))) {
        SimpleDateFormat pgfmt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        pgfmt.setTimeZone(TimeZone.getTimeZone("UTC"));
        Calendar recent = Calendar.getInstance();
        recent.setTimeZone(TimeZone.getTimeZone("UST"));
        recent.add(Calendar.HOUR, -2);
        Connection db;
        if (System.getProperty("user.name").toString().equals("ec2-user")) {
            db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS);
        } else {
            db = DriverManager.getConnection(PG_CON_LOCAL, PG_USER, PG_PASS);
        }
        Statement st = db.createStatement();
        String sql = "SELECT data FROM data WHERE pvs_unique_code='" + this.getUniqueCode()
                + "' AND created_at > '" + pgfmt.format(recent.getTime()) + "';";
        ResultSet rs = st.executeQuery(sql);
        while (rs.next()) {
            String jsons = rs.getString(1);
            try {
                JSONObject json = new JSONObject(jsons);
                JSONArray arr = json.getJSONObject("data").getJSONArray("sample");
                for (int i = 0; i < arr.length(); i++) {
                    JSONObject obj = arr.getJSONObject(i);
                    Date date = fmt.parse(obj.getString("time"));
                    for (int j = 0; j < obj.getJSONArray("powcon").length(); j++) {
                        if (obj.getJSONArray("powcon").getJSONObject(j).get("genKwh") != JSONObject.NULL) {
                            Double genkwh;
                            genkwh = obj.getJSONArray("powcon").getJSONObject(j).getDouble("genKwh");
                            JSONObject jobj;
                            if (data.containsKey(date)) {
                                jobj = data.get(date);
                            } else {
                                jobj = new JSONObject();
                            }
                            if (genkwh != null) {
                                jobj.put(String.valueOf(j), genkwh);
                            }
                            if (jobj.length() > 0) {
                                data.put(date, jobj);
                            }
                        }
                    }
                }
            } catch (JSONException e) {
                e.printStackTrace();
            }
        }
        rs.close();
        db.close();
    }
    return data;
}

From source file:jp.sanix.weatherData.java

License:Open Source License

public static void main(String[] args) throws IOException, SQLException, AmazonServiceException, JSONException,
        NullPointerException, ParseException {

    AmazonS3 s3 = new AmazonS3Client();
    s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1));

    BufferedReader br = new BufferedReader(new InputStreamReader(
            new FileInputStream(new File("src/main/web/?.csv")), "SJIS"));

    String line = null;//from ww w .  j a  v  a2  s.co m
    br.readLine(); // 1???
    while ((line = br.readLine()) != null) {

        String[] col = line.split(",");

        /* AWS S3????? */
        String bucketName = "weather-forecast";
        ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName)
                .withPrefix("day_insolation/" + col[0] + "/"));

        File file = File.createTempFile("temp", ".csv");
        file.deleteOnExit();
        BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file), "SJIS"));
        bw.write(
                ",?,?,???,,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23\n");

        System.out.println(col[0] + ":" + col[1] + col[2]);
        /* get data from s3 */
        int i = 0;
        do {
            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                String keyname = objectSummary.getKey();
                S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname));
                BufferedReader reader = new BufferedReader(new InputStreamReader(object.getObjectContent()));
                StringBuilder sb = new StringBuilder();
                String line2 = null;
                System.out.print(String.valueOf(i++) + "\r");
                while ((line2 = reader.readLine()) != null) {
                    sb.append(line2);
                }
                reader.close();
                object.close();
                try {
                    JSONObject json = new JSONObject(sb.toString());
                    bw.write(String.join(",", col) + "," + json.get("date").toString().replace("-", "/") + ",");
                    JSONArray jarr = json.getJSONArray("hour_data");
                    bw.write(jarr.join(",") + "\n");
                    //                  System.out.println(String.join(",", col) + "," + json.get("date").toString().replace("-", "/") + "," + jarr.join(","));
                } catch (JSONException e) {
                    //                  System.exit(1);
                }
            }
            objectListing = s3.listNextBatchOfObjects(objectListing);
        } while (objectListing.getMarker() != null);

        bw.flush();
        bw.close();
        if (i > 0) {
            s3.putObject(new PutObjectRequest("sanix-data-analysis",
                    STORE_PATH + col[1] + col[2] + "_insolation.csv", file));
        }
    }
    br.close();

    br = new BufferedReader(new InputStreamReader(
            new FileInputStream(new File("src/main/web/?.csv")), "SJIS"));
    br.readLine(); // 1???
    while ((line = br.readLine()) != null) {

        String[] col = line.split(",");

        /* AWS S3????? */
        String bucketName = "weather-forecast";
        ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName)
                .withPrefix("day_temperature/" + col[0] + "/"));

        File file = File.createTempFile("temp", ".csv");
        file.deleteOnExit();
        BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file), "SJIS"));
        bw.write(
                ",?,?,???,,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23\n");

        System.out.println(col[0] + ":" + col[1] + col[2]);
        /* get data from s3 */
        int i = 0;
        do {
            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                String keyname = objectSummary.getKey();
                S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname));
                BufferedReader reader = new BufferedReader(new InputStreamReader(object.getObjectContent()));
                StringBuilder sb = new StringBuilder();
                String line2 = null;
                System.out.print(String.valueOf(i++) + "\r");
                while ((line2 = reader.readLine()) != null) {
                    sb.append(line2);
                }
                reader.close();
                object.close();
                try {
                    JSONObject json = new JSONObject(sb.toString());
                    bw.write(String.join(",", col) + "," + json.get("date").toString().replace("-", "/") + ",");
                    JSONArray jarr = json.getJSONArray("hour_data");
                    bw.write(jarr.join(",") + "\n");
                    //                  System.out.println(String.join(",", col) + "," + json.get("date").toString().replace("-", "/") + "," + jarr.join(","));
                } catch (JSONException e) {
                    //                  System.exit(1);
                }
            }
            objectListing = s3.listNextBatchOfObjects(objectListing);
        } while (objectListing.getMarker() != null);

        bw.flush();
        bw.close();
        if (i > 0) {
            s3.putObject(new PutObjectRequest("sanix-data-analysis",
                    STORE_PATH + col[1] + col[2] + "_temperture.csv", file));
        }
    }
    br.close();
}

From source file:jp.sanix.yokusei.java

License:Open Source License

public static void main(String[] args) throws IOException, SQLException, AmazonServiceException, JSONException,
        NullPointerException, ParseException {

    String id = "A0002441";
    String datefrom = "2015/10/01";
    String dateto = "2015/10/12";

    SimpleDateFormat format = new SimpleDateFormat("yyyy/MM/dd");
    format.setTimeZone(TimeZone.getTimeZone("JST"));
    SimpleDateFormat pgformat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
    pgformat.setTimeZone(TimeZone.getTimeZone("UTC"));

    Calendar cal = Calendar.getInstance();
    Calendar end = Calendar.getInstance();
    String today = toDate(cal);/*from  ww w .  j  av a  2  s  .c  o  m*/

    try {
        cal.setTime(format.parse(datefrom));
    } catch (ParseException e) {
    }
    try {
        end.setTime(format.parse(dateto));
        end.add(Calendar.DAY_OF_MONTH, 1);
    } catch (ParseException e) {
    }

    AmazonS3 s3 = new AmazonS3Client();
    s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1));

    Connection db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS);
    Statement st = db.createStatement();
    ResultSet rs = st.executeQuery(
            "SELECT data, pvs_unique_code FROM data WHERE pvs_serial_id='" + id + "' OFFSET 0 LIMIT 1;");
    rs.next();
    String json = rs.getString(1);
    String key = rs.getString(2);
    rs.close();
    db.close();
    Date recent = new Date();

    xlsSheetYokusei xls = new xlsSheetYokusei(json);
    while (cal.before(end)) {
        System.out.println("Getting data of " + toDate(cal));

        /* AWS S3????? */
        String bucketName = "pvdata-storage-production";
        System.out.println("Get s3 data by key='" + bucketName + "/data/" + key + "/" + toDate(cal) + "/'");
        ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName)
                .withPrefix("data/" + key + "/" + toDate(cal) + "/"));

        /* get data from s3 */
        do {
            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                String keyname = objectSummary.getKey();
                S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname));
                BufferedReader reader = new BufferedReader(
                        new InputStreamReader(new GZIPInputStream(object.getObjectContent())));
                String line;
                while ((line = reader.readLine()) != null) {
                    try {
                        json = line.substring(line.indexOf("{"));
                        xls.putData(json);
                    } catch (NullPointerException e) {
                    }
                }
                reader.close();
                object.close();
            }
            objectListing = s3.listNextBatchOfObjects(objectListing);
        } while (objectListing.getMarker() != null);

        /* if today, read postgres to get recent data */
        if (toDate(cal).equals(today)) {
            System.out.println("Get recent data from postgres");
            try {
                db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS);
                st = db.createStatement();
                String sql = "SELECT data FROM data WHERE pvs_unique_code='" + key + "' AND created_at > '"
                        + pgformat.format(recent) + "';";
                System.out.println(sql);
                rs = st.executeQuery(sql);
                while (rs.next()) {
                    json = rs.getString(1);
                    xls.putData(json);
                }
                rs.close();
                db.close();
            } catch (PSQLException e) {
            } catch (ParseException e) {
            }
        }
        System.out.println("Write Buffer");
        xls.writeBuffer();

        cal.add(Calendar.DAY_OF_MONTH, 1);
    }
    File file = new File(
            "C:\\Users\\SANIX_CORERD\\Desktop\\" + id + "-Diamond" + toDate(cal).replace("/", "-") + ".xlsx");
    xls.putFile(new FileOutputStream(file));
    System.out.println("Finished: " + toDate(cal));
}

From source file:mail.server.storage.AWSStorageDelete.java

License:GNU General Public License

protected void deleteBucketContents(AmazonS3 s3, String bucketName) throws Exception {
    while (true) {
        List<String> keys = new ArrayList<String>();

        log.debug("creating batch delete");

        ObjectListing listing = s3.listObjects(bucketName);
        for (S3ObjectSummary i : listing.getObjectSummaries()) {
            log.debug("key", i.getKey());
            keys.add(i.getKey());/*from www  .j  a  v a2  s  . c  om*/
        }

        if (keys.isEmpty())
            break;

        DeleteObjectsRequest req = new DeleteObjectsRequest(bucketName).withKeys(keys.toArray(new String[0]));
        log.debug("deleting");
        s3.deleteObjects(req);
    }
}

From source file:net.geoprism.data.aws.AmazonEndpoint.java

License:Open Source License

private List<String> listFiles(String prefix) {
    List<String> files = new LinkedList<String>();

    try {/*ww w.  ja v a  2 s.  c  o  m*/
        AmazonS3 s3Client = new AmazonS3Client(new ClasspathPropertiesFileCredentialsProvider());

        ListObjectsRequest request = new ListObjectsRequest();
        request = request.withBucketName("geodashboarddata");
        request = request.withPrefix(prefix);

        ObjectListing listing;

        do {
            listing = s3Client.listObjects(request);

            List<S3ObjectSummary> summaries = listing.getObjectSummaries();

            for (S3ObjectSummary summary : summaries) {
                String key = summary.getKey();

                if (key.endsWith(".xml.gz")) {
                    files.add(key);
                }
            }

            request.setMarker(listing.getNextMarker());
        } while (listing != null && listing.isTruncated());
    } catch (Exception e) {
        logger.error("Unable to retrieve files", e);
    }

    return files;
}