List of usage examples for com.amazonaws.services.s3 AmazonS3 getObject
public S3Object getObject(GetObjectRequest getObjectRequest) throws SdkClientException, AmazonServiceException;
Gets the object stored in Amazon S3 under the specified bucket and key.
From source file:jp.sanix.weatherData.java
License:Open Source License
public static void main(String[] args) throws IOException, SQLException, AmazonServiceException, JSONException, NullPointerException, ParseException { AmazonS3 s3 = new AmazonS3Client(); s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1)); BufferedReader br = new BufferedReader(new InputStreamReader( new FileInputStream(new File("src/main/web/?.csv")), "SJIS")); String line = null;/* www . j av a 2s. c o m*/ br.readLine(); // 1??? while ((line = br.readLine()) != null) { String[] col = line.split(","); /* AWS S3????? */ String bucketName = "weather-forecast"; ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName) .withPrefix("day_insolation/" + col[0] + "/")); File file = File.createTempFile("temp", ".csv"); file.deleteOnExit(); BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file), "SJIS")); bw.write( ",?,?,???,,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23\n"); System.out.println(col[0] + ":" + col[1] + col[2]); /* get data from s3 */ int i = 0; do { for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { String keyname = objectSummary.getKey(); S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname)); BufferedReader reader = new BufferedReader(new InputStreamReader(object.getObjectContent())); StringBuilder sb = new StringBuilder(); String line2 = null; System.out.print(String.valueOf(i++) + "\r"); while ((line2 = reader.readLine()) != null) { sb.append(line2); } reader.close(); object.close(); try { JSONObject json = new JSONObject(sb.toString()); bw.write(String.join(",", col) + "," + json.get("date").toString().replace("-", "/") + ","); JSONArray jarr = json.getJSONArray("hour_data"); bw.write(jarr.join(",") + "\n"); // System.out.println(String.join(",", col) + "," + json.get("date").toString().replace("-", "/") + "," + jarr.join(",")); } catch (JSONException e) { // System.exit(1); } } objectListing = s3.listNextBatchOfObjects(objectListing); } while (objectListing.getMarker() != null); bw.flush(); bw.close(); if (i > 0) { s3.putObject(new PutObjectRequest("sanix-data-analysis", STORE_PATH + col[1] + col[2] + "_insolation.csv", file)); } } br.close(); br = new BufferedReader(new InputStreamReader( new FileInputStream(new File("src/main/web/?.csv")), "SJIS")); br.readLine(); // 1??? while ((line = br.readLine()) != null) { String[] col = line.split(","); /* AWS S3????? */ String bucketName = "weather-forecast"; ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName) .withPrefix("day_temperature/" + col[0] + "/")); File file = File.createTempFile("temp", ".csv"); file.deleteOnExit(); BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file), "SJIS")); bw.write( ",?,?,???,,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23\n"); System.out.println(col[0] + ":" + col[1] + col[2]); /* get data from s3 */ int i = 0; do { for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { String keyname = objectSummary.getKey(); S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname)); BufferedReader reader = new BufferedReader(new InputStreamReader(object.getObjectContent())); StringBuilder sb = new StringBuilder(); String line2 = null; System.out.print(String.valueOf(i++) + "\r"); while ((line2 = reader.readLine()) != null) { sb.append(line2); } reader.close(); object.close(); try { JSONObject json = new JSONObject(sb.toString()); bw.write(String.join(",", col) + "," + json.get("date").toString().replace("-", "/") + ","); JSONArray jarr = json.getJSONArray("hour_data"); bw.write(jarr.join(",") + "\n"); // System.out.println(String.join(",", col) + "," + json.get("date").toString().replace("-", "/") + "," + jarr.join(",")); } catch (JSONException e) { // System.exit(1); } } objectListing = s3.listNextBatchOfObjects(objectListing); } while (objectListing.getMarker() != null); bw.flush(); bw.close(); if (i > 0) { s3.putObject(new PutObjectRequest("sanix-data-analysis", STORE_PATH + col[1] + col[2] + "_temperture.csv", file)); } } br.close(); }
From source file:jp.sanix.yokusei.java
License:Open Source License
public static void main(String[] args) throws IOException, SQLException, AmazonServiceException, JSONException, NullPointerException, ParseException { String id = "A0002441"; String datefrom = "2015/10/01"; String dateto = "2015/10/12"; SimpleDateFormat format = new SimpleDateFormat("yyyy/MM/dd"); format.setTimeZone(TimeZone.getTimeZone("JST")); SimpleDateFormat pgformat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); pgformat.setTimeZone(TimeZone.getTimeZone("UTC")); Calendar cal = Calendar.getInstance(); Calendar end = Calendar.getInstance(); String today = toDate(cal);// w ww .jav a 2 s . c o m try { cal.setTime(format.parse(datefrom)); } catch (ParseException e) { } try { end.setTime(format.parse(dateto)); end.add(Calendar.DAY_OF_MONTH, 1); } catch (ParseException e) { } AmazonS3 s3 = new AmazonS3Client(); s3.setRegion(Region.getRegion(Regions.AP_NORTHEAST_1)); Connection db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS); Statement st = db.createStatement(); ResultSet rs = st.executeQuery( "SELECT data, pvs_unique_code FROM data WHERE pvs_serial_id='" + id + "' OFFSET 0 LIMIT 1;"); rs.next(); String json = rs.getString(1); String key = rs.getString(2); rs.close(); db.close(); Date recent = new Date(); xlsSheetYokusei xls = new xlsSheetYokusei(json); while (cal.before(end)) { System.out.println("Getting data of " + toDate(cal)); /* AWS S3????? */ String bucketName = "pvdata-storage-production"; System.out.println("Get s3 data by key='" + bucketName + "/data/" + key + "/" + toDate(cal) + "/'"); ObjectListing objectListing = s3.listObjects(new ListObjectsRequest().withBucketName(bucketName) .withPrefix("data/" + key + "/" + toDate(cal) + "/")); /* get data from s3 */ do { for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { String keyname = objectSummary.getKey(); S3Object object = s3.getObject(new GetObjectRequest(bucketName, keyname)); BufferedReader reader = new BufferedReader( new InputStreamReader(new GZIPInputStream(object.getObjectContent()))); String line; while ((line = reader.readLine()) != null) { try { json = line.substring(line.indexOf("{")); xls.putData(json); } catch (NullPointerException e) { } } reader.close(); object.close(); } objectListing = s3.listNextBatchOfObjects(objectListing); } while (objectListing.getMarker() != null); /* if today, read postgres to get recent data */ if (toDate(cal).equals(today)) { System.out.println("Get recent data from postgres"); try { db = DriverManager.getConnection(PG_CON, PG_USER, PG_PASS); st = db.createStatement(); String sql = "SELECT data FROM data WHERE pvs_unique_code='" + key + "' AND created_at > '" + pgformat.format(recent) + "';"; System.out.println(sql); rs = st.executeQuery(sql); while (rs.next()) { json = rs.getString(1); xls.putData(json); } rs.close(); db.close(); } catch (PSQLException e) { } catch (ParseException e) { } } System.out.println("Write Buffer"); xls.writeBuffer(); cal.add(Calendar.DAY_OF_MONTH, 1); } File file = new File( "C:\\Users\\SANIX_CORERD\\Desktop\\" + id + "-Diamond" + toDate(cal).replace("/", "-") + ".xlsx"); xls.putFile(new FileOutputStream(file)); System.out.println("Finished: " + toDate(cal)); }
From source file:net.geoprism.data.aws.AmazonEndpoint.java
License:Open Source License
@Override public void copyFiles(File directory, List<String> keys, boolean preserveDirectories) { try {//from w ww . ja v a 2 s . c o m List<File> files = new LinkedList<File>(); AmazonS3 client = new AmazonS3Client(new ClasspathPropertiesFileCredentialsProvider()); for (String key : keys) { GetObjectRequest request = new GetObjectRequest("geodashboarddata", key); S3Object object = client.getObject(request); InputStream istream = object.getObjectContent(); try { String targetPath = this.getTargetPath(preserveDirectories, key); File file = new File(directory, targetPath); FileUtils.copyInputStreamToFile(istream, file); files.add(file); } finally { // Process the objectData stream. istream.close(); } } } catch (IOException e) { throw new ProgrammingErrorException(e); } }
From source file:net.oletalk.hellospringboot.dao.S3Dao.java
public void getObjectData(String bucketName, String key, OutputStream out) throws S3Exception { AmazonS3 s3client = new AmazonS3Client(new ClasspathPropertiesFileCredentialsProvider()); LOG.info("fetching requested object " + key); try {//from ww w . j a v a 2 s . co m S3Object object = s3client.getObject(new GetObjectRequest(bucketName, key)); LOG.info("fetched, serving up"); try { IOUtils.copy(object.getObjectContent(), out); } catch (IOException ioe) { LOG.error("Problem writing message: " + ioe.getMessage()); } } catch (AmazonS3Exception e) { LOG.error("Problem fetching from S3: ", e); String errorMsg = "Error fetching document"; try { out.write(errorMsg.getBytes(Charset.defaultCharset())); throw new S3Exception("Error fetching document"); } catch (IOException ioe) { LOG.error("Problem writing message: " + ioe.getMessage()); } } }
From source file:net.smartcosmos.plugin.service.aws.storage.AwsS3StorageService.java
License:Apache License
@Override public InputStream retrieve(IFile file) throws IOException { Preconditions.checkArgument((file != null), "file must not be null"); AmazonS3 s3 = new AmazonS3Client(credentials, new ClientConfiguration().withProtocol(Protocol.HTTPS)); GetObjectRequest getObjectRequest = new GetObjectRequest(getBucketName(), file.getFileName()); S3Object storedObject = s3.getObject(getObjectRequest); return storedObject.getObjectContent(); }
From source file:org.alanwilliamson.amazon.s3.Read.java
License:Open Source License
private cfData readToFile(AmazonS3 s3Client, String bucket, String key, String localpath, boolean overwrite, String aes256key, int retry, int retryseconds) throws Exception { File localFile = new File(localpath); if (localFile.isFile()) { if (!overwrite) throw new Exception("The file specified exists: " + localpath); else/*from ww w.j a v a2s . c om*/ localFile.delete(); } // Let us run around the number of attempts int attempts = 0; while (attempts < retry) { try { GetObjectRequest gor = new GetObjectRequest(bucket, key); if (aes256key != null && !aes256key.isEmpty()) gor.setSSECustomerKey(new SSECustomerKey(aes256key)); S3Object s3object = s3Client.getObject(gor); FileOutputStream outStream = null; try { outStream = new FileOutputStream(localFile); StreamUtil.copyTo(s3object.getObjectContent(), outStream, false); } finally { StreamUtil.closeStream(outStream); } return new cfStringData(localFile.toString()); } catch (Exception e) { cfEngine.log("Failed: AmazonS3Read(bucket=" + bucket + "; key=" + key + "; attempt=" + (attempts + 1) + "; exception=" + e.getMessage() + ")"); attempts++; if (attempts == retry) throw e; else Thread.sleep(retryseconds * 1000); } } return null; // should never }
From source file:org.alanwilliamson.amazon.s3.Read.java
License:Open Source License
private cfData readToMemory(AmazonS3 s3Client, String bucket, String key, String aes256key, int retry, int retryseconds) throws Exception { // Let us run around the number of attempts int attempts = 0; while (attempts < retry) { try {/*from ww w .j ava 2s.c o m*/ GetObjectRequest gor = new GetObjectRequest(bucket, key); if (aes256key != null && !aes256key.isEmpty()) gor.setSSECustomerKey(new SSECustomerKey(aes256key)); S3Object s3object = s3Client.getObject(gor); String contentType = s3object.getObjectMetadata().getContentType(); ByteArrayOutputStream baos = new ByteArrayOutputStream(32000); StreamUtil.copyTo(s3object.getObjectContent(), baos, false); if (contentType.indexOf("text") != -1 || contentType.indexOf("javascript") != -1) { return new cfStringData(baos.toString()); } else { return new cfBinaryData(baos.toByteArray()); } } catch (Exception e) { cfEngine.log("Failed: AmazonS3Read(bucket=" + bucket + "; key=" + key + "; attempt=" + (attempts + 1) + "; exception=" + e.getMessage() + ")"); attempts++; if (attempts == retry) throw e; else Thread.sleep(retryseconds * 1000); } } return null; // should never }
From source file:org.apache.manifoldcf.crawler.connectors.amazons3.AmazonS3Connector.java
License:Apache License
@Override public void processDocuments(String[] documentIdentifiers, IExistingVersions statuses, Specification spec, IProcessActivity activities, int jobMode, boolean usesDefaultAuthority) throws ManifoldCFException, ServiceInterruption { AmazonS3 amazons3Client = getClient(); if (amazons3Client == null) throw new ManifoldCFException("Amazon client can not connect at the moment"); String[] acls = null;/*from w w w . j av a2 s. c o m*/ // loop documents and process for (String documentIdentifier : documentIdentifiers) { try { if (documentIdentifier != null && StringUtils.isNotEmpty(documentIdentifier)) { String versionString; String[] aclsToUse; if (documentIdentifier.split(STD_SEPARATOR_BUCKET_AND_KEY) == null && documentIdentifier.length() < 1) { continue; } S3Artifact s3Artifact = getS3Artifact(documentIdentifier); S3Object s3Obj = amazons3Client .getObject(new GetObjectRequest(s3Artifact.getBucketName(), s3Artifact.getKey())); if (s3Obj == null) { // no such document in the bucket now // delete document activities.deleteDocument(documentIdentifier); continue; } Logging.connectors.info("Content-Type: " + s3Obj.getObjectMetadata().getContentType()); ObjectMetadata objectMetadata = s3Obj.getObjectMetadata(); Date lastModified = objectMetadata.getLastModified(); StringBuilder sb = new StringBuilder(); if (lastModified == null) { // remove the content activities.deleteDocument(documentIdentifier); continue; } aclsToUse = new String[0]; AccessControlList objectAcl = amazons3Client.getObjectAcl(s3Artifact.getBucketName(), s3Artifact.getKey()); Set<Grant> grants = objectAcl.getGrants(); String[] users = getUsers(grants); // sort aclsToUse = users; Arrays.sort(aclsToUse); packList(sb, aclsToUse, '+'); if (aclsToUse.length > 0) { sb.append('+'); pack(sb, AmazonS3Config.defaultAuthorityDenyToken, '+'); } else sb.append('-'); // sb.append(lastModified.toString()); versionString = sb.toString(); Logging.connectors.debug("version string : " + versionString); if (versionString.length() > 0 && !activities.checkDocumentNeedsReindexing(documentIdentifier, versionString)) { Logging.connectors.info("Document need not to be reindexed : " + documentIdentifier); continue; } Logging.connectors.debug("JIRA: Processing document identifier '" + documentIdentifier + "'"); long startTime = System.currentTimeMillis(); String errorCode = null; String errorDesc = null; Long fileSize = null; try { String mimeType = "text/plain";// default // tika works starts InputStream in = null; String document = null; try { in = s3Obj.getObjectContent(); parser.parse(in, handler, metadata, context); mimeType = tika.detect(in); document = handler.toString(); if (document == null) continue; metadata.set(Metadata.CONTENT_TYPE, mimeType); } catch (Exception e) { Logging.connectors.error("Error while parsing tika contents", e); } finally { if (in != null) IOUtils.closeQuietly(in); } String documentURI = getDocumentURI(s3Artifact); Logging.connectors.debug("document : " + documentURI); // need some investigation if (!activities.checkURLIndexable(documentURI)) { errorCode = activities.EXCLUDED_URL; errorDesc = "Excluded because of URL ('" + documentURI + "')"; activities.noDocument(documentIdentifier, versionString); continue; } if (!activities.checkMimeTypeIndexable(mimeType)) { errorCode = activities.EXCLUDED_MIMETYPE; errorDesc = "Excluded because of mime type ('" + mimeType + "')"; activities.noDocument(documentIdentifier, versionString); continue; } if (!activities.checkDateIndexable(lastModified)) { errorCode = activities.EXCLUDED_DATE; errorDesc = "Excluded because of date (" + lastModified + ")"; activities.noDocument(documentIdentifier, versionString); continue; } // otherwise process RepositoryDocument rd = new RepositoryDocument(); // Turn into acls and add into // description String[] denyAclsToUse; if (aclsToUse.length > 0) denyAclsToUse = new String[] { AmazonS3Config.defaultAuthorityDenyToken }; else denyAclsToUse = new String[0]; rd.setSecurity(RepositoryDocument.SECURITY_TYPE_DOCUMENT, aclsToUse, denyAclsToUse); rd.setMimeType(mimeType); if (lastModified != null) rd.setModifiedDate(lastModified); // set all meta-data fields addAllMetaData(rd, metadata); // get document try { byte[] documentBytes = document.getBytes(StandardCharsets.UTF_8); long fileLength = documentBytes.length; if (!activities.checkLengthIndexable(fileLength)) { errorCode = activities.EXCLUDED_LENGTH; errorDesc = "Excluded because of document length (" + fileLength + ")"; activities.noDocument(documentIdentifier, versionString); continue; } InputStream is = new ByteArrayInputStream(documentBytes); try { rd.setBinary(is, fileLength); activities.ingestDocumentWithException(documentIdentifier, versionString, documentURI, rd); errorCode = "OK"; fileSize = new Long(fileLength); } finally { if (is != null) IOUtils.closeQuietly(is); } } catch (Exception e) { Logging.connectors.error(e); } } catch (Exception e) { Logging.connectors.error(e); } } } catch (AmazonServiceException e) { Logging.connectors.error(e); } catch (AmazonClientException e) { Logging.connectors.error(e); } } }
From source file:org.apache.nifi.processors.aws.s3.FetchS3Object.java
License:Apache License
@Override public void onTrigger(final ProcessContext context, final ProcessSession session) { FlowFile flowFile = session.get();/* w w w . j a va2s . co m*/ if (flowFile == null) { return; } final long startNanos = System.nanoTime(); final String bucket = context.getProperty(BUCKET).evaluateAttributeExpressions(flowFile).getValue(); final String key = context.getProperty(KEY).evaluateAttributeExpressions(flowFile).getValue(); final String versionId = context.getProperty(VERSION_ID).evaluateAttributeExpressions(flowFile).getValue(); final AmazonS3 client = getClient(); final GetObjectRequest request; if (versionId == null) { request = new GetObjectRequest(bucket, key); } else { request = new GetObjectRequest(bucket, key, versionId); } final Map<String, String> attributes = new HashMap<>(); try (final S3Object s3Object = client.getObject(request)) { flowFile = session.importFrom(s3Object.getObjectContent(), flowFile); attributes.put("s3.bucket", s3Object.getBucketName()); final ObjectMetadata metadata = s3Object.getObjectMetadata(); if (metadata.getContentDisposition() != null) { final String fullyQualified = metadata.getContentDisposition(); final int lastSlash = fullyQualified.lastIndexOf("/"); if (lastSlash > -1 && lastSlash < fullyQualified.length() - 1) { attributes.put(CoreAttributes.PATH.key(), fullyQualified.substring(0, lastSlash)); attributes.put(CoreAttributes.ABSOLUTE_PATH.key(), fullyQualified); attributes.put(CoreAttributes.FILENAME.key(), fullyQualified.substring(lastSlash + 1)); } else { attributes.put(CoreAttributes.FILENAME.key(), metadata.getContentDisposition()); } } if (metadata.getContentMD5() != null) { attributes.put("hash.value", metadata.getContentMD5()); attributes.put("hash.algorithm", "MD5"); } if (metadata.getContentType() != null) { attributes.put(CoreAttributes.MIME_TYPE.key(), metadata.getContentType()); } if (metadata.getETag() != null) { attributes.put("s3.etag", metadata.getETag()); } if (metadata.getExpirationTime() != null) { attributes.put("s3.expirationTime", String.valueOf(metadata.getExpirationTime().getTime())); } if (metadata.getExpirationTimeRuleId() != null) { attributes.put("s3.expirationTimeRuleId", metadata.getExpirationTimeRuleId()); } if (metadata.getUserMetadata() != null) { attributes.putAll(metadata.getUserMetadata()); } if (metadata.getSSEAlgorithm() != null) { attributes.put("s3.sseAlgorithm", metadata.getSSEAlgorithm()); } if (metadata.getVersionId() != null) { attributes.put("s3.version", metadata.getVersionId()); } } catch (final IOException | AmazonClientException ioe) { getLogger().error("Failed to retrieve S3 Object for {}; routing to failure", new Object[] { flowFile, ioe }); flowFile = session.penalize(flowFile); session.transfer(flowFile, REL_FAILURE); return; } if (!attributes.isEmpty()) { flowFile = session.putAllAttributes(flowFile, attributes); } session.transfer(flowFile, REL_SUCCESS); final long transferMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos); getLogger().info("Successfully retrieved S3 Object for {} in {} millis; routing to success", new Object[] { flowFile, transferMillis }); session.getProvenanceReporter().fetch(flowFile, "http://" + bucket + ".amazonaws.com/" + key, transferMillis); }
From source file:org.applicationMigrator.migrationclient.FileTransferClient.java
License:Apache License
public void downloadFiles(String outputFilesPaths[]) throws IOException, InterruptedException, ClassNotFoundException { if (outputFilesPaths == null) { return;// w ww. j av a2 s . c o m } AWSCredentials awsCredentials = currentUser.getAwsCredentials(); Environment.getExternalStorageDirectory().setReadable(true, false); Environment.getExternalStorageDirectory().setWritable(true, false); for (String outputFilePath : outputFilesPaths) { String fileName = getFileName(outputFilePath); String userName = currentUser.getUserNameString(); String keyNameString = userName + "/" + applicationName + "/" + fileName; AmazonS3 s3client = new AmazonS3Client(awsCredentials); S3Object s3Object = s3client.getObject(new GetObjectRequest(BUCKET_NAME, keyNameString)); try { writeObjectToFile(s3Object, outputFilePath); File outputFile = new File(outputFilePath); outputFile.setReadable(true, false); outputFile.setWritable(true, false); } catch (Exception ignored) { } } }