List of usage examples for com.amazonaws.services.s3.model S3Object getObjectContent
public S3ObjectInputStream getObjectContent()
From source file:common.S3Processor.java
License:Open Source License
public static RestaurantDO getObject(String key) { GetObjectRequest request = new GetObjectRequest(BUCKET_NAME, key); S3Object object = s3.getObject(request); ObjectMapper mapper = new ObjectMapper(); RestaurantDO actualObj;/*w w w.j av a2 s. c om*/ try { actualObj = mapper.readValue(object.getObjectContent(), RestaurantDO.class); return actualObj; } catch (Exception e) { } return null; }
From source file:ConnectionUtils.AWSS3Utils.java
public static ImageDTO loadImage(String bucketName, ImageDTO imageDTO, IMAGE_TYPE imageType) { Properties awsCredentialsProperties = new Properties(); try {// w w w .ja va 2 s .c o m awsCredentialsProperties .load(AWSS3Utils.class.getClassLoader().getResourceAsStream("prefs.properties")); } catch (Exception ex) { Logger.getLogger(AWSS3Utils.class.getName()).log(Level.SEVERE, null, ex); } String awsAccessKey = awsCredentialsProperties.getProperty("AWSACCESSKEY"); String awsSecretKey = awsCredentialsProperties.getProperty("SECRETACCESSKEY"); try { BasicAWSCredentials awsCredentials = new BasicAWSCredentials(awsAccessKey, awsSecretKey); AmazonS3 s3Client = new AmazonS3Client(awsCredentials); GetObjectRequest request = null; if (imageType.equals(IMAGE_TYPE.FULL)) { request = new GetObjectRequest(bucketName, imageDTO.getIID() + ".jpg"); } else if (imageType.equals(IMAGE_TYPE.THUMB)) { request = new GetObjectRequest(bucketName, imageDTO.getIID() + "_" + IMAGE_TYPE.THUMB.toString() + ".jpg"); } S3Object object = s3Client.getObject(request); S3ObjectInputStream objectContent = object.getObjectContent(); if (imageType.equals(IMAGE_TYPE.FULL)) { imageDTO.setImageBlob(objectContent); } else if (imageType.equals(IMAGE_TYPE.THUMB)) { imageDTO.setThumbnailBlob(objectContent); } return (imageDTO); } catch (Exception ex) { Logger.getLogger(AWSS3Utils.class.getName()).log(Level.SEVERE, null, ex); return null; } }
From source file:cz.pichlik.goodsentiment.server.repository.S3RepositoryBase.java
License:Apache License
public InputStream load(String bucket, String key) { GetObjectRequest getReq = new GetObjectRequest(bucket, key); S3Object object = s3Client.getObject(getReq); return object.getObjectContent(); }
From source file:dashboard.AmazonLogs.java
License:Open Source License
public int readAmazonLogs(int n, String AWS_USER, String AWS_PASS, String IPfile, String ERRfile, String bucketName, String DELETE_PROCESSED_LOGS, String API_KEY, String TOKEN, String apiuser, String apipass) throws Exception { if (n < 1) return 0; int eventsNumber = 0; String line = null;//from w ww .j a va 2 s. c o m int begin = 0; int zips = 0; int deletedZips = 0; int mixpanelStatus = 0; String registrant = ""; String ip = ""; String prevIP = ""; Mixpanel mix = new Mixpanel(); Whois w = new Whois(apiuser, apipass); int index = -1; Registrant r; ArrayList<Registrant> rList = new ArrayList<Registrant>(); ArrayList<Registrant> eList = new ArrayList<Registrant>(); IPList ipl = new IPList(); IPList errl = new IPList(); // Log files Bucket AWSCredentials credentials = new BasicAWSCredentials(AWS_USER, AWS_PASS); AmazonS3Client s3Client = new AmazonS3Client(credentials); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName); BufferedReader br = null; ObjectListing objectListing = s3Client.listObjects(listObjectsRequest); ObjectListing nextObjectListing = objectListing; zips = 0; Boolean more = true; if (objectListing == null) more = false; else { ipl.loadList(rList, IPfile); ipl.printList(rList, 30); } while (more) { // Reads 1000 files for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { // Handle ZIP files try { // Open and send to mixpanel events of one ZIP file String key = objectSummary.getKey(); S3Object object = s3Client.getObject(new GetObjectRequest(bucketName, key)); // Extract ZIP and read Object to reader br = new BufferedReader(new InputStreamReader(new GZIPInputStream(object.getObjectContent()))); zips++; // Read the lines from the unzipped file, break it and send to Mixpanel while ((line = br.readLine()) != null) { if (line.startsWith("#")) continue; if (line.trim().equals("")) continue; String[] values = line.split("\\s"); String eventTime = values[0] + " " + values[1]; ip = values[4]; if (ip != prevIP) { prevIP = ip; index = ipl.ipInList(ip, rList); if (index >= 0) { r = rList.get(index); registrant = r.name; // Update counter for this IP r.counter = r.counter + 1; rList.set(index, r); } else { // WHOIS - Check registrant of this IP address registrant = w.whoisIP(ip); // if there was an error, try again if (registrant.equals("ERROR")) registrant = w.whoisIP(ip); // if there was a second error, add it to errors list if (registrant.equals("ERROR")) { eList.add(new Registrant(ip, registrant, 1)); } else { // If name includes a comma, exclude the comma registrant = registrant.replace(",", ""); rList.add(new Registrant(ip, registrant, 1)); } } } String method = values[5]; String fileName = values[7]; String statusCode = values[8]; String userAgent = values[10]; String fName = fileName; if (fileName.contains("gigaspaces-")) { begin = fileName.lastIndexOf("gigaspaces-") + 11; fName = fileName.substring(begin, fileName.length()); } eventsNumber++; System.out.println(eventsNumber + ": " + eventTime + " " + ip + " " + registrant); // ==================================================== // Track the event in Mixpanel (using the POST import) // ==================================================== mixpanelStatus = mix.postCDNEventToMixpanel(API_KEY, TOKEN, ip, "Cloudfront CDN", eventTime, method, fileName, fName, userAgent, statusCode, registrant); } // while on ZIP file lines if (mixpanelStatus == 1 & DELETE_PROCESSED_LOGS.equals("YES")) { // Delete the CDN log ZIP file s3Client.deleteObject(bucketName, key); System.out.println("========= Deleted Zip " + zips + " ===== List Size " + rList.size() + " =========="); deletedZips++; } } catch (IOException e) { e.printStackTrace(); return eventsNumber; } finally { if (br != null) { br.close(); } if (eventsNumber >= n) { System.out.println("\n>>> " + eventsNumber + " events in " + zips + " Zip files. Deleted " + deletedZips + " Zip files.\n"); ipl.printList(rList, 100); ipl.saveList(rList, IPfile); if (!eList.isEmpty()) { SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd-HH-mm"); String fName = ERRfile + sdf.format(new Date()) + ".txt"; System.out.println("\n>>> " + eList.size() + " DomainTools errors:"); errl.saveList(eList, fName); } else System.out.println("\n>>> No DomainTools errors"); return eventsNumber; } } } // for (continue to next ZIP file // If there are more ZIP files, read next batch of 1000 if (objectListing.isTruncated()) { nextObjectListing = s3Client.listNextBatchOfObjects(objectListing); objectListing = nextObjectListing; } else more = false; // no more files } // while next objectListing System.out.println("\n>>> " + eventsNumber + " events in " + zips + " Zip files. Deleted " + deletedZips + " Zip files.\n"); ipl.printList(rList, 50); ipl.saveList(rList, IPfile); if (!eList.isEmpty()) { SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd-HH-mm"); String fName = ERRfile + sdf.format(new Date()) + ".txt"; System.out.println("\n>>> " + eList.size() + " DomainTools errors:"); errl.saveList(eList, fName); } else System.out.println("\n>>> No DomainTools errors"); return eventsNumber; }
From source file:dashboard.ImportCDN.java
License:Open Source License
public static int readAmazonLogs(int n) throws Exception { if (n < 1) return 0; int eventsNumber = 0; int begin = 0; int zips = 0; int postFailures = 0; int deletedZips = 0; int mixpanelStatus = 0; String line = null;// w w w. ja v a2 s . com // Log files Bucket AWSCredentials credentials = new BasicAWSCredentials(AWS_USER, AWS_PASS); AmazonS3Client s3Client = new AmazonS3Client(credentials); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName); // Set MARKER - from which Log File to start reading // listObjectsRequest.setMarker("E2DXXJR0N8BXOK.2013-03-18-10.ICK6IvaY.gz"); BufferedReader br = null; ObjectListing objectListing = s3Client.listObjects(listObjectsRequest); ObjectListing nextObjectListing = objectListing; zips = 0; Boolean more = true; if (objectListing == null) more = false; while (more) { // Reads 1000 files for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { // Handle ZIP files try { // Open and send to mixpanel events of one ZIP file String key = objectSummary.getKey(); S3Object object = s3Client.getObject(new GetObjectRequest(bucketName, key)); // Extract ZIP and read Object to reader br = new BufferedReader(new InputStreamReader(new GZIPInputStream(object.getObjectContent()))); zips++; // Read the lines from the unzipped file, break it and send to Mixpanel while ((line = br.readLine()) != null) { if (line.startsWith("#")) continue; if (line.trim().equals("")) continue; String[] values = line.split("\\s"); String eventTime = values[0] + " " + values[1]; String ip = values[4]; String method = values[5]; String fileName = values[7]; String statusCode = values[8]; String userAgent = values[10]; String fName = fileName; if (fileName.contains("gigaspaces-")) { begin = fileName.lastIndexOf("gigaspaces-") + 11; fName = fileName.substring(begin, fileName.length()); } eventsNumber++; System.out.println(eventsNumber + ": " + eventTime + " " + ip); // Track the event in Mixpanel (using the POST import) mixpanelStatus = postCDNEventToMixpanel(ip, "Cloudfront CDN", eventTime, method, fileName, fName, userAgent, statusCode); // If failed if (mixpanelStatus != 1) { postFailures++; System.out.println(" >>> POST event to Mixpanel Failed!! " + postFailures); } } // while on ZIP file lines if (mixpanelStatus == 1 & DELETE_PROCESSED_LOGS.equals("YES")) { // Delete the CDN log ZIP file s3Client.deleteObject(bucketName, key); System.out.println("============ Deleted Zip " + zips + " ============"); deletedZips++; } else System.out.println("============ Zip " + zips + " (not deleted) ============"); } catch (IOException e) { e.printStackTrace(); return eventsNumber; } finally { if (br != null) { br.close(); } if (eventsNumber >= n) { System.out.println("\n>>> " + eventsNumber + " events in " + zips + " Zip files."); System.out.println("\n>>> " + deletedZips + " Zip files deleted."); System.out.println("\n>>> " + postFailures + " post Failures\n"); return eventsNumber; } } } // for (continue to next ZIP file // If there are more ZIP files, read next batch of 1000 if (objectListing.isTruncated()) { nextObjectListing = s3Client.listNextBatchOfObjects(objectListing); objectListing = nextObjectListing; } else more = false; // no more files } // while next objectListing System.out.println("\n>>> " + eventsNumber + " events in " + zips + " Zip files."); System.out.println("\n>>> " + deletedZips + " Zip files deleted."); System.out.println("\n>>> " + postFailures + " post Failures\n"); return eventsNumber; }
From source file:dataprocessing.amazonwebservices.S3Client.java
License:GNU General Public License
/** ************************************************************* * @param filename name of file to retrieve from s3 * @return file name of local file// w ww. j a v a 2s . c o m * Gets file from S3 and writes to local file */ public List<String> readS3File(String filename) { List<String> lines = new ArrayList<>(); try { S3Object object = client.getObject(bucket, filename); S3ObjectInputStream stream = object.getObjectContent(); BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(stream)); String line; while ((line = bufferedReader.readLine()) != null) lines.add(line); bufferedReader.close(); stream.close(); } catch (Exception e) { e.printStackTrace(); } return lines; }
From source file:dbs.Getfroms3.java
public static void main(String[] args) throws IOException { AmazonS3 s3Client = AmazonS3ClientBuilder.standard().withRegion(Regions.AP_SOUTH_1) .withCredentials(new AWSStaticCredentialsProvider(credentials)).build(); //AmazonS3 s3Client = new AmazonS3Client(new ProfileCredentialsProvider("inhack2hire28")); try {//from w w w.j a v a 2s . co m System.out.println("Downloading an object"); S3Object s3object = s3Client.getObject(new GetObjectRequest(bucketName, key)); System.out.println("Content-Type: " + s3object.getObjectMetadata().getContentType()); displayTextInputStream(s3object.getObjectContent()); // Get a range of bytes from an object. GetObjectRequest rangeObjectRequest = new GetObjectRequest(bucketName, key); rangeObjectRequest.setRange(0, 10); S3Object objectPortion = s3Client.getObject(rangeObjectRequest); System.out.println("Printing bytes retrieved."); displayTextInputStream(objectPortion.getObjectContent()); } catch (AmazonServiceException ase) { System.out.println("Caught an AmazonServiceException, which" + " means your request made it " + "to Amazon S3, but was rejected with an error response" + " for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { System.out.println("Caught an AmazonClientException, which means" + " the client encountered " + "an internal error while trying to " + "communicate with S3, " + "such as not being able to access the network."); System.out.println("Error Message: " + ace.getMessage()); } }
From source file:dk.dma.nogoservice.service.S3DataLoader.java
License:Apache License
<T> T loadData(String key, Class<T> clazz) throws IOException { Stopwatch stopwatch = Stopwatch.createStarted(); S3Object object = amazonS3.getObject(S3DataLoader.DATA_BUCKET, key); File cacheFile = new File(tempDir, object.getObjectMetadata().getETag() + key); ObjectMapper objectMapper = new ObjectMapper(); if (cacheLocally) { if (cacheFile.exists()) { log.info("Using local cached file {}", cacheFile.getAbsolutePath()); return objectMapper.readValue(cacheFile, clazz); }/*www. j a va 2s .c o m*/ } try (S3ObjectInputStream objectContent = object.getObjectContent()) { T data = objectMapper.readValue(objectContent, clazz); if (cacheLocally) { log.info("caching S3 file locally in {}", cacheFile.getAbsolutePath()); objectMapper.writeValue(cacheFile, data); } return data; } finally { log.info("Loaded file {} from Amazon S3 in {} ms", key, stopwatch.stop().elapsed(TimeUnit.MILLISECONDS)); } }
From source file:eu.openg.aws.s3.internal.FakeS3Object.java
License:Apache License
FakeS3Object(S3Object object, Clock clock) { this.clock = clock; this.object = object; setContent(object.getObjectContent()); updateMetadata(object.getObjectMetadata()); }
From source file:eu.stratosphere.nephele.fs.s3.S3DataInputStream.java
License:Apache License
/** * Constructs a new input stream which reads its data from the specified S3 object. * /*from w w w .j a va 2 s. com*/ * @param s3Client * the S3 client to connect to Amazon S3. * @param bucket * the name of the S3 bucket the object is stored in * @param object * the name of the S3 object whose content shall be read * @throws IOException * thrown if an error occurs while accessing the specified S3 object */ S3DataInputStream(final AmazonS3Client s3Client, final String bucket, final String object) throws IOException { S3Object s3o = null; try { s3o = s3Client.getObject(bucket, object); } catch (AmazonServiceException e) { throw new IOException(StringUtils.stringifyException(e)); } this.inputStream = s3o.getObjectContent(); }