List of usage examples for com.amazonaws.services.s3 AmazonS3Client getObject
@Override
public S3Object getObject(GetObjectRequest getObjectRequest) throws SdkClientException, AmazonServiceException
From source file:dashboard.ImportCDN.java
License:Open Source License
public static int readAmazonLogs(int n) throws Exception { if (n < 1) return 0; int eventsNumber = 0; int begin = 0; int zips = 0; int postFailures = 0; int deletedZips = 0; int mixpanelStatus = 0; String line = null;//from w ww.j av a2s . c o m // Log files Bucket AWSCredentials credentials = new BasicAWSCredentials(AWS_USER, AWS_PASS); AmazonS3Client s3Client = new AmazonS3Client(credentials); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName); // Set MARKER - from which Log File to start reading // listObjectsRequest.setMarker("E2DXXJR0N8BXOK.2013-03-18-10.ICK6IvaY.gz"); BufferedReader br = null; ObjectListing objectListing = s3Client.listObjects(listObjectsRequest); ObjectListing nextObjectListing = objectListing; zips = 0; Boolean more = true; if (objectListing == null) more = false; while (more) { // Reads 1000 files for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { // Handle ZIP files try { // Open and send to mixpanel events of one ZIP file String key = objectSummary.getKey(); S3Object object = s3Client.getObject(new GetObjectRequest(bucketName, key)); // Extract ZIP and read Object to reader br = new BufferedReader(new InputStreamReader(new GZIPInputStream(object.getObjectContent()))); zips++; // Read the lines from the unzipped file, break it and send to Mixpanel while ((line = br.readLine()) != null) { if (line.startsWith("#")) continue; if (line.trim().equals("")) continue; String[] values = line.split("\\s"); String eventTime = values[0] + " " + values[1]; String ip = values[4]; String method = values[5]; String fileName = values[7]; String statusCode = values[8]; String userAgent = values[10]; String fName = fileName; if (fileName.contains("gigaspaces-")) { begin = fileName.lastIndexOf("gigaspaces-") + 11; fName = fileName.substring(begin, fileName.length()); } eventsNumber++; System.out.println(eventsNumber + ": " + eventTime + " " + ip); // Track the event in Mixpanel (using the POST import) mixpanelStatus = postCDNEventToMixpanel(ip, "Cloudfront CDN", eventTime, method, fileName, fName, userAgent, statusCode); // If failed if (mixpanelStatus != 1) { postFailures++; System.out.println(" >>> POST event to Mixpanel Failed!! " + postFailures); } } // while on ZIP file lines if (mixpanelStatus == 1 & DELETE_PROCESSED_LOGS.equals("YES")) { // Delete the CDN log ZIP file s3Client.deleteObject(bucketName, key); System.out.println("============ Deleted Zip " + zips + " ============"); deletedZips++; } else System.out.println("============ Zip " + zips + " (not deleted) ============"); } catch (IOException e) { e.printStackTrace(); return eventsNumber; } finally { if (br != null) { br.close(); } if (eventsNumber >= n) { System.out.println("\n>>> " + eventsNumber + " events in " + zips + " Zip files."); System.out.println("\n>>> " + deletedZips + " Zip files deleted."); System.out.println("\n>>> " + postFailures + " post Failures\n"); return eventsNumber; } } } // for (continue to next ZIP file // If there are more ZIP files, read next batch of 1000 if (objectListing.isTruncated()) { nextObjectListing = s3Client.listNextBatchOfObjects(objectListing); objectListing = nextObjectListing; } else more = false; // no more files } // while next objectListing System.out.println("\n>>> " + eventsNumber + " events in " + zips + " Zip files."); System.out.println("\n>>> " + deletedZips + " Zip files deleted."); System.out.println("\n>>> " + postFailures + " post Failures\n"); return eventsNumber; }
From source file:hu.mta.sztaki.lpds.cloud.entice.imageoptimizer.iaashandler.amazontarget.Storage.java
License:Apache License
/** * @param endpoint S3 endpoint URL//from ww w. j a v a 2 s . c o m * @param accessKey Access key * @param secretKey Secret key * @param bucket Bucket name * @param path Key name of the object to download (path + file name) * @param file Local file to download to * @throws Exception On any error */ public static void download(String endpoint, String accessKey, String secretKey, String bucket, String path, File file) throws Exception { AmazonS3Client amazonS3Client = null; InputStream in = null; OutputStream out = null; try { AWSCredentials awsCredentials = new BasicAWSCredentials(accessKey, secretKey); ClientConfiguration clientConfiguration = new ClientConfiguration(); clientConfiguration.setMaxConnections(MAX_CONNECTIONS); clientConfiguration.setMaxErrorRetry(PredefinedRetryPolicies.DEFAULT_MAX_ERROR_RETRY); clientConfiguration.setConnectionTimeout(ClientConfiguration.DEFAULT_CONNECTION_TIMEOUT); amazonS3Client = new AmazonS3Client(awsCredentials, clientConfiguration); S3ClientOptions clientOptions = new S3ClientOptions().withPathStyleAccess(true); amazonS3Client.setS3ClientOptions(clientOptions); amazonS3Client.setEndpoint(endpoint); S3Object object = amazonS3Client.getObject(new GetObjectRequest(bucket, path)); in = object.getObjectContent(); byte[] buf = new byte[BUFFER_SIZE]; out = new FileOutputStream(file); int count; while ((count = in.read(buf)) != -1) out.write(buf, 0, count); out.close(); in.close(); } catch (AmazonServiceException x) { Shrinker.myLogger.info("download error: " + x.getMessage()); throw new Exception("download exception", x); } catch (AmazonClientException x) { Shrinker.myLogger.info("download error: " + x.getMessage()); throw new Exception("download exception", x); } catch (IOException x) { Shrinker.myLogger.info("download error: " + x.getMessage()); throw new Exception("download exception", x); } finally { if (in != null) { try { in.close(); } catch (Exception e) { } } if (out != null) { try { out.close(); } catch (Exception e) { } } if (amazonS3Client != null) { try { amazonS3Client.shutdown(); } catch (Exception e) { } } } }
From source file:io.seldon.resources.external.S3FileStreamer.java
License:Apache License
public InputStream getResourceStream(String reference) throws IOException { logger.info("Reading file from s3://" + reference); AmazonS3Client client; if (creds != null) { client = new AmazonS3Client(creds); } else {/*from w w w .j a va 2 s.c o m*/ client = new AmazonS3Client(); } String[] bucketAndFile = reference.split("/", 2); if (bucketAndFile.length != 2) { return null; } S3Object object = client.getObject(new GetObjectRequest(bucketAndFile[0], bucketAndFile[1])); if (reference.endsWith(".gz")) { return new S3ObjectInputStreamWrapper(new GZIPInputStream(object.getObjectContent()), client); } else { return new S3ObjectInputStreamWrapper(object.getObjectContent(), client); } }
From source file:org.apache.usergrid.apm.service.CrashUtil.java
License:Apache License
public static String getCrashSummary(String fullAppName, String s3CrashFileName) { DeploymentConfig config = DeploymentConfig.geDeploymentConfig(); AWSCredentials credentials = new BasicAWSCredentials(config.getAccessKey(), config.getSecretKey()); AmazonS3Client s3Client = new AmazonS3Client(credentials); String s3FullFileName = AWSUtil.formS3CrashFileUrl(fullAppName, s3CrashFileName); log.info("Crash file bucket " + config.getS3LogBucket() + " and file name : " + s3FullFileName); GetObjectRequest objectRequest = new GetObjectRequest(config.getS3LogBucket(), s3FullFileName); String crashSummary = null;//w w w . ja va 2 s . co m try { S3Object s3Object = s3Client.getObject(objectRequest); InputStream is = s3Object.getObjectContent(); String fileContents = IOUtils.toString(is); CrashLogParser parser = null; if (fileContents != null) { if (s3CrashFileName.endsWith(".crash")) { parser = new iOSCrashLogParser(); if (parser.parseCrashLog(fileContents)) crashSummary = parser.getCrashSummary(); else { log.error("problem parsing ios crash file for app " + fullAppName + " file: " + s3CrashFileName); crashSummary = "Not able to get summary for iOS crash log"; } } else if (s3CrashFileName.endsWith(".stacktrace")) { parser = new AndroidCrashLogParser(); if (parser.parseCrashLog(fileContents)) crashSummary = parser.getCrashSummary(); else { log.error("problem parsing Android crash file for app " + fullAppName + " file: " + s3CrashFileName); crashSummary = "Not able to get summary for Android crash log"; } } } } catch (AmazonServiceException e1) { e1.printStackTrace(); log.error("Promblem downloading crash file from S3 for " + s3FullFileName, e1); } catch (Exception e) { e.printStackTrace(); log.error("Promblem downloading crash file from S3 for S3 for " + s3FullFileName, e); } log.info("Crash summary " + crashSummary); if (crashSummary != null && crashSummary.length() > 250) { crashSummary = crashSummary.substring(0, 249); } return crashSummary; }
From source file:org.broadleafcommerce.vendor.amazon.s3.S3FileServiceProvider.java
License:Apache License
@Override public File getResource(String name, FileApplicationType fileApplicationType) { final S3Configuration s3config = s3ConfigurationService.lookupS3Configuration(); final String resourceName = buildResourceName(s3config, name); final File returnFile = blFileService.getLocalResource(resourceName); final String s3Uri = String.format("s3://%s/%s", s3config.getDefaultBucketName(), resourceName); OutputStream outputStream = null; InputStream inputStream = null; try {//from ww w .j a va2 s.c o m final AmazonS3Client s3 = getAmazonS3Client(s3config); final S3Object object = s3.getObject( new GetObjectRequest(s3config.getDefaultBucketName(), buildResourceName(s3config, name))); if (LOG.isTraceEnabled()) { LOG.trace("retrieving " + s3Uri); } inputStream = object.getObjectContent(); if (!returnFile.getParentFile().exists()) { if (!returnFile.getParentFile().mkdirs()) { // Other thread could have created - check one more time. if (!returnFile.getParentFile().exists()) { throw new RuntimeException("Unable to create parent directories for file: " + name); } } } outputStream = new FileOutputStream(returnFile); int read = 0; byte[] bytes = new byte[1024]; while ((read = inputStream.read(bytes)) != -1) { outputStream.write(bytes, 0, read); } } catch (IOException ioe) { throw new RuntimeException(String.format("Error writing %s to local file system at %s", s3Uri, returnFile.getAbsolutePath()), ioe); } catch (AmazonS3Exception s3Exception) { LOG.error(String.format("%s for %s; name = %s, resourceName = %s, returnFile = %s", s3Exception.getErrorCode(), s3Uri, name, resourceName, returnFile.getAbsolutePath())); if ("NoSuchKey".equals(s3Exception.getErrorCode())) { //return new File("this/path/should/not/exist/" + UUID.randomUUID()); return null; } else { throw s3Exception; } } finally { if (inputStream != null) { try { inputStream.close(); } catch (IOException e) { throw new RuntimeException("Error closing input stream while writing s3 file to file system", e); } } if (outputStream != null) { try { outputStream.close(); } catch (IOException e) { throw new RuntimeException("Error closing output stream while writing s3 file to file system", e); } } } return returnFile; }
From source file:org.opencb.opencga.core.data.source.S3Source.java
License:Apache License
public InputStream getInputStream(String path) { String ak = "AKIAI3BZQ2VG6GPWQBVA"; String sk = "oDDIv+OAQeQVj9sy1CcWeeJsOMAhbh9KIpJ7hiDK"; String bucket = "nacho-s3"; AWSCredentials myCredentials = new BasicAWSCredentials(ak, sk); AmazonS3Client s3Client = new AmazonS3Client(myCredentials); S3Object object = s3Client.getObject(new GetObjectRequest(bucket, path)); return object.getObjectContent(); }
From source file:org.zalando.stups.fullstop.controller.S3Controller.java
License:Apache License
@RequestMapping(method = RequestMethod.GET, value = "/download") public void downloadFiles(@RequestParam(value = "bucket") final String bucket, @RequestParam(value = "location") final String location, @RequestParam(value = "page") final int page) { try {//from w ww . j a va 2 s . co m log.info("Creating fullstop directory here: {}", fullstopLoggingDir); boolean mkdirs = new File(fullstopLoggingDir).mkdirs(); } catch (SecurityException e) { // do nothing } AmazonS3Client amazonS3Client = new AmazonS3Client(); amazonS3Client.setRegion(Region.getRegion(Regions .fromName((String) cloudTrailProcessingLibraryProperties.getAsProperties().get(S3_REGION_KEY)))); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket) // .withPrefix(location) // .withMaxKeys(page); ObjectListing objectListing = amazonS3Client.listObjects(listObjectsRequest); final List<S3ObjectSummary> s3ObjectSummaries = objectListing.getObjectSummaries(); while (objectListing.isTruncated()) { objectListing = amazonS3Client.listNextBatchOfObjects(objectListing); s3ObjectSummaries.addAll(objectListing.getObjectSummaries()); } for (S3ObjectSummary s3ObjectSummary : s3ObjectSummaries) { String bucketName = s3ObjectSummary.getBucketName(); String key = s3ObjectSummary.getKey(); S3Object object = amazonS3Client.getObject(new GetObjectRequest(bucketName, key)); InputStream inputStream = object.getObjectContent(); File file = new File(fullstopLoggingDir, object.getBucketName() + object.getObjectMetadata().getETag() + JSON_GZ); copyInputStreamToFile(inputStream, file); log.info("File saved here: {}", file.getAbsolutePath()); } }