List of usage examples for com.amazonaws.services.s3 AmazonS3Client AmazonS3Client
@SdkInternalApi AmazonS3Client(AmazonS3ClientParams s3ClientParams)
From source file:ConnectionUtils.AWSS3Utils.java
public static String saveImage(String bucketName, ImageDTO imageDTO) { InputStream imageIS = null;//from ww w. j a v a 2 s .c o m Properties awsCredentialsProperties = new Properties(); try { awsCredentialsProperties .load(AWSS3Utils.class.getClassLoader().getResourceAsStream("prefs.properties")); } catch (Exception ex) { Logger.getLogger(AWSS3Utils.class.getName()).log(Level.SEVERE, null, ex); } String awsAccessKey = awsCredentialsProperties.getProperty("AWSACCESSKEY"); String awsSecretKey = awsCredentialsProperties.getProperty("SECRETACCESSKEY"); try { // AccessControlList accessControlList = new AccessControlList(); // accessControlList.grantPermission(GroupGrantee.AllUsers, Permission.FullControl); BasicAWSCredentials awsCredentials = new BasicAWSCredentials(awsAccessKey, awsSecretKey); AmazonS3 s3Client = new AmazonS3Client(awsCredentials); ObjectMetadata objectMetadata = new ObjectMetadata(); imageIS = imageDTO.getImageBlob(); PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, imageDTO.getIID() + ".jpg", imageIS, objectMetadata); s3Client.putObject(putObjectRequest);//.withAccessControlList(accessControlList)); imageIS = imageDTO.getThumbnailBlob(); putObjectRequest = new PutObjectRequest(bucketName, imageDTO.getIID() + "_" + IMAGE_TYPE.THUMB.toString() + ".jpg", imageIS, objectMetadata); s3Client.putObject(putObjectRequest);//.withAccessControlList(accessControlList)); return ("Image Saved"); } catch (Exception ex) { Logger.getLogger(AWSS3Utils.class.getName()).log(Level.SEVERE, null, ex); return "Image Not Saved"; } finally { try { if (imageIS != null) { imageIS.close(); } } catch (IOException ex) { Logger.getLogger(AWSS3Utils.class.getName()).log(Level.SEVERE, null, ex); } } }
From source file:ConnectionUtils.AWSS3Utils.java
public static ImageDTO loadImage(String bucketName, ImageDTO imageDTO, IMAGE_TYPE imageType) { Properties awsCredentialsProperties = new Properties(); try {/*ww w.j a v a 2 s .c om*/ awsCredentialsProperties .load(AWSS3Utils.class.getClassLoader().getResourceAsStream("prefs.properties")); } catch (Exception ex) { Logger.getLogger(AWSS3Utils.class.getName()).log(Level.SEVERE, null, ex); } String awsAccessKey = awsCredentialsProperties.getProperty("AWSACCESSKEY"); String awsSecretKey = awsCredentialsProperties.getProperty("SECRETACCESSKEY"); try { BasicAWSCredentials awsCredentials = new BasicAWSCredentials(awsAccessKey, awsSecretKey); AmazonS3 s3Client = new AmazonS3Client(awsCredentials); GetObjectRequest request = null; if (imageType.equals(IMAGE_TYPE.FULL)) { request = new GetObjectRequest(bucketName, imageDTO.getIID() + ".jpg"); } else if (imageType.equals(IMAGE_TYPE.THUMB)) { request = new GetObjectRequest(bucketName, imageDTO.getIID() + "_" + IMAGE_TYPE.THUMB.toString() + ".jpg"); } S3Object object = s3Client.getObject(request); S3ObjectInputStream objectContent = object.getObjectContent(); if (imageType.equals(IMAGE_TYPE.FULL)) { imageDTO.setImageBlob(objectContent); } else if (imageType.equals(IMAGE_TYPE.THUMB)) { imageDTO.setThumbnailBlob(objectContent); } return (imageDTO); } catch (Exception ex) { Logger.getLogger(AWSS3Utils.class.getName()).log(Level.SEVERE, null, ex); return null; } }
From source file:controllers.s3locationmodify.java
License:Open Source License
public static void main(String[] args) throws Exception { /*/*from ww w . j av a 2 s . c o m*/ * The ProfileCredentialsProvider will return your [default] * credential profile by reading from the credentials file located at * (/home/sravya/.aws/credentials). * * TransferManager manages a pool of threads, so we create a * single instance and share it throughout our application. */ try { credentials = new ProfileCredentialsProvider("default").getCredentials(); } catch (Exception e) { throw new AmazonClientException("Cannot load the credentials from the credential profiles file. " + "Please make sure that your credentials file is at the correct " + "location (/home/sravya/.aws/credentials), and is in valid format.", e); } int argLen = args.length; Region reg = Region.getRegion(Regions.US_WEST_2); int hack = 0; int userrequests; try { userrequests = Integer.parseInt(args[argLen - 1]); } catch (NumberFormatException e) { userrequests = 1; String use = args[argLen - 1]; if (use.equals("Australia")) { hack = 0; } else if (use.equals("SouthAfrica")) { hack = 1; } else if (use.equals("India")) { hack = 2; } else if (use.equals("UnitedKingdom")) { hack = 3; } else if (use.equals("China")) { hack = 4; } else if (use.equals("Germany")) { hack = 5; } else if (use.equals("France")) { hack = 6; } else if (use.equals("Japan")) { hack = 7; } else if (use.equals("Thailand")) { hack = 8; } else if (use.equals("Spain")) { hack = 9; } } int filecount = 0; for (int m = 0; m < argLen - 1; m++) { filecount = filecount + 1; } int numphotos = filecount; int numIDCs = numphotos; int locationInd = 0; long[] cusize = new long[6]; long photosspace = 0; for (int mm = 0; mm < userrequests; mm++) { for (int i = 0; i < cusize.length; i++) cusize[i] = 0; ArrayList<Integer> originalgarph = new ArrayList<Integer>(); loadObj ob = calculateload(); long[] regionload = new long[5]; regionload = ob.load; for (int i = 0; i < regionload.length; i++) { if (regionload[i] == 0) { regionload[i] = 1000; } } /*for (int i=0; i<5 ;i++) { System.out.println(regionload[i]); double diffload=0; double avgload=0; int count=0; for(int j=0;j<5; j++) { if(j!=i) { avgload = avgload + regionload[j]; count++; } } avgload= (avgload / count); diffload= (regionload[i]/avgload) ; System.out.println("avgload: "+ avgload); System.out.println("diffload: "+ diffload); if(diffload < 1.8) { originalgarph.add(i+1); } }*/ for (int i = 0; i < 5; i++) { originalgarph.add(i + 1); } availSpaceNorthCal = maxsize - regionload[0]; photosspace = numphotos * 6000; if (availSpaceNorthCal < photosspace) { availSpaceNorthCal = maxsize; cusize[1] = availSpaceNorthCal / 6000; } else { cusize[1] = availSpaceNorthCal / 6000; } availSpaceOregon = maxsize - regionload[1]; photosspace = numphotos * 6000; if (availSpaceOregon < photosspace) { availSpaceOregon = maxsize; cusize[2] = availSpaceOregon / 6000; } else { cusize[2] = availSpaceOregon / 6000; } availSpaceSingapore = maxsize - regionload[2]; photosspace = numphotos * 6000; if (availSpaceSingapore < photosspace) { availSpaceSingapore = maxsize; cusize[3] = availSpaceSingapore / 6000; } else { cusize[3] = availSpaceSingapore / 6000; } availSpaceTokyo = maxsize - regionload[3]; photosspace = numphotos * 6000; if (availSpaceTokyo < photosspace) { availSpaceTokyo = maxsize; cusize[4] = availSpaceTokyo / 6000; } else { cusize[4] = availSpaceTokyo / 6000; } availSpaceSydney = maxsize - regionload[4]; photosspace = numphotos * 6000; if (availSpaceSydney < photosspace) { availSpaceSydney = maxsize; cusize[5] = availSpaceSydney / 6000; } else { cusize[5] = availSpaceSydney / 6000; } int cou = originalgarph.size(); String fileName = algorithmPath + "request.alg"; PrintWriter writer = new PrintWriter(fileName, "UTF-8"); for (int i = 0; i < cou; i++) { int value = originalgarph.get(i); writer.print(value + " "); System.out.println(" IDC " + String.valueOf(value)); } writer.println(); for (int i = 0; i < originalgarph.size(); i++) { int j = originalgarph.get(i); writer.print(cusize[j] + " "); } writer.println(); writer.println(1); if (userrequests != 1) { locationInd = randInt(0, 9); } else { locationInd = hack; } writer.println(locationInd); System.out.println(" locationInd " + String.valueOf(locationInd)); writer.println(numphotos); System.out.println(" numphotos " + String.valueOf(numphotos)); writer.println(numIDCs); System.out.println(" numIDCs " + String.valueOf(numIDCs)); writer.close(); originalgarph.clear(); try { String prg = "import sys\nprint int(sys.argv[1])+int(sys.argv[2])\n"; String pythonCmd = "/usr/bin/python " + algorithmPath + "ramd.py"; Process p = Runtime.getRuntime().exec(pythonCmd); try { Thread.sleep(2000); //1000 milliseconds is one second. } catch (InterruptedException ex) { Thread.currentThread().interrupt(); } String fileName1 = algorithmPath + "work.alg"; File log = new File(fileName1); int filenumber = 0; String fileName2 = algorithmPath + "filenumber.alg"; Scanner numberscan = new Scanner(new File(fileName2)); if (numberscan.hasNextLine()) { filenumber = numberscan.nextInt(); } else { filenumber = 1; } numberscan.close(); ArrayList<String> photofnames = new ArrayList<String>(); ArrayList<String> argFNames = new ArrayList<String>(); for (int ll = 0; ll < argLen - 1; ll++) { photofnames.add(photosPath + args[ll] + ".bmp"); argFNames.add(args[ll]); System.out.println("Will upload " + photosPath + args[ll] + ".bmp"); } String sCurrentLine; BufferedReader br = null; br = new BufferedReader(new FileReader(fileName1)); int currLine = 0; Integer userNumber = 0; ArrayList<Integer> idcSet = new ArrayList<Integer>(); ArrayList<Integer> photonos = new ArrayList<Integer>(); int currU = 0; while ((sCurrentLine = br.readLine()) != null) { if (currLine % 3 == 0) { userNumber = Integer.parseInt(sCurrentLine); idcSet.clear(); photonos.clear(); } if (currLine % 3 == 1) { String[] idcnums = sCurrentLine.split(" "); String regions = ""; String regionsFileName = algorithmPath + "regions.alg"; PrintWriter regionwriter = new PrintWriter(regionsFileName, "UTF-8"); for (int numIdcs = 0; numIdcs < idcnums.length; numIdcs++) { idcSet.add(Integer.parseInt(idcnums[numIdcs])); if (idcnums[numIdcs].equals("1")) { regions = "region1"; regionwriter.print(regions + " "); } else if (idcnums[numIdcs].equals("2")) { regions = "region2"; regionwriter.print(regions + " "); } else if (idcnums[numIdcs].equals("3")) { regions = "region3"; regionwriter.print(regions + " "); } else if (idcnums[numIdcs].equals("4")) { regions = "region4"; regionwriter.print(regions + " "); } else if (idcnums[numIdcs].equals("5")) { regions = "region5"; regionwriter.print(regions + " "); } System.out.println("IDCs: " + idcnums[numIdcs]); } regionwriter.close(); } if (currLine % 3 == 2) { String[] idcpnums = sCurrentLine.split(" "); for (int numIdcs = 0; numIdcs < idcpnums.length; numIdcs++) { photonos.add(Integer.parseInt(idcpnums[numIdcs])); } ArrayList<String> smallestBnames = new ArrayList<String>(); ArrayList<String> bucketnames = new ArrayList<String>(); for (int tot = 0; tot < idcSet.size(); tot++) { smallestBnames.add(ob.bnames[idcSet.get(tot) - 1]); } AmazonS3 s3client = new AmazonS3Client(new ProfileCredentialsProvider()); int currPno = 0; ArrayList<String> transferThese = new ArrayList<String>(); ArrayList<Integer> transferThesebno = new ArrayList<Integer>(); System.out.println(String.valueOf(idcpnums.length)); //upload everything to 1 bucket for (int numIdcs = 0; numIdcs < idcpnums.length; numIdcs++) { for (int numP = 0; numP < photonos.get(numIdcs); numP++) { String uploadFileName = photofnames.get(currPno); String keyName = String.valueOf(userNumber) + "_" + argFNames.get(currPno) + '_' + filenumber++ + ".bmp"; if (numIdcs > 0) { transferThese.add(keyName); transferThesebno.add(numIdcs); } try { System.out.println("Uploading " + uploadFileName + " to " + smallestBnames.get(0) + " with keyname " + keyName); File file = new File(uploadFileName); s3client.putObject(new PutObjectRequest(smallestBnames.get(0), keyName, file)); } catch (AmazonServiceException ase) { System.out.println("Caught an AmazonServiceException, which " + "means your request made it " + "to Amazon S3, but was rejected with an error response" + " for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } currPno++; } } //transfer other files System.out.println("Number of files to transfer " + String.valueOf(transferThese.size())); for (int tot = 0; tot < transferThese.size(); tot++) { String source = smallestBnames.get(0); String dest = smallestBnames.get(transferThesebno.get(tot)); String fname = transferThese.get(tot); String src = "s3://" + source + "/" + fname; String d = "s3://" + dest; String cmd = "aws s3 mv " + src + " " + d + "\n"; System.out.println("Moving " + src + " to " + d + "\n"); Process p1 = Runtime.getRuntime().exec(cmd); } currU++; if (currU >= 1) { transferThese.clear(); transferThesebno.clear(); photofnames.clear(); argFNames.clear(); smallestBnames.clear(); break; } } currLine++; } String fileNumberFilePath = algorithmPath + "filenumber.alg"; PrintWriter numberwriter = new PrintWriter(fileNumberFilePath, "UTF-8"); numberwriter.println(filenumber); numberwriter.close(); } catch (Exception e) { } } // String s = null; // Process p1 = Runtime.getRuntime().exec("ls -alrt"); // // BufferedReader stdInput = new BufferedReader(new // InputStreamReader(p1.getInputStream())); // // BufferedReader stdError = new BufferedReader(new // InputStreamReader(p1.getErrorStream())); // // // read the output from the command // System.out.println("Here is the standard output of the command:\n"); // while ((s = stdInput.readLine()) != null) { // System.out.println(s); // } }
From source file:controllers.s3locationmodify.java
License:Open Source License
private static long calculateregionload(String regionname) { AmazonS3 s3 = new AmazonS3Client(credentials); long size = 0; AmazonS3 s3client = new AmazonS3Client(new ProfileCredentialsProvider()); //try {/*from ww w .j a v a 2 s .c om*/ ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(regionname); ObjectListing objectListing; do { objectListing = s3client.listObjects(listObjectsRequest); for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { size = objectSummary.getSize(); } listObjectsRequest.setMarker(objectListing.getNextMarker()); } while (objectListing.isTruncated()); return size; }
From source file:controllers.s3modify.java
License:Open Source License
public static void main(String[] args) throws Exception { /*/* w ww . ja v a 2 s .co m*/ * The ProfileCredentialsProvider will return your [default] * credential profile by reading from the credentials file located at * (/home/sravya/.aws/credentials). * * TransferManager manages a pool of threads, so we create a * single instance and share it throughout our application. */ try { credentials = new ProfileCredentialsProvider("default").getCredentials(); } catch (Exception e) { throw new AmazonClientException("Cannot load the credentials from the credential profiles file. " + "Please make sure that your credentials file is at the correct " + "location (/home/sravya/.aws/credentials), and is in valid format.", e); } int argLen = args.length; Region reg = Region.getRegion(Regions.US_WEST_2); int hack = 0; int userrequests; try { userrequests = Integer.parseInt(args[argLen - 1]); } catch (NumberFormatException e) { userrequests = 1; String use = args[argLen - 1]; if (use.equals("Australia")) { hack = 0; } else if (use.equals("SouthAfrica")) { hack = 1; } else if (use.equals("India")) { hack = 2; } else if (use.equals("UnitedKingdom")) { hack = 3; } else if (use.equals("China")) { hack = 4; } else if (use.equals("Germany")) { hack = 5; } else if (use.equals("France")) { hack = 6; } else if (use.equals("Japan")) { hack = 7; } else if (use.equals("Thailand")) { hack = 8; } else if (use.equals("Spain")) { hack = 9; } } int filecount = 0; for (int m = 0; m < argLen - 1; m++) { filecount = filecount + 1; } int numphotos = filecount; int numIDCs = numphotos; int locationInd = 0; long[] cusize = new long[6]; long photosspace = 0; for (int mm = 0; mm < userrequests; mm++) { for (int i = 0; i < cusize.length; i++) cusize[i] = 0; ArrayList<Integer> originalgarph = new ArrayList<Integer>(); loadObj ob = calculateload(); long[] regionload = new long[5]; regionload = ob.load; for (int i = 0; i < regionload.length; i++) { if (regionload[i] == 0) { regionload[i] = 1000; } } for (int i = 0; i < 5; i++) { System.out.println(regionload[i]); double diffload = 0; double avgload = 0; int count = 0; for (int j = 0; j < 5; j++) { if (j != i) { avgload = avgload + regionload[j]; count++; } } avgload = (avgload / count); diffload = (regionload[i] / avgload); System.out.println("avgload: " + avgload); System.out.println("diffload: " + diffload); if (diffload < 1.8) { originalgarph.add(i + 1); } } availSpaceNorthCal = maxsize - regionload[0]; photosspace = numphotos * 6000; if (availSpaceNorthCal < photosspace) { availSpaceNorthCal = maxsize; cusize[1] = availSpaceNorthCal / 6000; } else { cusize[1] = availSpaceNorthCal / 6000; } availSpaceOregon = maxsize - regionload[1]; photosspace = numphotos * 6000; if (availSpaceOregon < photosspace) { availSpaceOregon = maxsize; cusize[2] = availSpaceOregon / 6000; } else { cusize[2] = availSpaceOregon / 6000; } availSpaceSingapore = maxsize - regionload[2]; photosspace = numphotos * 6000; if (availSpaceSingapore < photosspace) { availSpaceSingapore = maxsize; cusize[3] = availSpaceSingapore / 6000; } else { cusize[3] = availSpaceSingapore / 6000; } availSpaceTokyo = maxsize - regionload[3]; photosspace = numphotos * 6000; if (availSpaceTokyo < photosspace) { availSpaceTokyo = maxsize; cusize[4] = availSpaceTokyo / 6000; } else { cusize[4] = availSpaceTokyo / 6000; } availSpaceSydney = maxsize - regionload[4]; photosspace = numphotos * 6000; if (availSpaceSydney < photosspace) { availSpaceSydney = maxsize; cusize[5] = availSpaceSydney / 6000; } else { cusize[5] = availSpaceSydney / 6000; } int cou = originalgarph.size(); String fileName = algorithmPath + "request.alg"; PrintWriter writer = new PrintWriter(fileName, "UTF-8"); for (int i = 0; i < cou; i++) { int value = originalgarph.get(i); writer.print(value + " "); System.out.println(" IDC " + String.valueOf(value)); } writer.println(); for (int i = 0; i < originalgarph.size(); i++) { int j = originalgarph.get(i); writer.print(cusize[j] + " "); } writer.println(); writer.println(1); if (userrequests != 1) { locationInd = randInt(0, 9); } else { locationInd = hack; } writer.println(locationInd); System.out.println(" locationInd " + String.valueOf(locationInd)); writer.println(numphotos); System.out.println(" numphotos " + String.valueOf(numphotos)); writer.println(numIDCs); System.out.println(" numIDCs " + String.valueOf(numIDCs)); writer.close(); originalgarph.clear(); try { String prg = "import sys\nprint int(sys.argv[1])+int(sys.argv[2])\n"; String pythonCmd = "/usr/bin/python " + algorithmPath + "ramd.py"; Process p = Runtime.getRuntime().exec(pythonCmd); try { Thread.sleep(2000); //1000 milliseconds is one second. } catch (InterruptedException ex) { Thread.currentThread().interrupt(); } String fileName1 = algorithmPath + "work.alg"; File log = new File(fileName1); int filenumber = 0; String fileName2 = algorithmPath + "filenumber.alg"; Scanner numberscan = new Scanner(new File(fileName2)); if (numberscan.hasNextLine()) { filenumber = numberscan.nextInt(); } else { filenumber = 1; } numberscan.close(); ArrayList<String> photofnames = new ArrayList<String>(); ArrayList<String> argFNames = new ArrayList<String>(); for (int ll = 0; ll < argLen - 1; ll++) { photofnames.add(photosPath + args[ll] + ".bmp"); argFNames.add(args[ll]); System.out.println("Will upload " + photosPath + args[ll] + ".bmp"); } String sCurrentLine; BufferedReader br = null; br = new BufferedReader(new FileReader(fileName1)); int currLine = 0; Integer userNumber = 0; ArrayList<Integer> idcSet = new ArrayList<Integer>(); ArrayList<Integer> photonos = new ArrayList<Integer>(); int currU = 0; while ((sCurrentLine = br.readLine()) != null) { if (currLine % 3 == 0) { userNumber = Integer.parseInt(sCurrentLine); idcSet.clear(); photonos.clear(); } if (currLine % 3 == 1) { String[] idcnums = sCurrentLine.split(" "); String regions = ""; String regionsFileName = algorithmPath + "regions.alg"; PrintWriter regionwriter = new PrintWriter(regionsFileName, "UTF-8"); for (int numIdcs = 0; numIdcs < idcnums.length; numIdcs++) { idcSet.add(Integer.parseInt(idcnums[numIdcs])); if (idcnums[numIdcs].equals("1")) { regions = "region1"; regionwriter.print(regions + " "); } else if (idcnums[numIdcs].equals("2")) { regions = "region2"; regionwriter.print(regions + " "); } else if (idcnums[numIdcs].equals("3")) { regions = "region3"; regionwriter.print(regions + " "); } else if (idcnums[numIdcs].equals("4")) { regions = "region4"; regionwriter.print(regions + " "); } else if (idcnums[numIdcs].equals("5")) { regions = "region5"; regionwriter.print(regions + " "); } System.out.println("IDCs: " + idcnums[numIdcs]); } regionwriter.close(); } if (currLine % 3 == 2) { String[] idcpnums = sCurrentLine.split(" "); for (int numIdcs = 0; numIdcs < idcpnums.length; numIdcs++) { photonos.add(Integer.parseInt(idcpnums[numIdcs])); } ArrayList<String> smallestBnames = new ArrayList<String>(); ArrayList<String> bucketnames = new ArrayList<String>(); for (int tot = 0; tot < idcSet.size(); tot++) { smallestBnames.add(ob.bnames[idcSet.get(tot) - 1]); } AmazonS3 s3client = new AmazonS3Client(new ProfileCredentialsProvider()); int currPno = 0; ArrayList<String> transferThese = new ArrayList<String>(); ArrayList<Integer> transferThesebno = new ArrayList<Integer>(); System.out.println(String.valueOf(idcpnums.length)); //upload everything to 1 bucket for (int numIdcs = 0; numIdcs < idcpnums.length; numIdcs++) { for (int numP = 0; numP < photonos.get(numIdcs); numP++) { String uploadFileName = photofnames.get(currPno); String keyName = String.valueOf(userNumber) + "_" + argFNames.get(currPno) + '_' + filenumber++ + ".bmp"; if (numIdcs > 0) { transferThese.add(keyName); transferThesebno.add(numIdcs); } try { System.out.println("Uploading " + uploadFileName + " to " + smallestBnames.get(0) + " with keyname " + keyName); File file = new File(uploadFileName); s3client.putObject(new PutObjectRequest(smallestBnames.get(0), keyName, file)); } catch (AmazonServiceException ase) { System.out.println("Caught an AmazonServiceException, which " + "means your request made it " + "to Amazon S3, but was rejected with an error response" + " for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } currPno++; } } //transfer other files System.out.println("Number of files to transfer " + String.valueOf(transferThese.size())); for (int tot = 0; tot < transferThese.size(); tot++) { String source = smallestBnames.get(0); String dest = smallestBnames.get(transferThesebno.get(tot)); String fname = transferThese.get(tot); String src = "s3://" + source + "/" + fname; String d = "s3://" + dest; String cmd = "aws s3 mv " + src + " " + d + "\n"; System.out.println("Moving " + src + " to " + d + "\n"); Process p1 = Runtime.getRuntime().exec(cmd); } currU++; if (currU >= 1) { transferThese.clear(); transferThesebno.clear(); photofnames.clear(); argFNames.clear(); smallestBnames.clear(); break; } } currLine++; } String fileNumberFilePath = algorithmPath + "filenumber.alg"; PrintWriter numberwriter = new PrintWriter(fileNumberFilePath, "UTF-8"); numberwriter.println(filenumber); numberwriter.close(); } catch (Exception e) { } } // String s = null; // Process p1 = Runtime.getRuntime().exec("ls -alrt"); // // BufferedReader stdInput = new BufferedReader(new // InputStreamReader(p1.getInputStream())); // // BufferedReader stdError = new BufferedReader(new // InputStreamReader(p1.getErrorStream())); // // // read the output from the command // System.out.println("Here is the standard output of the command:\n"); // while ((s = stdInput.readLine()) != null) { // System.out.println(s); // } }
From source file:core.connector.s3.sync.S3Connector.java
License:GNU General Public License
public void open() throws ConnectorException { try {//from ww w . ja va 2 s .c o m s3 = new AmazonS3Client(new SimpleAWSCredentials(info.getAccessId(), info.getSecretKey())); s3.setEndpoint(info.getBucketEndpoint()); } catch (Exception e) { throw new ConnectorException(e); } }
From source file:dashboard.AmazonLogs.java
License:Open Source License
public int readAmazonLogs(int n, String AWS_USER, String AWS_PASS, String IPfile, String ERRfile, String bucketName, String DELETE_PROCESSED_LOGS, String API_KEY, String TOKEN, String apiuser, String apipass) throws Exception { if (n < 1) return 0; int eventsNumber = 0; String line = null;//from w ww.j av a 2s . co m int begin = 0; int zips = 0; int deletedZips = 0; int mixpanelStatus = 0; String registrant = ""; String ip = ""; String prevIP = ""; Mixpanel mix = new Mixpanel(); Whois w = new Whois(apiuser, apipass); int index = -1; Registrant r; ArrayList<Registrant> rList = new ArrayList<Registrant>(); ArrayList<Registrant> eList = new ArrayList<Registrant>(); IPList ipl = new IPList(); IPList errl = new IPList(); // Log files Bucket AWSCredentials credentials = new BasicAWSCredentials(AWS_USER, AWS_PASS); AmazonS3Client s3Client = new AmazonS3Client(credentials); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName); BufferedReader br = null; ObjectListing objectListing = s3Client.listObjects(listObjectsRequest); ObjectListing nextObjectListing = objectListing; zips = 0; Boolean more = true; if (objectListing == null) more = false; else { ipl.loadList(rList, IPfile); ipl.printList(rList, 30); } while (more) { // Reads 1000 files for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { // Handle ZIP files try { // Open and send to mixpanel events of one ZIP file String key = objectSummary.getKey(); S3Object object = s3Client.getObject(new GetObjectRequest(bucketName, key)); // Extract ZIP and read Object to reader br = new BufferedReader(new InputStreamReader(new GZIPInputStream(object.getObjectContent()))); zips++; // Read the lines from the unzipped file, break it and send to Mixpanel while ((line = br.readLine()) != null) { if (line.startsWith("#")) continue; if (line.trim().equals("")) continue; String[] values = line.split("\\s"); String eventTime = values[0] + " " + values[1]; ip = values[4]; if (ip != prevIP) { prevIP = ip; index = ipl.ipInList(ip, rList); if (index >= 0) { r = rList.get(index); registrant = r.name; // Update counter for this IP r.counter = r.counter + 1; rList.set(index, r); } else { // WHOIS - Check registrant of this IP address registrant = w.whoisIP(ip); // if there was an error, try again if (registrant.equals("ERROR")) registrant = w.whoisIP(ip); // if there was a second error, add it to errors list if (registrant.equals("ERROR")) { eList.add(new Registrant(ip, registrant, 1)); } else { // If name includes a comma, exclude the comma registrant = registrant.replace(",", ""); rList.add(new Registrant(ip, registrant, 1)); } } } String method = values[5]; String fileName = values[7]; String statusCode = values[8]; String userAgent = values[10]; String fName = fileName; if (fileName.contains("gigaspaces-")) { begin = fileName.lastIndexOf("gigaspaces-") + 11; fName = fileName.substring(begin, fileName.length()); } eventsNumber++; System.out.println(eventsNumber + ": " + eventTime + " " + ip + " " + registrant); // ==================================================== // Track the event in Mixpanel (using the POST import) // ==================================================== mixpanelStatus = mix.postCDNEventToMixpanel(API_KEY, TOKEN, ip, "Cloudfront CDN", eventTime, method, fileName, fName, userAgent, statusCode, registrant); } // while on ZIP file lines if (mixpanelStatus == 1 & DELETE_PROCESSED_LOGS.equals("YES")) { // Delete the CDN log ZIP file s3Client.deleteObject(bucketName, key); System.out.println("========= Deleted Zip " + zips + " ===== List Size " + rList.size() + " =========="); deletedZips++; } } catch (IOException e) { e.printStackTrace(); return eventsNumber; } finally { if (br != null) { br.close(); } if (eventsNumber >= n) { System.out.println("\n>>> " + eventsNumber + " events in " + zips + " Zip files. Deleted " + deletedZips + " Zip files.\n"); ipl.printList(rList, 100); ipl.saveList(rList, IPfile); if (!eList.isEmpty()) { SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd-HH-mm"); String fName = ERRfile + sdf.format(new Date()) + ".txt"; System.out.println("\n>>> " + eList.size() + " DomainTools errors:"); errl.saveList(eList, fName); } else System.out.println("\n>>> No DomainTools errors"); return eventsNumber; } } } // for (continue to next ZIP file // If there are more ZIP files, read next batch of 1000 if (objectListing.isTruncated()) { nextObjectListing = s3Client.listNextBatchOfObjects(objectListing); objectListing = nextObjectListing; } else more = false; // no more files } // while next objectListing System.out.println("\n>>> " + eventsNumber + " events in " + zips + " Zip files. Deleted " + deletedZips + " Zip files.\n"); ipl.printList(rList, 50); ipl.saveList(rList, IPfile); if (!eList.isEmpty()) { SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd-HH-mm"); String fName = ERRfile + sdf.format(new Date()) + ".txt"; System.out.println("\n>>> " + eList.size() + " DomainTools errors:"); errl.saveList(eList, fName); } else System.out.println("\n>>> No DomainTools errors"); return eventsNumber; }
From source file:dashboard.ImportCDN.java
License:Open Source License
public static int readAmazonLogs(int n) throws Exception { if (n < 1) return 0; int eventsNumber = 0; int begin = 0; int zips = 0; int postFailures = 0; int deletedZips = 0; int mixpanelStatus = 0; String line = null;// w ww . java 2 s .co m // Log files Bucket AWSCredentials credentials = new BasicAWSCredentials(AWS_USER, AWS_PASS); AmazonS3Client s3Client = new AmazonS3Client(credentials); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName); // Set MARKER - from which Log File to start reading // listObjectsRequest.setMarker("E2DXXJR0N8BXOK.2013-03-18-10.ICK6IvaY.gz"); BufferedReader br = null; ObjectListing objectListing = s3Client.listObjects(listObjectsRequest); ObjectListing nextObjectListing = objectListing; zips = 0; Boolean more = true; if (objectListing == null) more = false; while (more) { // Reads 1000 files for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { // Handle ZIP files try { // Open and send to mixpanel events of one ZIP file String key = objectSummary.getKey(); S3Object object = s3Client.getObject(new GetObjectRequest(bucketName, key)); // Extract ZIP and read Object to reader br = new BufferedReader(new InputStreamReader(new GZIPInputStream(object.getObjectContent()))); zips++; // Read the lines from the unzipped file, break it and send to Mixpanel while ((line = br.readLine()) != null) { if (line.startsWith("#")) continue; if (line.trim().equals("")) continue; String[] values = line.split("\\s"); String eventTime = values[0] + " " + values[1]; String ip = values[4]; String method = values[5]; String fileName = values[7]; String statusCode = values[8]; String userAgent = values[10]; String fName = fileName; if (fileName.contains("gigaspaces-")) { begin = fileName.lastIndexOf("gigaspaces-") + 11; fName = fileName.substring(begin, fileName.length()); } eventsNumber++; System.out.println(eventsNumber + ": " + eventTime + " " + ip); // Track the event in Mixpanel (using the POST import) mixpanelStatus = postCDNEventToMixpanel(ip, "Cloudfront CDN", eventTime, method, fileName, fName, userAgent, statusCode); // If failed if (mixpanelStatus != 1) { postFailures++; System.out.println(" >>> POST event to Mixpanel Failed!! " + postFailures); } } // while on ZIP file lines if (mixpanelStatus == 1 & DELETE_PROCESSED_LOGS.equals("YES")) { // Delete the CDN log ZIP file s3Client.deleteObject(bucketName, key); System.out.println("============ Deleted Zip " + zips + " ============"); deletedZips++; } else System.out.println("============ Zip " + zips + " (not deleted) ============"); } catch (IOException e) { e.printStackTrace(); return eventsNumber; } finally { if (br != null) { br.close(); } if (eventsNumber >= n) { System.out.println("\n>>> " + eventsNumber + " events in " + zips + " Zip files."); System.out.println("\n>>> " + deletedZips + " Zip files deleted."); System.out.println("\n>>> " + postFailures + " post Failures\n"); return eventsNumber; } } } // for (continue to next ZIP file // If there are more ZIP files, read next batch of 1000 if (objectListing.isTruncated()) { nextObjectListing = s3Client.listNextBatchOfObjects(objectListing); objectListing = nextObjectListing; } else more = false; // no more files } // while next objectListing System.out.println("\n>>> " + eventsNumber + " events in " + zips + " Zip files."); System.out.println("\n>>> " + deletedZips + " Zip files deleted."); System.out.println("\n>>> " + postFailures + " post Failures\n"); return eventsNumber; }
From source file:dataMappers.PictureDataMapper.java
public static void addPictureToReport(DBConnector dbconnector, HttpServletRequest request) throws FileUploadException, IOException, SQLException { if (!ServletFileUpload.isMultipartContent(request)) { System.out.println("Invalid upload request"); return;/*w w w .j av a 2s.c o m*/ } // Define limits for disk item DiskFileItemFactory factory = new DiskFileItemFactory(); factory.setSizeThreshold(THRESHOLD_SIZE); // Define limit for servlet upload ServletFileUpload upload = new ServletFileUpload(factory); upload.setFileSizeMax(MAX_FILE_SIZE); upload.setSizeMax(MAX_REQUEST_SIZE); FileItem itemFile = null; int reportID = 0; // Get list of items in request (parameters, files etc.) List formItems = upload.parseRequest(request); Iterator iter = formItems.iterator(); // Loop items while (iter.hasNext()) { FileItem item = (FileItem) iter.next(); if (!item.isFormField()) { itemFile = item; // If not form field, must be item } else if (item.getFieldName().equalsIgnoreCase("reportID")) { // else it is a form field try { System.out.println(item.getString()); reportID = Integer.parseInt(item.getString()); } catch (NumberFormatException e) { reportID = 0; } } } // This will be null if no fields were declared as image/upload. // Also, reportID must be > 0 if (itemFile != null || reportID == 0) { try { // Create credentials from final vars BasicAWSCredentials awsCredentials = new BasicAWSCredentials(AMAZON_ACCESS_KEY, AMAZON_SECRET_KEY); // Create client with credentials AmazonS3 s3client = new AmazonS3Client(awsCredentials); // Set region s3client.setRegion(Region.getRegion(Regions.EU_WEST_1)); // Set content length (size) of file ObjectMetadata om = new ObjectMetadata(); om.setContentLength(itemFile.getSize()); // Get extension for file String ext = FilenameUtils.getExtension(itemFile.getName()); // Generate random filename String keyName = UUID.randomUUID().toString() + '.' + ext; // This is the actual upload command s3client.putObject(new PutObjectRequest(S3_BUCKET_NAME, keyName, itemFile.getInputStream(), om)); // Picture was uploaded to S3 if we made it this far. Now we insert the row into the database for the report. PreparedStatement stmt = dbconnector.getCon() .prepareStatement("INSERT INTO reports_pictures" + "(REPORTID, PICTURE) VALUES (?,?)"); stmt.setInt(1, reportID); stmt.setString(2, keyName); stmt.executeUpdate(); stmt.close(); } catch (AmazonServiceException ase) { System.out.println("Caught an AmazonServiceException, which " + "means your request made it " + "to Amazon S3, but was rejected with an error response" + " for some reason."); System.out.println("Error Message: " + ase.getMessage()); System.out.println("HTTP Status Code: " + ase.getStatusCode()); System.out.println("AWS Error Code: " + ase.getErrorCode()); System.out.println("Error Type: " + ase.getErrorType()); System.out.println("Request ID: " + ase.getRequestId()); } catch (AmazonClientException ace) { System.out.println("Caught an AmazonClientException, which " + "means the client encountered " + "an internal error while trying to " + "communicate with S3, " + "such as not being able to access the network."); System.out.println("Error Message: " + ace.getMessage()); } } }
From source file:datameer.awstasks.ant.s3.S3Task.java
License:Apache License
public AmazonS3Client createS3Service() { BasicAWSCredentials awsCredentials = new BasicAWSCredentials(_accessKey, _accessSecret); return new AmazonS3Client(awsCredentials); }