Java tutorial
/* * Copyright (C) 2000 - 2014 TagServlet Ltd * * This file is part of Open BlueDragon (OpenBD) CFML Server Engine. * * OpenBD is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * Free Software Foundation,version 3. * * OpenBD is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with OpenBD. If not, see http://www.gnu.org/licenses/ * * Additional permission under GNU GPL version 3 section 7 * * If you modify this Program, or any covered work, by linking or combining * it with any of the JARS listed in the README.txt (or a modified version of * (that library), containing parts covered by the terms of that JAR, the * licensors of this Program grant you additional permission to convey the * resulting work. * README.txt @ http://www.openbluedragon.org/license/README.txt * * http://openbd.org/ * $Id: BackgroundUploader.java 2474 2015-01-13 15:24:17Z alan $ */ package org.alanwilliamson.amazon.s3; import java.io.File; import java.util.Date; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; import org.alanwilliamson.amazon.AmazonBase; import org.alanwilliamson.amazon.AmazonKey; import org.aw20.io.FileUtil; import org.aw20.util.DateUtil; import com.amazonaws.AmazonClientException; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.PutObjectRequest; import com.amazonaws.services.s3.model.SSECustomerKey; import com.amazonaws.services.s3.model.StorageClass; import com.amazonaws.services.s3.transfer.TransferManager; import com.amazonaws.services.s3.transfer.Upload; import com.bluedragon.plugin.ObjectCFC; import com.bluedragon.plugin.PluginManager; import com.nary.io.FileUtils; import com.naryx.tagfusion.cfm.application.cfAPPLICATION; import com.naryx.tagfusion.cfm.application.cfApplicationData; import com.naryx.tagfusion.cfm.engine.cfEngine; import com.naryx.tagfusion.cfm.engine.cfSession; import com.naryx.tagfusion.cfm.engine.cfmRunTimeException; import com.naryx.tagfusion.cfm.engine.variableStore; public class BackgroundUploader extends Thread { private static BackgroundUploader thisInst = null; public static void onStart() { try { thisInst = new BackgroundUploader(FileUtils.checkAndCreateDirectory( cfEngine.thisPlatform.getFileIO().getWorkingDirectory(), "amazons3uploader", false)); } catch (Exception E) { cfEngine.log("AmazonS3Write.BackgroundUploader failed to create all the CFMAIL spooling directorys: " + cfEngine.thisPlatform.getFileIO().getWorkingDirectory() + "/amazons3uploader"); } } public static void onShutdown() { thisInst.bRunning = false; thisInst.interrupt(); try { thisInst.join(); } catch (InterruptedException e) { } } public static void acceptFile(AmazonKey amazonKey, String bucket, String key, Map<String, String> metadata, StorageClass storage, String localpath, int retry, int retryseconds, boolean deletefile, String callback, String callbackdata, String appname, String acl, String aes256key, Map<String, String> customheaders) { if (thisInst == null) { cfEngine.log( "AmazonS3Write.BackgroundUploader not active due to missing directory. File failed to be uploaded"); return; } Map<String, Object> jobFile = new HashMap<String, Object>(); jobFile.put("id", com.nary.util.UUID.generateKey()); jobFile.put("amazonkey", amazonKey); jobFile.put("bucket", bucket); jobFile.put("key", key); jobFile.put("storage", storage); jobFile.put("localpath", localpath); jobFile.put("retry", retry); jobFile.put("retryms", retryseconds * 1000); jobFile.put("deletefile", deletefile); if (metadata != null && !metadata.isEmpty()) jobFile.put("metadata", metadata); if (acl != null && !acl.isEmpty()) jobFile.put("acl", acl); if (aes256key != null && !aes256key.isEmpty()) jobFile.put("aes256key", aes256key); if (callback != null && !callback.isEmpty()) jobFile.put("callback", callback); if (callbackdata != null && !callbackdata.isEmpty()) jobFile.put("callbackdata", callbackdata); if (customheaders != null && !customheaders.isEmpty()) jobFile.put("customheaders", customheaders); if (appname != null) jobFile.put("appname", appname); jobFile.put("attempt", 0); jobFile.put("attemptdate", System.currentTimeMillis() - 1000); thisInst.acceptFile(jobFile); } private File workingDirectory; private List<Map<String, Object>> fileList; private boolean bRunning = true; private BackgroundUploader(File workingDirectory) { super("AmazonS3BackgroundUploader"); this.workingDirectory = workingDirectory; fileList = new LinkedList<Map<String, Object>>(); // Load the previous files File[] jobsDisk = workingDirectory.listFiles(); if (jobsDisk != null && jobsDisk.length > 0) { for (int x = 0; x < jobsDisk.length; x++) { if (!jobsDisk[x].getName().endsWith(".job")) continue; @SuppressWarnings("unchecked") Map<String, Object> jobFile = (Map<String, Object>) FileUtil.loadClass(jobsDisk[x]); if (jobFile == null) { jobsDisk[x].delete(); } else { fileList.add(jobFile); } } if (!fileList.isEmpty()) { cfEngine.log("AmazonS3Write.BackgroundUploader files to upload=" + fileList.size()); } } setPriority(MIN_PRIORITY); setDaemon(true); start(); } private void acceptFile(Map<String, Object> jobFile) { saveFile(jobFile); synchronized (fileList) { fileList.add(jobFile); fileList.notify(); } } public void run() { while (bRunning) { // Wait until we have a file in the list while (fileList.isEmpty()) { try { synchronized (fileList) { fileList.wait(60 * 1000); } } catch (InterruptedException e) { break; } } Map<String, Object> jobFile = takeNextJob(); if (jobFile == null) { /** * If we get here; then we have no jobs that are ready to be sent to S3 yet; their retrytimeout hasn't * been expired. */ try { sleep(1000); } catch (InterruptedException e) { break; } } else { uploadFile(jobFile); } } cfEngine.log("AmazonS3Write.BackgroundUploader: Shutdown"); } private void uploadFile(Map<String, Object> jobFile) { File localFile = new File((String) jobFile.get("localpath")); if (!localFile.isFile()) { removeJobFile(jobFile); callbackCfc(jobFile, false, "local file no longer exists"); cfEngine.log("AmazonS3Write.BackgroundUploader: file no longer exists=" + localFile.getName()); return; } // Setup the object data ObjectMetadata omd = new ObjectMetadata(); if (jobFile.containsKey("metadata")) omd.setUserMetadata((Map<String, String>) jobFile.get("metadata")); TransferManager tm = null; AmazonS3 s3Client = null; try { AmazonKey amazonKey = (AmazonKey) jobFile.get("amazonkey"); s3Client = new AmazonBase().getAmazonS3(amazonKey); PutObjectRequest por = new PutObjectRequest((String) jobFile.get("bucket"), (String) jobFile.get("key"), localFile); por.setMetadata(omd); por.setStorageClass((StorageClass) jobFile.get("storage")); if (jobFile.containsKey("acl")) por.setCannedAcl(amazonKey.getAmazonCannedAcl((String) jobFile.get("acl"))); if (jobFile.containsKey("aes256key")) por.setSSECustomerKey(new SSECustomerKey((String) jobFile.get("aes256key"))); if (jobFile.containsKey("customheaders")) { Map<String, String> customheaders = (Map) jobFile.get("customheaders"); Iterator<String> it = customheaders.keySet().iterator(); while (it.hasNext()) { String k = it.next(); por.putCustomRequestHeader(k, customheaders.get(k)); } } long startTime = System.currentTimeMillis(); tm = new TransferManager(s3Client); Upload upload = tm.upload(por); upload.waitForCompletion(); log(jobFile, "Uploaded; timems=" + (System.currentTimeMillis() - startTime)); removeJobFile(jobFile); callbackCfc(jobFile, true, null); if ((Boolean) jobFile.get("deletefile")) localFile.delete(); } catch (Exception e) { log(jobFile, "Failed=" + e.getMessage()); callbackCfc(jobFile, false, e.getMessage()); int retry = (Integer) jobFile.get("retry"); int attempt = (Integer) jobFile.get("attempt") + 1; if (retry == attempt) { removeJobFile(jobFile); } else { jobFile.put("attempt", attempt); jobFile.put("attemptdate", System.currentTimeMillis() + (Long) jobFile.get("retryms")); acceptFile(jobFile); } if (s3Client != null) cleanupMultiPartUploads(s3Client, (String) jobFile.get("bucket")); } finally { if (tm != null) tm.shutdownNow(true); } } private void cleanupMultiPartUploads(AmazonS3 s3Client, String bucket) { TransferManager tm = new TransferManager(s3Client); try { tm.abortMultipartUploads(bucket, new Date(System.currentTimeMillis() - DateUtil.DAY_MS)); } catch (AmazonClientException amazonClientException) { cfEngine.log("AmazonS3Write.BackgroundUploader.cleanupMultiPartUploads():" + amazonClientException.getMessage()); } tm.shutdownNow(true); } private void callbackCfc(Map<String, Object> jobFile, boolean success, String errMessage) { String callback = (String) jobFile.get("callback"); if (callback == null) return; String callbackdata = (String) jobFile.get("callbackdata"); if (callbackdata == null) callbackdata = ""; String appname = (String) jobFile.get("appname"); final cfSession tmpSession = PluginManager.getPlugInManager().createBlankSession(); if (appname != null) { cfApplicationData appData = cfAPPLICATION.getAppManager().getAppData(tmpSession, appname); tmpSession.setQualifiedData(variableStore.APPLICATION_SCOPE, appData); } // Create the CFC we want to call try { final ObjectCFC cfc = PluginManager.getPlugInManager().createCFC(tmpSession, callback); cfc.addArgument("file", (String) jobFile.get("localpath")); cfc.addArgument("success", success); cfc.addArgument("callbackdata", callbackdata); cfc.addArgument("error", errMessage == null ? "" : errMessage); new Thread() { public void run() { setName("AmazonS3Write.BackgroundUploader.onAmazonS3Write()"); try { cfc.runMethod(tmpSession, "onAmazonS3Write"); } catch (cfmRunTimeException rte) { rte.handleException(tmpSession); } catch (Exception e) { cfEngine.log("AmazonS3Write.BackgroundUploader.onAmazonS3Write.thread():" + e.getMessage()); } finally { tmpSession.pageEnd(); tmpSession.close(); } } }.start(); } catch (Exception e) { cfEngine.log("AmazonS3Write.BackgroundUploader.onAmazonS3Write():" + e.getMessage()); } } private void log(Map<String, Object> jobFile, String action) { StringBuilder sb = new StringBuilder(64); File localFile = new File((String) jobFile.get("localpath")); sb.append("AmazonS3Write.BackgroundUploader: ").append("bucket=").append((String) jobFile.get("bucket")) .append("; ").append("key=").append((String) jobFile.get("key")).append("; ").append("localfile=") .append(localFile.getName()).append("; ").append("size=").append(localFile.length()).append("; ") .append(action); cfEngine.log(sb.toString()); } private void removeJobFile(Map<String, Object> jobFile) { new File(workingDirectory, jobFile.get("id") + ".job").delete(); } private void saveFile(Map<String, Object> jobFile) { FileUtil.saveClass(new File(workingDirectory, jobFile.get("id") + ".job"), jobFile); } private Map<String, Object> takeNextJob() { synchronized (fileList) { Iterator<Map<String, Object>> it = fileList.iterator(); while (it.hasNext()) { Map<String, Object> jobFile = it.next(); if ((Long) jobFile.get("attemptdate") <= System.currentTimeMillis()) { it.remove(); return jobFile; } } } return null; } }