Java tutorial
/** * This file is part of PackRat, an Android app for managing media collections. * Copyright (C) 2009-2012 Jens Finkhaeuser <jens@finkhaeuser.de> * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. */ package de.unwesen.web; import android.net.Uri; import android.content.Context; import java.io.File; import java.security.MessageDigest; import java.math.BigInteger; import android.os.Handler; import android.os.Message; import android.os.Environment; import java.util.concurrent.ConcurrentLinkedQueue; import org.apache.http.impl.client.DefaultHttpClient; import org.apache.http.client.methods.HttpGet; import org.apache.http.HttpResponse; import java.io.InputStream; import java.io.FileOutputStream; import java.util.List; import java.util.LinkedList; import java.util.Map; import java.util.TreeMap; import java.lang.System; import java.lang.ref.WeakReference; import android.util.Log; /** * DownloadCache * - Downloads files from the internet for you, notifying you when the download * is finished. * - Downloads files sequentially. You cannot stop jobs. XXX Can add stopping * jobs that are not yet in progress. * - Caches those files locally. * - You can use more than one cache in parallel - cache directories are * prefixed with a custom string you supply. **/ public class DownloadCache { /*************************************************************************** * Public constants **/ // SUCCESS means the message's obj is a CacheItem with mFilename pointing to // the cache entry. public static final int CACHE_SUCCESS = 100; // FAILURE means the message's obj is a CacheItem possibly with mFilename == null public static final int CACHE_FAILURE = 101; // LINK_FAILURE means the item could not be downloaded. public static final int CACHE_LINK_FAILURE = 102; // TEMP_LINK_FAILURE means the item could not be downloaded, but that // might change with the next attempt. public static final int CACHE_TEMP_LINK_FAILURE = 103; // CANCELLED means the message's obj is a CacheItem with mFilename == null public static final int CACHE_CANCELLED = 104; /*************************************************************************** * Private constants **/ // Log ID private static final String LTAG = "DownloadCache"; private static final String HASH_ALGO = "MD5"; private static final int HASH_ALGO_SIZE = 128; private static final int SEGMENT_SIZE = 1; // Fetcher thread wakes up at this interval (msec). private static final int FETCHER_SLEEP_TIME = 60 * 1000; // Delay before Fetcher thread tries to grab anything from the queue (msec). private static final int FETCHER_DELAY = 200; // Chunk size to read when downloading. private static final int READ_CHUNK_SIZE = 8 * 1024; // Ignore files that are less than this amount of milliseconds old. private static final long MIN_DELETE_AGE = 60 * 1000; /*************************************************************************** * Tunables class; you can set tunables from the outside to configure * the cache. **/ public static interface Tunables { /** * Return the maximum cache size. **/ public long getMaxCacheSize(); /** * Return wether or not LRU cache cleaning should be performed. **/ public boolean flushLRUEntries(); /** * Return whether the cache should be internal or external; if there is * no external directory, we'll default to internal anyway. **/ public boolean useExternalStorage(); } /*************************************************************************** * CacheItem class; information required by the Fetcher below. **/ protected static class CacheItem { public Uri mDownloadUri; public String mFilename; public Handler mHandler; public String toString() { return String.format("<%s:%s>", mDownloadUri.toString(), mFilename); } public void sendMessage(int id) { mHandler.obtainMessage(id, this).sendToTarget(); } } /*************************************************************************** * LRU clearing thread. **/ private class LRUThread extends Thread { public boolean keepRunning = true; @Override public void run() { while (keepRunning) { // Sleep long if the queue is empty. try { sleep(FETCHER_SLEEP_TIME); clearLRU(); } catch (java.lang.InterruptedException ex) { // pass } } } } /*************************************************************************** * Fetcher class, for downloading. **/ private class Fetcher extends Thread { public boolean keepRunning = true; @Override public void run() { while (keepRunning) { if (mFetcherQueue.isEmpty()) { // Sleep long if the queue is empty. try { sleep(FETCHER_SLEEP_TIME); } catch (java.lang.InterruptedException ex) { // pass } } else { // Othewise, sleep a short time before processing what's in the queue try { sleep(FETCHER_DELAY); processQueue(this); } catch (java.lang.InterruptedException ex) { // pass } } } } } /*************************************************************************** * Data **/ // Execution context private WeakReference<Context> mContext; // Fetcher Thread private Fetcher mFetcher; // Thread for LRU clearing private LRUThread mLRUThread; // Queue of items the Fetcher works on. ConcurrentLinkedQueue<CacheItem> mFetcherQueue = new ConcurrentLinkedQueue<CacheItem>(); // Cache prefix private String mCachePrefix; // Cache tunables private Tunables mTunables; /*************************************************************************** * Implementation **/ // The cachePrevix parameter specifies a directory name under which cache // content is stored. public DownloadCache(Context context, String cachePrefix, Tunables tunables) { if (null == context || null == tunables) { Log.e(LTAG, "DownloadCache instanciated without context or tunables."); throw new IllegalArgumentException("DownloadCache instanciated without " + "context or tunables."); } mContext = new WeakReference<Context>(context); mTunables = tunables; mCachePrefix = cachePrefix; if (null == mCachePrefix) { // Simple default for the cache prefix. mCachePrefix = "cache"; } mFetcher = new Fetcher(); mFetcher.start(); mLRUThread = new LRUThread(); mLRUThread.start(); } public String get(Uri uri) { String filename = getCacheFileName(uri); File cachefile = new File(filename); if (cachefile.exists()) { return filename; } return null; } public void cancelFetching() { cancelFetching(false); } public void cancelFetching(boolean interruptFetcher) { // Grab cache items. We need this to send cancelled events. Object[] items = mFetcherQueue.toArray(); // Clear the cache queue. We don't (always) interrupt - which means that // the thread may process any request currently underway. mFetcherQueue.clear(); if (interruptFetcher) { mFetcher.interrupt(); } // Now send cancel events for each item in the array. if (null != items) { for (Object obj : items) { CacheItem item = (CacheItem) obj; item.sendMessage(CACHE_CANCELLED); } } } public void fetch(CacheItem item, Handler handler) { if (null == item || null == handler) { return; } List<CacheItem> items = new LinkedList<CacheItem>(); items.add(item); fetch(items, handler); } public void fetch(List<? extends CacheItem> items, Handler handler) { if (null == items || 0 == items.size() || null == handler) { return; } int added = 0; for (CacheItem item : items) { // Check whether file already exists. String filename = get(item.mDownloadUri); if (null != filename) { item.mFilename = filename; handler.obtainMessage(CACHE_SUCCESS, item).sendToTarget(); continue; } // Otherwise add the item to the fetcher queue. item.mHandler = handler; mFetcherQueue.add(item); ++added; } if (0 != added) { mFetcher.interrupt(); } } private String getCacheHash(Uri uri) { if (null == uri) { return null; } // Create hash of URL MessageDigest m = null; try { m = MessageDigest.getInstance(HASH_ALGO); } catch (java.security.NoSuchAlgorithmException ex) { Log.e(LTAG, "No " + HASH_ALGO + " supported, caching is disabled!"); return null; } m.update(uri.toString().getBytes()); String digest = new BigInteger(1, m.digest()).toString(16); while (digest.length() < ((HASH_ALGO_SIZE / 8) * 2)) { digest = "0" + digest; } return digest; } private String getCacheFileName(String digest) { if (null == digest) { return null; } // Split hash every N bytes, i.e. N * 2 characters String basename = ""; while (digest.length() > 0) { String seg = digest.substring(0, SEGMENT_SIZE * 2); digest = digest.substring(SEGMENT_SIZE * 2); basename += File.separator + seg; } return getCacheDir() + basename; } private String getCacheDir() { Context ctx = mContext.get(); if (null == ctx) { return null; } File cache_dir = ctx.getCacheDir(); String root = cache_dir.getPath(); if (mTunables.useExternalStorage() && Environment.MEDIA_MOUNTED.equals(Environment.getExternalStorageState())) { cache_dir = Environment.getExternalStorageDirectory(); root = cache_dir.getPath() + File.separator + "data" + File.separator + ctx.getPackageName(); } // Log.d(LTAG, "Cache dir: " + root + File.separator + mCachePrefix + File.separator); return root + File.separator + mCachePrefix + File.separator; } public String getCacheFileName(Uri uri) { return getCacheFileName(getCacheHash(uri)); } private void clearLRU() { // Don't flush the cache if that's not desired. if (!mTunables.flushLRUEntries()) { return; } List<File> files = recursivelyFindCacheFiles(new File(getCacheDir())); if (null == files) { // Empty cache, there's nothing to do! return; } // Sort files by age, and also determine the cache size. Map<Long, List<File>> sorted = new TreeMap<Long, List<File>>(); long cachesize = 0; for (File f : files) { cachesize += f.length(); List<File> list = sorted.get(f.lastModified()); if (null == list) { list = new LinkedList<File>(); } list.add(f); sorted.put(f.lastModified(), list); } // While the cache size is larger than desired, delete files with the oldest // age. We go by chunks here; if there are more files at the oldest timestamp // than we'd need to bring the cache size under the limit, then we'll delete // them anyway. long min_age = System.currentTimeMillis() - MIN_DELETE_AGE; for (Map.Entry<Long, List<File>> entry : sorted.entrySet()) { if (cachesize < mTunables.getMaxCacheSize()) { // All done! break; } // We want to ignore files that are too young. if (entry.getKey() > min_age) { // Ignore these continue; } // Right, we'll clear this chunk of files. for (File f : entry.getValue()) { cachesize -= f.length(); f.delete(); } } } private List<File> recursivelyFindCacheFiles(File parent) { String[] children = parent.list(); if (null == children || 0 == children.length) { return null; } List<File> retval = new LinkedList<File>(); for (String childname : children) { File child = new File(parent.getPath() + File.separator + childname); if (child.isDirectory()) { List<File> grandchildren = recursivelyFindCacheFiles(child); if (null != grandchildren) { retval.addAll(grandchildren); } } else { retval.add(child); } } return retval; } protected boolean copyFile(String target, InputStream istream) { try { FileOutputStream ostream = new FileOutputStream(target); try { // Read bytes from the input stream in chunks and write // them into the output stream byte[] bytes = new byte[READ_CHUNK_SIZE]; int bytes_read = 0; while (-1 != (bytes_read = istream.read(bytes))) { ostream.write(bytes, 0, bytes_read); } ostream.close(); return true; } catch (java.io.IOException ex) { Log.e(LTAG, "Failed copying file: " + ex.getMessage()); return false; } } catch (java.io.FileNotFoundException ex) { Log.e(LTAG, "Failed to open output file: " + ex.getMessage()); return false; } } private void processQueue(Thread thread) { while (true) { // Grab and process next item on the queue CacheItem info = mFetcherQueue.poll(); if (null == info) { break; } // Check whether file exists. String filename = getCacheFileName(info.mDownloadUri); File cachefile = new File(filename); if (cachefile.exists()) { // Log.d(LTAG, "Url is already downloaded and cached."); info.mFilename = filename; info.sendMessage(CACHE_SUCCESS); continue; } // Download file. DefaultHttpClient httpClient = new DefaultHttpClient(); HttpGet request = new HttpGet(info.mDownloadUri.toString()); try { HttpResponse response = httpClient.execute(request); // Log non-200 response codes, return a link failure if (200 != response.getStatusLine().getStatusCode()) { Log.w(LTAG, "Could not retrieve URI: " + info.mDownloadUri); info.sendMessage(CACHE_LINK_FAILURE); continue; } InputStream istream = response.getEntity().getContent(); // Create path for cache file. String pathname = cachefile.getParentFile().getPath(); File path = new File(pathname); if (!path.exists() && !path.mkdirs()) { Log.e(LTAG, "Could not create cache directory '" + pathname + "'!"); info.sendMessage(CACHE_FAILURE); continue; } // Write to temporary file. XXX temp_filename assumes the same download // isn't started more than once in the same millisecond - fairly safe, // but not foolproof. It's safe insofar as the DownloadCache class // itself does not download in parallel. String temp_filename = filename + "." + System.currentTimeMillis(); File tempfile = new File(temp_filename); // Log.d(LTAG, "Downloading to temporary location '" + tempfile.getAbsolutePath() + "'..."); if (!copyFile(temp_filename, istream)) { info.sendMessage(CACHE_FAILURE); continue; } // Now move the temp file over to filename. if (!tempfile.renameTo(cachefile)) { Log.e(LTAG, "Could not finalize download!"); info.sendMessage(CACHE_FAILURE); continue; } // Final check. if (!cachefile.exists()) { Log.e(LTAG, "Unknown error when retrieving file from cache."); info.sendMessage(CACHE_FAILURE); continue; } // Success! info.mFilename = cachefile.getPath(); info.sendMessage(CACHE_SUCCESS); } catch (java.net.UnknownHostException ex) { Log.w(LTAG, "Unknown host: " + ex.getMessage()); info.sendMessage(CACHE_TEMP_LINK_FAILURE); } catch (Exception ex) { Log.e(LTAG, "Could not download '" + info.mDownloadUri + "': " + ex.getMessage()); info.sendMessage(CACHE_LINK_FAILURE); } catch (OutOfMemoryError ex) { Log.e(LTAG, "Out of memory: " + ex.getMessage()); info.sendMessage(CACHE_LINK_FAILURE); } } // Finally, clear the least recently used items. // mLRUThread.interrupt(); //clearLRU(); } protected Context getContext() { return mContext.get(); } }