Java tutorial
/* * Copyright 2012 Brendan McCarthy (brendan@oddsoftware.net) * * This file is part of Feedscribe. * * Feedscribe is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License version 3 * as published by the Free Software Foundation. * * Feedscribe is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with Feedscribe. If not, see <http://www.gnu.org/licenses/>. */ package net.oddsoftware.android.html; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.net.ProxySelector; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; import java.util.Locale; import java.util.TimeZone; import java.util.UUID; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; import java.util.zip.InflaterInputStream; import net.oddsoftware.android.feedscribe.Globals; import net.oddsoftware.android.feedscribe.data.FeedManager; import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.HttpHost; import org.apache.http.HttpResponse; import org.apache.http.StatusLine; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.impl.client.DefaultHttpClient; import org.apache.http.impl.conn.ProxySelectorRoutePlanner; import org.apache.http.protocol.BasicHttpContext; import org.apache.http.protocol.ExecutionContext; import org.apache.http.protocol.HttpContext; import android.content.ContentResolver; import android.content.Context; import android.util.Log; public class HttpCache { private File mCacheDirectory; private ContentResolver mContentResolver; public static final int MAX_ETAG_LENGTH = 200; // default to 6 hour cache time protected static final int DEFAULT_EXPIRES = 6 * 60 * 60 * 1000; // if it hasn't been accessed for 24 hours protected static final int MAX_CACHE_IDLE_TIME = 24 * 60 * 60 * 1000; // don't cache anything bigger than this (?) protected static final int MAX_CONTENT_LENGTH = 300 * 1024; protected static final boolean ALWAYS_PREFER_CACHE = true; public HttpCache(Context context) { mCacheDirectory = context.getCacheDir(); mContentResolver = context.getContentResolver(); } public InputStream getResource(String url, boolean forceDownload) throws IOException { CacheItem cacheItem = CacheItem.getByURL(mContentResolver, url); if (cacheItem == null) { cacheItem = new CacheItem(url); download(cacheItem); } else { cacheItem.mHitTime = new Date().getTime(); cacheItem.update(mContentResolver); } /*// for now we will always use the cache if it's there if( (! ALWAYS_PREFER_CACHE) && cacheItem.mExpiresAt < new Date().getTime() ) { download(cacheItem); } */ File cacheFile = new File(cacheItem.mFilename); if (!forceDownload) { if (cacheFile.exists()) { return new GZIPInputStream(new FileInputStream(cacheFile)); } } download(cacheItem); if (cacheFile.exists()) { return new GZIPInputStream(new FileInputStream(cacheFile)); } return null; } public String getLastUrl(String url) { CacheItem cacheItem = CacheItem.getByURL(mContentResolver, url); if (cacheItem == null) { return null; } else { return cacheItem.mLastUrl; } } private void download(CacheItem cacheItem) { try { // check to see if file exist, if so check etag and last-modified if (cacheItem.mFilename.length() > 0) { File f = new File(cacheItem.mFilename); try { InputStream is = new FileInputStream(f); is.close(); } catch (IOException exc) { // no file, nuke the cache stats cacheItem.mETag = ""; cacheItem.mLastModified = 0; } } else { cacheItem.mFilename = mCacheDirectory + File.separator + UUID.randomUUID().toString() + ".html.gz"; } HttpContext httpContext = new BasicHttpContext(); HttpClient client = createHttpClient(); HttpUriRequest request = createHttpRequest(cacheItem.mUrl, cacheItem.mETag, cacheItem.mLastModified); if (request == null || request.getURI() == null || request.getURI().getHost() == null || request.getURI().getHost().length() == 0) { if (Globals.LOGGING) Log.e(Globals.LOG_TAG, "unable to create http request for url " + cacheItem.mUrl); return; // sadness } HttpResponse response = client.execute(request, httpContext); StatusLine status = response.getStatusLine(); HttpEntity entity = response.getEntity(); if (status.getStatusCode() == 304) { if (Globals.LOGGING) Log.d(Globals.LOG_TAG, "received 304 not modified"); cacheItem.mHitTime = new Date().getTime(); cacheItem.update(mContentResolver); return; } if (status.getStatusCode() == 200) { InputStream inputStream = null; if (entity != null) { inputStream = entity.getContent(); } else { return; } long contentLength = entity.getContentLength(); if (contentLength > MAX_CONTENT_LENGTH) { if (Globals.LOGGING) Log.w(Globals.LOG_TAG, "HttpCache.download item " + cacheItem.mUrl + " content length is too big " + contentLength); return; } Header encodingHeader = entity.getContentEncoding(); boolean encoded = false; if (encodingHeader != null) { if (encodingHeader.getValue().equalsIgnoreCase("gzip")) { inputStream = new GZIPInputStream(inputStream); encoded = true; } else if (encodingHeader.getValue().equalsIgnoreCase("deflate")) { inputStream = new InflaterInputStream(inputStream); encoded = true; } } File tmpFile = File.createTempFile("httpcache", ".html.gz.tmp", mCacheDirectory); OutputStream os = new GZIPOutputStream(new FileOutputStream(tmpFile)); byte[] buffer = new byte[4096]; int count = 0; long fileSize = 0; while ((count = inputStream.read(buffer)) != -1) { os.write(buffer, 0, count); fileSize += count; } inputStream.close(); os.close(); if (!encoded && contentLength > 0 && fileSize != contentLength) { Log.e(Globals.LOG_TAG, "HttpCache.download: content-length: " + contentLength + " but file size: " + fileSize + " aborting"); tmpFile.delete(); return; } tmpFile.renameTo(new File(cacheItem.mFilename)); // if the parse was ok, update these attributes // ETag: "6050003-78e5-4981d775e87c0" Header etagHeader = response.getFirstHeader("ETag"); if (etagHeader != null) { if (etagHeader.getValue().length() < MAX_ETAG_LENGTH) { cacheItem.mETag = etagHeader.getValue(); } else { if (Globals.LOGGING) Log.e(Globals.LOG_TAG, "etag length was too big: " + etagHeader.getValue().length()); } } // Last-Modified: Fri, 24 Dec 2010 00:57:11 GMT Header lastModifiedHeader = response.getFirstHeader("Last-Modified"); if (lastModifiedHeader != null) { try { cacheItem.mLastModified = FeedManager.parseRFC822Date(lastModifiedHeader.getValue()) .getTime(); } catch (ParseException exc) { if (Globals.LOGGING) Log.e(Globals.LOG_TAG, "unable to parse date", exc); } } // Expires: Thu, 01 Dec 1994 16:00:00 GMT Header expiresHeader = response.getFirstHeader("Expires"); if (expiresHeader != null) { try { cacheItem.mExpiresAt = FeedManager.parseRFC822Date(expiresHeader.getValue()).getTime(); } catch (ParseException exc) { if (Globals.LOGGING) Log.e(Globals.LOG_TAG, "unable to parse expires", exc); } } long now = new Date().getTime() + DEFAULT_EXPIRES; if (cacheItem.mExpiresAt < now) { cacheItem.mExpiresAt = now; } HttpUriRequest currentReq = (HttpUriRequest) httpContext .getAttribute(ExecutionContext.HTTP_REQUEST); HttpHost currentHost = (HttpHost) httpContext.getAttribute(ExecutionContext.HTTP_TARGET_HOST); String currentUrl = currentHost.toURI() + currentReq.getURI(); if (Globals.LOGGING) Log.w(Globals.LOG_TAG, "loaded redirect from " + request.getURI().toString() + " to " + currentUrl); cacheItem.mLastUrl = currentUrl; cacheItem.mHitTime = new Date().getTime(); cacheItem.update(mContentResolver); } } catch (IOException exc) { if (Globals.LOGGING) { Log.e(Globals.LOG_TAG, "error downloading file to cache", exc); } } } private HttpClient createHttpClient() { // use apache http client lib to set parameters from feedStatus DefaultHttpClient client = new DefaultHttpClient(); // set up proxy handler ProxySelectorRoutePlanner routePlanner = new ProxySelectorRoutePlanner( client.getConnectionManager().getSchemeRegistry(), ProxySelector.getDefault()); client.setRoutePlanner(routePlanner); return client; } private HttpUriRequest createHttpRequest(String url, String eTag, long lastModified) { HttpGet request = null; try { request = new HttpGet(url); } catch (IllegalArgumentException exc) { if (Globals.LOGGING) Log.e(Globals.LOG_TAG, "createHttpRequest: error creating http get", exc); return null; } request.setHeader("User-Agent", FeedManager.USER_AGENT); request.setHeader("Accept-Encoding", "gzip,deflate"); // send etag if we have it if (eTag != null && eTag.length() > 0) { request.setHeader("If-None-Match", eTag); } // send If-Modified-Since if we have it if (lastModified > 0) { SimpleDateFormat dateFormat = new SimpleDateFormat("EEE', 'dd' 'MMM' 'yyyy' 'HH:mm:ss' GMT'", Locale.US); dateFormat.setTimeZone(TimeZone.getTimeZone("GMT")); String formattedTime = dateFormat.format(lastModified); // If-Modified-Since: Sat, 29 Oct 1994 19:43:31 GMT request.setHeader("If-Modified-Since", formattedTime); } return request; } @SuppressWarnings("unused") private HttpUriRequest createHttpRequest(String url, long resumeFrom) { HttpGet request = new HttpGet(url); request.setHeader("User-Agent", FeedManager.USER_AGENT); request.setHeader("Accept-Encoding", "gzip,deflate"); if (resumeFrom > 0) { request.setHeader("Range", "bytes=" + resumeFrom + "-"); } return request; } public void seed(String url) { CacheItem cacheItem = CacheItem.getByURL(mContentResolver, url); if (cacheItem == null) { if (Globals.LOGGING) Log.d(Globals.LOG_TAG, "HttpCache.seed: seeding" + url); cacheItem = new CacheItem(url); download(cacheItem); } else { cacheItem.mHitTime = new Date().getTime(); cacheItem.update(mContentResolver); } } public void maintainCache() { ArrayList<CacheItem> allItems = CacheItem.getAllItems(mContentResolver); long now = new Date().getTime(); @SuppressWarnings("unused") long cacheSize = 0; for (CacheItem item : allItems) { if (item.mHitTime + MAX_CACHE_IDLE_TIME < now) { if (Globals.LOGGING) { Log.e(Globals.LOG_TAG, "HttpCache: maintainCache - removing item " + item.mUrl + " that hasn't been accessed for " + ((now - item.mHitTime) / 1000) + " seconds "); } try { File f = new File(item.mFilename); f.delete(); item.delete(mContentResolver); } catch (SecurityException exc) { if (Globals.LOGGING) Log.e(Globals.LOG_TAG, "HttpCache: maintainCache", exc); } } try { File f = new File(item.mFilename); cacheSize += f.length(); } catch (SecurityException exc) { if (Globals.LOGGING) Log.e(Globals.LOG_TAG, "HttpCache: maintainCache", exc); } } if (Globals.LOGGING) Log.d(Globals.LOG_TAG, "HttpCache.maintainCache - cache size is " + (cacheSize / 1024) + " kilobytes"); } }