Java tutorial
/* * Copyright 2007 T-Rank AS * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package no.trank.openpipe.wikipedia.download; import java.io.BufferedInputStream; import java.io.BufferedReader; import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.FileReader; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.net.URL; import java.security.DigestOutputStream; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.Date; import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.URI; import org.apache.commons.httpclient.URIException; import org.apache.commons.httpclient.methods.GetMethod; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import no.trank.openpipe.util.HexUtil; import no.trank.openpipe.wikipedia.producer.meta.RssMetaParser; /** * @version $Revision$ */ public class HttpDownloader { private static final Logger log = LoggerFactory.getLogger(HttpDownloader.class); private DownloadProgressListener progressListener = new NullProgressListener(); private File targetFile; private String rssUrl; private long nextProgressTS; private int progressInterval = 1000; private URL sourceUrl; private HttpClient httpClient; public File getTargetFile() { return targetFile; } public void setTargetFile(File targetFile) { this.targetFile = targetFile; } public String getRssUrl() { return rssUrl; } public void setRssUrl(String rssUrl) { this.rssUrl = rssUrl; } public int getProgressInterval() { return progressInterval; } public void setProgressInterval(int progressInterval) { this.progressInterval = progressInterval; } public URL getSourceUrl() { return sourceUrl; } public void setHttpClient(HttpClient httpClient) { this.httpClient = httpClient; } public void setProgressListener(DownloadProgressListener progressListener) { this.progressListener = progressListener == null ? new NullProgressListener() : progressListener; } public void init() { if (targetFile == null || !targetFile.getParentFile().isDirectory()) { throw new IllegalArgumentException("Invalid targetFile '" + targetFile + '\''); } if (rssUrl == null) { throw new NullPointerException("sourceUrl cannot be null"); } try { new URI(rssUrl, true); } catch (URIException e) { throw new IllegalArgumentException("rssUrl '" + rssUrl + "' must be a valid URL: " + e.getMessage()); } if (httpClient == null) { httpClient = new HttpClient(); } } public boolean isLastVersion() throws IOException { final RssMetaParser parser = new RssMetaParser(getContentAsString(rssUrl)); sourceUrl = parser.getFileUrl(); return !isModified(parser.getModifiedDate(), targetFile) || hasCorrectMd5(parser); } private boolean hasCorrectMd5(RssMetaParser parser) throws IOException { final String md5Sums = getContentAsString(parser.getMd5Url().toExternalForm()); return isSameMd5(targetFile, parser.findMd5(md5Sums)); } private String getContentAsString(String url) throws IOException { final GetMethod get = new GetMethod(url); try { final int status = httpClient.executeMethod(get); if (status >= 200 && status < 300) { return get.getResponseBodyAsString(); } log.warn("Could not get content from url: '{}' status: {}", url, status); return null; } finally { get.releaseConnection(); } } private static boolean isModified(Date modifiedDate, File file) { return modifiedDate.after(new Date(file.lastModified())) || modifiedDate.after(new Date(getMd5File(file).lastModified())); } private static boolean isSameMd5(File file, String md5Sum) { final File md5File = getMd5File(file); try { if (md5File.exists() && md5File.length() > 0) { BufferedReader reader = new BufferedReader(new FileReader(md5File)); try { return md5Sum.equalsIgnoreCase(reader.readLine().trim()); } finally { reader.close(); } } } catch (FileNotFoundException e) { // Do nothing } catch (IOException e) { log.error("Could not read md5 from file: " + md5File.getAbsolutePath()); } return false; } private static File getMd5File(File file) { return new File(file.getAbsolutePath() + ".md5"); } public int downloadFile() throws IOException { final GetMethod get = new GetMethod(sourceUrl.toExternalForm()); try { final int status = httpClient.executeMethod(get); if (status >= 200 && status < 300) { long size = get.getResponseContentLength(); log.debug("Downloading: " + size + " bytes"); final BufferedInputStream in = new BufferedInputStream(get.getResponseBodyAsStream()); try { writeFile(in, size); } finally { in.close(); } } return status; } finally { get.releaseConnection(); } } protected void writeFile(InputStream in, long size) throws IOException { OutputStream out; try { out = new DigestOutputStream(new FileOutputStream(targetFile), MessageDigest.getInstance("MD5")); } catch (NoSuchAlgorithmException e) { log.error("Could not make md5. MD5 not supported", e); out = new FileOutputStream(targetFile); } reportSize(size); try { long totalReadBytes = 0; final byte[] buf = new byte[1024 * 32]; int numBytes = in.read(buf); while (numBytes >= 0) { totalReadBytes += numBytes; reportProgress(totalReadBytes); out.write(buf, 0, numBytes); numBytes = in.read(buf); } } finally { try { out.close(); } catch (Exception e) { // Do nothing } reportDone(); if (out instanceof DigestOutputStream) { writeMd5File(((DigestOutputStream) out).getMessageDigest()); } } } private void writeMd5File(MessageDigest messageDigest) throws IOException { FileOutputStream fout = new FileOutputStream(getMd5File(targetFile)); try { fout.write(HexUtil.toHexString(messageDigest.digest()).getBytes("UTF-8")); } finally { try { fout.close(); } catch (Exception e) { // Do nothing } } } private void reportSize(final long size) { progressListener.totalSize(size); nextProgressTS = System.currentTimeMillis() + progressInterval; } private void reportProgress(final long totalReadBytes) { final long now = System.currentTimeMillis(); if (nextProgressTS <= now) { progressListener.progress(totalReadBytes); nextProgressTS = now + progressInterval; } } private void reportDone() { progressListener.done(); } }