Java tutorial
// Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information // regarding copyright ownership. The ASF licenses this file // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, // software distributed under the License is distributed on an // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. package com.cloud.storage.template; import java.io.BufferedInputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.RandomAccessFile; import java.net.Inet6Address; import java.net.InetAddress; import java.net.URI; import java.net.URISyntaxException; import java.net.UnknownHostException; import java.util.Date; import org.apache.commons.httpclient.ChunkedInputStream; import org.apache.commons.httpclient.Credentials; import org.apache.commons.httpclient.Header; import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.HttpException; import org.apache.commons.httpclient.HttpMethod; import org.apache.commons.httpclient.HttpMethodRetryHandler; import org.apache.commons.httpclient.HttpStatus; import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager; import org.apache.commons.httpclient.NoHttpResponseException; import org.apache.commons.httpclient.UsernamePasswordCredentials; import org.apache.commons.httpclient.auth.AuthScope; import org.apache.commons.httpclient.methods.GetMethod; import org.apache.commons.httpclient.params.HttpMethodParams; import org.apache.log4j.Logger; import com.cloud.agent.api.storage.DownloadCommand.Proxy; import com.cloud.agent.api.storage.DownloadCommand.ResourceType; import com.cloud.storage.StorageLayer; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.Pair; /** * Download a template file using HTTP * */ public class HttpTemplateDownloader implements TemplateDownloader { public static final Logger s_logger = Logger.getLogger(HttpTemplateDownloader.class.getName()); private static final MultiThreadedHttpConnectionManager s_httpClientManager = new MultiThreadedHttpConnectionManager(); private static final int CHUNK_SIZE = 1024 * 1024; //1M private String downloadUrl; private String toFile; public TemplateDownloader.Status status = TemplateDownloader.Status.NOT_STARTED; public String errorString = " "; private long remoteSize = 0; public long downloadTime = 0; public long totalBytes; private final HttpClient client; private GetMethod request; private boolean resume = false; private DownloadCompleteCallback completionCallback; StorageLayer _storage; boolean inited = true; private String toDir; private long MAX_TEMPLATE_SIZE_IN_BYTES; private ResourceType resourceType = ResourceType.TEMPLATE; private final HttpMethodRetryHandler myretryhandler; public HttpTemplateDownloader(StorageLayer storageLayer, String downloadUrl, String toDir, DownloadCompleteCallback callback, long maxTemplateSizeInBytes, String user, String password, Proxy proxy, ResourceType resourceType) { this._storage = storageLayer; this.downloadUrl = downloadUrl; this.setToDir(toDir); this.status = TemplateDownloader.Status.NOT_STARTED; this.resourceType = resourceType; this.MAX_TEMPLATE_SIZE_IN_BYTES = maxTemplateSizeInBytes; this.totalBytes = 0; this.client = new HttpClient(s_httpClientManager); myretryhandler = new HttpMethodRetryHandler() { public boolean retryMethod(final HttpMethod method, final IOException exception, int executionCount) { if (executionCount >= 2) { // Do not retry if over max retry count return false; } if (exception instanceof NoHttpResponseException) { // Retry if the server dropped connection on us return true; } if (!method.isRequestSent()) { // Retry if the request has not been sent fully or // if it's OK to retry methods that have been sent return true; } // otherwise do not retry return false; } }; try { this.request = new GetMethod(downloadUrl); this.request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler); this.completionCallback = callback; //this.request.setFollowRedirects(false); File f = File.createTempFile("dnld", "tmp_", new File(toDir)); if (_storage != null) { _storage.setWorldReadableAndWriteable(f); } toFile = f.getAbsolutePath(); Pair<String, Integer> hostAndPort = validateUrl(downloadUrl); if (proxy != null) { client.getHostConfiguration().setProxy(proxy.getHost(), proxy.getPort()); if (proxy.getUserName() != null) { Credentials proxyCreds = new UsernamePasswordCredentials(proxy.getUserName(), proxy.getPassword()); client.getState().setProxyCredentials(AuthScope.ANY, proxyCreds); } } if ((user != null) && (password != null)) { client.getParams().setAuthenticationPreemptive(true); Credentials defaultcreds = new UsernamePasswordCredentials(user, password); client.getState().setCredentials( new AuthScope(hostAndPort.first(), hostAndPort.second(), AuthScope.ANY_REALM), defaultcreds); s_logger.info("Added username=" + user + ", password=" + password + "for host " + hostAndPort.first() + ":" + hostAndPort.second()); } else { s_logger.info( "No credentials configured for host=" + hostAndPort.first() + ":" + hostAndPort.second()); } } catch (IllegalArgumentException iae) { errorString = iae.getMessage(); status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; inited = false; } catch (Exception ex) { errorString = "Unable to start download -- check url? "; status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; s_logger.warn("Exception in constructor -- " + ex.toString()); } catch (Throwable th) { s_logger.warn("throwable caught ", th); } } private Pair<String, Integer> validateUrl(String url) throws IllegalArgumentException { try { URI uri = new URI(url); if (!uri.getScheme().equalsIgnoreCase("http") && !uri.getScheme().equalsIgnoreCase("https")) { throw new IllegalArgumentException("Unsupported scheme for url"); } int port = uri.getPort(); if (!(port == 80 || port == 443 || port == -1)) { throw new IllegalArgumentException("Only ports 80 and 443 are allowed"); } if (port == -1 && uri.getScheme().equalsIgnoreCase("https")) { port = 443; } else if (port == -1 && uri.getScheme().equalsIgnoreCase("http")) { port = 80; } String host = uri.getHost(); try { InetAddress hostAddr = InetAddress.getByName(host); if (hostAddr.isAnyLocalAddress() || hostAddr.isLinkLocalAddress() || hostAddr.isLoopbackAddress() || hostAddr.isMulticastAddress()) { throw new IllegalArgumentException("Illegal host specified in url"); } if (hostAddr instanceof Inet6Address) { throw new IllegalArgumentException( "IPV6 addresses not supported (" + hostAddr.getHostAddress() + ")"); } return new Pair<String, Integer>(host, port); } catch (UnknownHostException uhe) { throw new IllegalArgumentException("Unable to resolve " + host); } } catch (IllegalArgumentException iae) { s_logger.warn("Failed uri validation check: " + iae.getMessage()); throw iae; } catch (URISyntaxException use) { s_logger.warn("Failed uri syntax check: " + use.getMessage()); throw new IllegalArgumentException(use.getMessage()); } } @Override public long download(boolean resume, DownloadCompleteCallback callback) { switch (status) { case ABORTED: case UNRECOVERABLE_ERROR: case DOWNLOAD_FINISHED: return 0; default: } int bytes = 0; File file = new File(toFile); try { long localFileSize = 0; if (file.exists() && resume) { localFileSize = file.length(); s_logger.info("Resuming download to file (current size)=" + localFileSize); } Date start = new Date(); int responseCode = 0; if (localFileSize > 0) { // require partial content support for resume request.addRequestHeader("Range", "bytes=" + localFileSize + "-"); if (client.executeMethod(request) != HttpStatus.SC_PARTIAL_CONTENT) { errorString = "HTTP Server does not support partial get"; status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; return 0; } } else if ((responseCode = client.executeMethod(request)) != HttpStatus.SC_OK) { status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; errorString = " HTTP Server returned " + responseCode + " (expected 200 OK) "; return 0; //FIXME: retry? } Header contentLengthHeader = request.getResponseHeader("Content-Length"); boolean chunked = false; long remoteSize2 = 0; if (contentLengthHeader == null) { Header chunkedHeader = request.getResponseHeader("Transfer-Encoding"); if (chunkedHeader == null || !"chunked".equalsIgnoreCase(chunkedHeader.getValue())) { status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; errorString = " Failed to receive length of download "; return 0; //FIXME: what status do we put here? Do we retry? } else if ("chunked".equalsIgnoreCase(chunkedHeader.getValue())) { chunked = true; } } else { remoteSize2 = Long.parseLong(contentLengthHeader.getValue()); } if (remoteSize == 0) { remoteSize = remoteSize2; } if (remoteSize > MAX_TEMPLATE_SIZE_IN_BYTES) { s_logger.info("Remote size is too large: " + remoteSize + " , max=" + MAX_TEMPLATE_SIZE_IN_BYTES); status = Status.UNRECOVERABLE_ERROR; errorString = "Download file size is too large"; return 0; } if (remoteSize == 0) { remoteSize = MAX_TEMPLATE_SIZE_IN_BYTES; } InputStream in = !chunked ? new BufferedInputStream(request.getResponseBodyAsStream()) : new ChunkedInputStream(request.getResponseBodyAsStream()); RandomAccessFile out = new RandomAccessFile(file, "rwd"); out.seek(localFileSize); s_logger.info("Starting download from " + getDownloadUrl() + " to " + toFile + " remoteSize=" + remoteSize + " , max size=" + MAX_TEMPLATE_SIZE_IN_BYTES); byte[] block = new byte[CHUNK_SIZE]; long offset = 0; boolean done = false; status = TemplateDownloader.Status.IN_PROGRESS; while (!done && status != Status.ABORTED && offset <= remoteSize) { if ((bytes = in.read(block, 0, CHUNK_SIZE)) > -1) { out.write(block, 0, bytes); offset += bytes; out.seek(offset); totalBytes += bytes; } else { done = true; } } Date finish = new Date(); String downloaded = "(incomplete download)"; if (totalBytes >= remoteSize) { status = TemplateDownloader.Status.DOWNLOAD_FINISHED; downloaded = "(download complete remote=" + remoteSize + "bytes)"; } errorString = "Downloaded " + totalBytes + " bytes " + downloaded; downloadTime += finish.getTime() - start.getTime(); out.close(); return totalBytes; } catch (HttpException hte) { status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; errorString = hte.getMessage(); } catch (IOException ioe) { status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; //probably a file write error? errorString = ioe.getMessage(); } finally { if (status == Status.UNRECOVERABLE_ERROR && file.exists() && !file.isDirectory()) { file.delete(); } request.releaseConnection(); if (callback != null) { callback.downloadComplete(status); } } return 0; } public String getDownloadUrl() { return downloadUrl; } public String getToFile() { File file = new File(toFile); return file.getAbsolutePath(); } public TemplateDownloader.Status getStatus() { return status; } public long getDownloadTime() { return downloadTime; } public long getDownloadedBytes() { return totalBytes; } @Override @SuppressWarnings("fallthrough") public boolean stopDownload() { switch (getStatus()) { case IN_PROGRESS: if (request != null) { request.abort(); } status = TemplateDownloader.Status.ABORTED; return true; case UNKNOWN: case NOT_STARTED: case RECOVERABLE_ERROR: case UNRECOVERABLE_ERROR: case ABORTED: status = TemplateDownloader.Status.ABORTED; case DOWNLOAD_FINISHED: File f = new File(toFile); if (f.exists()) { f.delete(); } return true; default: return true; } } @Override public int getDownloadPercent() { if (remoteSize == 0) { return 0; } return (int) (100.0 * totalBytes / remoteSize); } @Override public void run() { try { download(resume, completionCallback); } catch (Throwable t) { s_logger.warn("Caught exception during download " + t.getMessage(), t); errorString = "Failed to install: " + t.getMessage(); status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; } } @Override public void setStatus(TemplateDownloader.Status status) { this.status = status; } public boolean isResume() { return resume; } @Override public String getDownloadError() { return errorString; } @Override public String getDownloadLocalPath() { return getToFile(); } public void setResume(boolean resume) { this.resume = resume; } public void setToDir(String toDir) { this.toDir = toDir; } public String getToDir() { return toDir; } public long getMaxTemplateSizeInBytes() { return this.MAX_TEMPLATE_SIZE_IN_BYTES; } public static void main(String[] args) { String url = "http:// dev.mysql.com/get/Downloads/MySQL-5.0/mysql-noinstall-5.0.77-win32.zip/from/http://mirror.services.wisc.edu/mysql/"; try { URI uri = new java.net.URI(url); } catch (URISyntaxException e) { // TODO Auto-generated catch block e.printStackTrace(); } TemplateDownloader td = new HttpTemplateDownloader(null, url, "/tmp/mysql", null, TemplateDownloader.DEFAULT_MAX_TEMPLATE_SIZE_IN_BYTES, null, null, null, null); long bytes = td.download(true, null); if (bytes > 0) { System.out .println("Downloaded (" + bytes + " bytes)" + " in " + td.getDownloadTime() / 1000 + " secs"); } else { System.out.println("Failed download"); } } @Override public void setDownloadError(String error) { errorString = error; } @Override public boolean isInited() { return inited; } public ResourceType getResourceType() { return resourceType; } }