Java tutorial
/* * Copyright (c) 2007-2012 The Broad Institute, Inc. * SOFTWARE COPYRIGHT NOTICE * This software and its documentation are the copyright of the Broad Institute, Inc. All rights are reserved. * * This software is supplied without any warranty or guaranteed support whatsoever. The Broad Institute is not responsible for its use, misuse, or functionality. * * This software is licensed under the terms of the GNU Lesser General Public License (LGPL), * Version 2.1 which is available at http://www.opensource.org/licenses/lgpl-2.1.php. */ package org.broad.igv.util; import biz.source_code.base64Coder.Base64Coder; import net.sf.samtools.seekablestream.SeekableStream; import net.sf.samtools.util.ftp.FTPClient; import net.sf.samtools.util.ftp.FTPStream; import org.apache.log4j.Logger; import org.apache.tomcat.util.HttpDate; import org.broad.igv.Globals; import org.broad.igv.PreferenceManager; import org.broad.igv.exceptions.HttpResponseException; import org.broad.igv.gs.GSUtils; import org.broad.igv.ui.IGV; import org.broad.igv.ui.util.CancellableProgressDialog; import org.broad.igv.ui.util.ProgressMonitor; import org.broad.igv.util.collections.CI; import org.broad.igv.util.ftp.FTPUtils; import org.broad.igv.util.stream.IGVSeekableHTTPStream; import org.broad.igv.util.stream.IGVUrlHelper; import javax.net.ssl.HttpsURLConnection; import javax.net.ssl.SSLContext; import javax.net.ssl.TrustManager; import javax.net.ssl.X509TrustManager; import java.awt.*; import java.awt.event.ActionEvent; import java.awt.event.ActionListener; import java.io.*; import java.net.*; import java.security.KeyManagementException; import java.security.NoSuchAlgorithmException; import java.util.*; import java.util.List; import java.util.regex.Pattern; import java.util.zip.GZIPInputStream; /** * Wrapper utility class... for interacting with HttpURLConnection. * * @author Jim Robinson * @date 9/22/11 */ public class HttpUtils { private static Logger log = Logger.getLogger(HttpUtils.class); private static HttpUtils instance; private Map<String, Boolean> byteRangeTestMap; private ProxySettings proxySettings = null; private final int MAX_REDIRECTS = 5; private String defaultUserName = null; private char[] defaultPassword = null; private static Pattern URLmatcher = Pattern.compile(".{1,8}://.*"); // static provided to support unit testing private static boolean BYTE_RANGE_DISABLED = false; private Map<URL, Boolean> headURLCache = new HashMap<URL, Boolean>(); /** * @return the single instance */ public static HttpUtils getInstance() { if (instance == null) { instance = new HttpUtils(); } return instance; } private HttpUtils() { org.broad.tribble.util.ParsingUtils.registerHelperClass(IGVUrlHelper.class); disableCertificateValidation(); CookieHandler.setDefault(new IGVCookieManager()); Authenticator.setDefault(new IGVAuthenticator()); try { System.setProperty("java.net.useSystemProxies", "true"); } catch (Exception e) { log.info("Couldn't set useSystemProxies=true"); } byteRangeTestMap = Collections.synchronizedMap(new HashMap()); } public static boolean isRemoteURL(String string) { String lcString = string.toLowerCase(); return lcString.startsWith("http://") || lcString.startsWith("https://") || lcString.startsWith("ftp://"); } /** * Provided to support unit testing (force disable byte range requests) * * @return */ public static void disableByteRange(boolean b) { BYTE_RANGE_DISABLED = b; } /** * @param elements * @param joiner * @return * @deprecated HttpUtils.openConnection does URL encoding itself * <p/> * Join the {@code elements} with the character {@code joiner}, * URLencoding the {@code elements} along the way. {@code joiner} * is NOT URLEncoded * Example: * String[] parm_list = new String[]{"app les", "oranges", "bananas"}; * String formatted = buildURLString(Arrays.asList(parm_list), "+"); * <p/> * formatted will be "app%20les+oranges+bananas" */ @Deprecated public static String buildURLString(Iterable<String> elements, String joiner) { Iterator<String> iter = elements.iterator(); if (!iter.hasNext()) { return ""; } String wholequery = iter.next(); try { while (iter.hasNext()) { wholequery += joiner + URLEncoder.encode(iter.next(), "UTF-8"); } return wholequery; } catch (UnsupportedEncodingException e) { throw new IllegalArgumentException("Bad argument in genelist: " + e.getMessage()); } } /** * Return the contents of the url as a String. This method should only be used for queries expected to return * a small amount of data. * * @param url * @return * @throws IOException */ public String getContentsAsString(URL url) throws IOException { InputStream is = null; HttpURLConnection conn = openConnection(url, null); try { is = conn.getInputStream(); return readContents(is); } finally { if (is != null) is.close(); } } public String getContentsAsJSON(URL url) throws IOException { InputStream is = null; Map<String, String> reqProperties = new HashMap(); reqProperties.put("Accept", "application/json,text/plain"); HttpURLConnection conn = openConnection(url, reqProperties); try { is = conn.getInputStream(); return readContents(is); } finally { if (is != null) is.close(); } } /** * Open a connection stream for the URL. * * @param url * @return * @throws IOException */ public InputStream openConnectionStream(URL url) throws IOException { log.debug("Opening connection stream to " + url); if (url.getProtocol().toLowerCase().equals("ftp")) { String userInfo = url.getUserInfo(); String host = url.getHost(); String file = url.getPath(); FTPClient ftp = FTPUtils.connect(host, userInfo, new UserPasswordInputImpl()); ftp.pasv(); ftp.retr(file); return new FTPStream(ftp); } else { return openConnectionStream(url, null); } } public InputStream openConnectionStream(URL url, Map<String, String> requestProperties) throws IOException { HttpURLConnection conn = openConnection(url, requestProperties); if (conn == null) { return null; } boolean rangeRequestedNotReceived = isExpectedRangeMissing(conn, requestProperties); if (rangeRequestedNotReceived) { String msg = "Byte range requested, but no Content-Range header in response"; log.error(msg); // if(Globals.isTesting()){ // throw new IOException(msg); // } } InputStream input = conn.getInputStream(); if ("gzip".equals(conn.getContentEncoding())) { input = new GZIPInputStream(input); } return input; } boolean isExpectedRangeMissing(URLConnection conn, Map<String, String> requestProperties) { final boolean rangeRequested = (requestProperties != null) && (new CI.CIHashMap<String>(requestProperties)).containsKey("Range"); if (!rangeRequested) return false; Map<String, List<String>> headerFields = conn.getHeaderFields(); boolean rangeReceived = (headerFields != null) && (new CI.CIHashMap<List<String>>(headerFields)).containsKey("Content-Range"); return !rangeReceived; } public boolean resourceAvailable(URL url) { log.debug("Checking if resource is available: " + url); if (url.getProtocol().toLowerCase().equals("ftp")) { return FTPUtils.resourceAvailable(url); } else { try { HttpURLConnection conn = openConnectionHeadOrGet(url); int code = conn.getResponseCode(); return code >= 200 && code < 300; } catch (IOException e) { return false; } } } /** * First tries a HEAD request, then a GET request if the HEAD fails. * If the GET fails, the exception is thrown * * @param url * @return * @throws IOException */ private HttpURLConnection openConnectionHeadOrGet(URL url) throws IOException { boolean tryHead = headURLCache.containsKey(url) ? headURLCache.get(url) : true; if (tryHead) { try { HttpURLConnection conn = openConnection(url, null, "HEAD"); headURLCache.put(url, true); return conn; } catch (IOException e) { log.info("HEAD request failed for url: " + url.toExternalForm() + ". Trying GET"); headURLCache.put(url, false); } } return openConnection(url, null, "GET"); } public String getHeaderField(URL url, String key) throws IOException { HttpURLConnection conn = openConnectionHeadOrGet(url); if (conn == null) return null; return conn.getHeaderField(key); } public long getLastModified(URL url) throws IOException { HttpURLConnection conn = openConnectionHeadOrGet(url); if (conn == null) return 0; return conn.getLastModified(); } public long getContentLength(URL url) throws IOException { try { String contentLengthString = getHeaderField(url, "Content-Length"); if (contentLengthString == null) { return -1; } else { return Long.parseLong(contentLengthString); } } catch (Exception e) { log.error("Error fetching content length", e); return -1; } } /** * Compare a local and remote resource, returning true if it is believed that the * remote file is newer than the local file * * @param file * @param url * @param compareContentLength Whether to use the content length to compare files. If false, only * the modified date is used * @return true if the files are the same or the local file is newer, false if the remote file has been modified wrt the local one. * @throws IOException */ public boolean remoteIsNewer(File file, URL url, boolean compareContentLength) throws IOException { if (!file.exists()) { return false; } HttpURLConnection conn = openConnection(url, null, "HEAD"); // Check content-length first long contentLength = -1; String contentLengthString = conn.getHeaderField("Content-Length"); if (contentLengthString != null) { try { contentLength = Long.parseLong(contentLengthString); } catch (NumberFormatException e) { log.error("Error parsing content-length string: " + contentLengthString + " from URL: " + url.toString()); contentLength = -1; } } if (contentLength != file.length()) { return true; } // Compare last-modified dates String lastModifiedString = conn.getHeaderField("Last-Modified"); if (lastModifiedString == null) { return false; } else { HttpDate date = new HttpDate(); date.parse(lastModifiedString); long remoteModifiedTime = date.getTime(); long localModifiedTime = file.lastModified(); return remoteModifiedTime > localModifiedTime; } } public void updateProxySettings() { boolean useProxy; String proxyHost; int proxyPort = -1; boolean auth = false; String user = null; String pw = null; PreferenceManager prefMgr = PreferenceManager.getInstance(); useProxy = prefMgr.getAsBoolean(PreferenceManager.USE_PROXY); proxyHost = prefMgr.get(PreferenceManager.PROXY_HOST, null); try { proxyPort = Integer.parseInt(prefMgr.get(PreferenceManager.PROXY_PORT, "-1")); } catch (NumberFormatException e) { proxyPort = -1; } auth = prefMgr.getAsBoolean(PreferenceManager.PROXY_AUTHENTICATE); user = prefMgr.get(PreferenceManager.PROXY_USER, null); String pwString = prefMgr.get(PreferenceManager.PROXY_PW, null); if (pwString != null) { pw = Utilities.base64Decode(pwString); } String proxyTypeString = prefMgr.get(PreferenceManager.PROXY_TYPE, "HTTP"); Proxy.Type type = Proxy.Type.valueOf(proxyTypeString.trim().toUpperCase()); proxySettings = new ProxySettings(useProxy, user, pw, auth, proxyHost, proxyPort, type); } /** * Get the system defined proxy defined for the URI, or null if * not available. May also return a {@code Proxy} object which * represents a direct connection * * @param uri * @return */ private Proxy getSystemProxy(String uri) { try { log.debug("Getting system proxy for " + uri); ProxySelector selector = ProxySelector.getDefault(); List<Proxy> proxyList = selector.select(new URI(uri)); return proxyList.get(0); } catch (URISyntaxException e) { log.error(e.getMessage(), e); return null; } catch (NullPointerException e) { return null; } catch (Exception e) { log.error(e.getMessage(), e); return null; } } /** * Calls {@link #downloadFile(String, java.io.File, Frame, String)} * with {@code dialogsParent = null, title = null} * * @param url * @param outputFile * @return RunnableResult * @throws IOException */ public RunnableResult downloadFile(String url, File outputFile) throws IOException { URLDownloader downloader = downloadFile(url, outputFile, null, null); return downloader.getResult(); } /** * @param url * @param outputFile * @param dialogsParent Parent of dialog to show progress. If null, none shown * @return URLDownloader used to perform download * @throws IOException */ public URLDownloader downloadFile(String url, File outputFile, Frame dialogsParent, String dialogTitle) throws IOException { final URLDownloader urlDownloader = new URLDownloader(url, outputFile); boolean showProgressDialog = dialogsParent != null; if (!showProgressDialog) { urlDownloader.run(); return urlDownloader; } else { ProgressMonitor monitor = new ProgressMonitor(); urlDownloader.setMonitor(monitor); ActionListener buttonListener = new ActionListener() { @Override public void actionPerformed(ActionEvent e) { urlDownloader.cancel(true); } }; String permText = "Downloading " + url; String title = dialogTitle != null ? dialogTitle : permText; CancellableProgressDialog dialog = CancellableProgressDialog .showCancellableProgressDialog(dialogsParent, title, buttonListener, false, monitor); dialog.setPermText(permText); Dimension dms = new Dimension(600, 150); dialog.setPreferredSize(dms); dialog.setSize(dms); dialog.validate(); LongRunningTask.submit(urlDownloader); return urlDownloader; } } public void uploadGenomeSpaceFile(String uri, File file, Map<String, String> headers) throws IOException { HttpURLConnection urlconnection = null; OutputStream bos = null; URL url = new URL(uri); urlconnection = openConnection(url, headers, "PUT"); urlconnection.setDoOutput(true); urlconnection.setDoInput(true); bos = new BufferedOutputStream(urlconnection.getOutputStream()); BufferedInputStream bis = new BufferedInputStream(new FileInputStream(file)); int i; // read byte by byte until end of stream while ((i = bis.read()) > 0) { bos.write(i); } bos.close(); int responseCode = urlconnection.getResponseCode(); // Error messages below. if (responseCode >= 400) { String message = readErrorStream(urlconnection); throw new IOException("Error uploading " + file.getName() + " : " + message); } } public String createGenomeSpaceDirectory(URL url, String body) throws IOException { HttpURLConnection urlconnection = null; OutputStream bos = null; Map<String, String> headers = new HashMap<String, String>(); headers.put("Content-Type", "application/json"); headers.put("Content-Length", String.valueOf(body.getBytes().length)); urlconnection = openConnection(url, headers, "PUT"); urlconnection.setDoOutput(true); urlconnection.setDoInput(true); bos = new BufferedOutputStream(urlconnection.getOutputStream()); bos.write(body.getBytes()); bos.close(); int responseCode = urlconnection.getResponseCode(); // Error messages below. StringBuffer buf = new StringBuffer(); InputStream inputStream; if (responseCode >= 200 && responseCode < 300) { inputStream = urlconnection.getInputStream(); } else { inputStream = urlconnection.getErrorStream(); } BufferedReader br = new BufferedReader(new InputStreamReader(inputStream)); String nextLine; while ((nextLine = br.readLine()) != null) { buf.append(nextLine); buf.append('\n'); } inputStream.close(); if (responseCode >= 200 && responseCode < 300) { return buf.toString(); } else { throw new IOException("Error creating GS directory: " + buf.toString()); } } /** * Code for disabling SSL certification */ private void disableCertificateValidation() { // Create a trust manager that does not validate certificate chains TrustManager[] trustAllCerts = new TrustManager[] { new X509TrustManager() { public java.security.cert.X509Certificate[] getAcceptedIssuers() { return new java.security.cert.X509Certificate[0]; } public void checkClientTrusted(java.security.cert.X509Certificate[] certs, String authType) { } public void checkServerTrusted(java.security.cert.X509Certificate[] certs, String authType) { } } }; // Install the all-trusting trust manager try { SSLContext sc = SSLContext.getInstance("SSL"); sc.init(null, trustAllCerts, null); HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory()); } catch (NoSuchAlgorithmException e) { } catch (KeyManagementException e) { } } private String readContents(InputStream is) throws IOException { BufferedInputStream bis = new BufferedInputStream(is); ByteArrayOutputStream bos = new ByteArrayOutputStream(); int b; while ((b = bis.read()) >= 0) { bos.write(b); } return new String(bos.toByteArray()); } private String readErrorStream(HttpURLConnection connection) throws IOException { InputStream inputStream = null; try { inputStream = connection.getErrorStream(); if (inputStream == null) { return null; } return readContents(inputStream); } finally { if (inputStream != null) inputStream.close(); } } public HttpURLConnection delete(URL url) throws IOException { return openConnection(url, Collections.<String, String>emptyMap(), "DELETE"); } HttpURLConnection openConnection(URL url, Map<String, String> requestProperties) throws IOException { return openConnection(url, requestProperties, "GET"); } private HttpURLConnection openConnection(URL url, Map<String, String> requestProperties, String method) throws IOException { return openConnection(url, requestProperties, method, 0); } /** * The "real" connection method * * @param url * @param requestProperties * @param method * @return * @throws java.io.IOException */ private HttpURLConnection openConnection(URL url, Map<String, String> requestProperties, String method, int redirectCount) throws IOException { //Encode query string portions url = StringUtils.encodeURLQueryString(url); if (log.isTraceEnabled()) { log.trace(url); } //Encode base portions. Right now just spaces, most common case //TODO This is a hack and doesn't work for all characters which need it if (StringUtils.countChar(url.toExternalForm(), ' ') > 0) { String newPath = url.toExternalForm().replaceAll(" ", "%20"); url = new URL(newPath); } Proxy sysProxy = null; boolean igvProxySettingsExist = proxySettings != null && proxySettings.useProxy; //Only check for system proxy if igv proxy settings not found if (!igvProxySettingsExist) { sysProxy = getSystemProxy(url.toExternalForm()); } boolean useProxy = sysProxy != null || igvProxySettingsExist; HttpURLConnection conn; if (useProxy) { Proxy proxy = sysProxy; if (igvProxySettingsExist) { if (proxySettings.type == Proxy.Type.DIRECT) { proxy = Proxy.NO_PROXY; } else { proxy = new Proxy(proxySettings.type, new InetSocketAddress(proxySettings.proxyHost, proxySettings.proxyPort)); } } conn = (HttpURLConnection) url.openConnection(proxy); if (igvProxySettingsExist && proxySettings.auth && proxySettings.user != null && proxySettings.pw != null) { byte[] bytes = (proxySettings.user + ":" + proxySettings.pw).getBytes(); String encodedUserPwd = String.valueOf(Base64Coder.encode(bytes)); conn.setRequestProperty("Proxy-Authorization", "Basic " + encodedUserPwd); } } else { conn = (HttpURLConnection) url.openConnection(); } if (GSUtils.isGenomeSpace(url)) { conn.setRequestProperty("Accept", "application/json,text/plain"); } else { conn.setRequestProperty("Accept", "text/plain"); } //------// //There seems to be a bug with JWS caches //So we avoid caching //This default is persistent, really should be available statically but isn't conn.setDefaultUseCaches(false); conn.setUseCaches(false); //------// conn.setConnectTimeout(Globals.CONNECT_TIMEOUT); conn.setReadTimeout(Globals.READ_TIMEOUT); conn.setRequestMethod(method); conn.setRequestProperty("Connection", "Keep-Alive"); if (requestProperties != null) { for (Map.Entry<String, String> prop : requestProperties.entrySet()) { conn.setRequestProperty(prop.getKey(), prop.getValue()); } } conn.setRequestProperty("User-Agent", Globals.applicationString()); if (method.equals("PUT")) { return conn; } else { int code = conn.getResponseCode(); if (log.isDebugEnabled()) { //logHeaders(conn); } // Redirects. These can occur even if followRedirects == true if there is a change in protocol, // for example http -> https. if (code >= 300 && code < 400) { if (redirectCount > MAX_REDIRECTS) { throw new IOException("Too many redirects"); } String newLocation = conn.getHeaderField("Location"); log.debug("Redirecting to " + newLocation); return openConnection(new URL(newLocation), requestProperties, method, redirectCount++); } // TODO -- handle other response codes. else if (code >= 400) { String message; if (code == 404) { message = "File not found: " + url.toString(); throw new FileNotFoundException(message); } else if (code == 401) { // Looks like this only happens when user hits "Cancel". // message = "Not authorized to view this file"; // JOptionPane.showMessageDialog(null, message, "HTTP error", JOptionPane.ERROR_MESSAGE); redirectCount = MAX_REDIRECTS + 1; return null; } else { message = conn.getResponseMessage(); } String details = readErrorStream(conn); log.error("URL: " + url.toExternalForm() + ". error stream: " + details); log.error("Code: " + code + ". " + message); HttpResponseException exc = new HttpResponseException(code); throw exc; } } return conn; } //Used for testing sometimes, please do not delete private void logHeaders(HttpURLConnection conn) { Map<String, List<String>> headerFields = conn.getHeaderFields(); log.debug("Headers for " + conn.getURL()); for (Map.Entry<String, List<String>> header : headerFields.entrySet()) { log.debug(header.getKey() + ": " + org.apache.commons.lang.StringUtils.join(header.getValue(), ',')); } } public void setDefaultPassword(String defaultPassword) { this.defaultPassword = defaultPassword.toCharArray(); } public void setDefaultUserName(String defaultUserName) { this.defaultUserName = defaultUserName; } public void clearDefaultCredentials() { this.defaultPassword = null; this.defaultUserName = null; } /** * Test to see if this client can successfully retrieve a portion of a remote file using the byte-range header. * This is not a test of the server, but the client. In some environments the byte-range header gets removed * by filters after the request is made by IGV. * * @return */ public boolean useByteRange(URL url) { if (BYTE_RANGE_DISABLED) return false; // We can test byte-range success for hosts we can reach. synchronized (byteRangeTestMap) { final String host = url.getHost(); if (byteRangeTestMap.containsKey(host)) { return byteRangeTestMap.get(host); } else { SeekableStream str = null; try { boolean byteRangeTestSuccess = true; if (host.contains("broadinstitute.org")) { byteRangeTestSuccess = testBroadHost(host); } else { // Non-broad URL int l = (int) Math.min(1000, HttpUtils.getInstance().getContentLength(url)); if (l > 100) { byte[] firstBytes = new byte[l]; str = new IGVSeekableHTTPStream(url); str.readFully(firstBytes); int end = firstBytes.length; int start = end - 100; str.seek(start); int len = end - start; byte[] buffer = new byte[len]; int n = 0; while (n < len) { int count = str.read(buffer, n, len - n); if (count < 0) throw new EOFException(); n += count; } for (int i = 0; i < len; i++) { if (buffer[i] != firstBytes[i + start]) { byteRangeTestSuccess = false; break; } } } else { // Too small a sample to test, or unknown content length. Return "true" but don't record // this host as tested. return true; } } if (byteRangeTestSuccess) { log.info("Range-byte request succeeded"); } else { log.info("Range-byte test failed -- problem with client network environment."); } byteRangeTestMap.put(host, byteRangeTestSuccess); return byteRangeTestSuccess; } catch (IOException e) { log.error("Error while testing byte range " + e.getMessage()); // We could not reach the test server, so we can't know if this client can do byte-range tests or // not. Take the "optimistic" view. return true; } finally { if (str != null) try { str.close(); } catch (IOException e) { log.error("Error closing stream (" + url.toExternalForm() + ")", e); } } } } } private boolean testBroadHost(String host) throws IOException { // Test broad urls for successful byte range requests. log.info("Testing range-byte request on host: " + host); String testURL; if (host.startsWith("igvdata.broadinstitute.org")) { // Amazon cloud front testURL = "http://igvdata.broadinstitute.org/genomes/seq/hg19/chr12.txt"; } else if (host.startsWith("igv.broadinstitute.org")) { // Amazon S3 testURL = "http://igv.broadinstitute.org/genomes/seq/hg19/chr12.txt"; } else { // All others testURL = "http://www.broadinstitute.org/igvdata/annotations/seq/hg19/chr12.txt"; } byte[] expectedBytes = { 'T', 'C', 'G', 'C', 'T', 'T', 'G', 'A', 'A', 'C', 'C', 'C', 'G', 'G', 'G', 'A', 'G', 'A', 'G', 'G' }; IGVSeekableHTTPStream str = null; try { str = new IGVSeekableHTTPStream(new URL(testURL)); str.seek(25350000); byte[] buffer = new byte[80000]; str.read(buffer); String result = new String(buffer); for (int i = 0; i < expectedBytes.length; i++) { if (buffer[i] != expectedBytes[i]) { return false; } } return true; } finally { if (str != null) str.close(); } } public void shutdown() { // Do any cleanup required here } /** * Checks if the string is a URL (not necessarily remote, can be any protocol) * * @param f * @return */ public static boolean isURL(String f) { return f.startsWith("http:") || f.startsWith("ftp:") || f.startsWith("https:") || URLmatcher.matcher(f).matches(); } public static Map<String, String> parseQueryString(String query) { String[] params = query.split("&"); Map<String, String> map = new HashMap<String, String>(); for (String param : params) { String[] name_val = param.split("=", 2); if (name_val.length == 2) { map.put(name_val[0], name_val[1]); } } return map; } public static class ProxySettings { boolean auth = false; String user; String pw; boolean useProxy; String proxyHost; int proxyPort = -1; Proxy.Type type; public ProxySettings(boolean useProxy, String user, String pw, boolean auth, String proxyHost, int proxyPort) { this(useProxy, user, pw, auth, proxyHost, proxyPort, Proxy.Type.HTTP); this.auth = auth; } public ProxySettings(boolean useProxy, String user, String pw, boolean auth, String proxyHost, int proxyPort, Proxy.Type proxyType) { this.auth = auth; this.proxyHost = proxyHost; this.proxyPort = proxyPort; this.pw = pw; this.useProxy = useProxy; this.user = user; this.type = proxyType; } } /** * The default authenticator */ public class IGVAuthenticator extends Authenticator { Hashtable<String, PasswordAuthentication> pwCache = new Hashtable<String, PasswordAuthentication>(); HashSet<String> cacheAttempts = new HashSet<String>(); /** * Called when password authentication is needed. * * @return */ @Override protected synchronized PasswordAuthentication getPasswordAuthentication() { RequestorType type = getRequestorType(); String urlString = getRequestingURL().toString(); boolean isProxyChallenge = type == RequestorType.PROXY; // Cache user entered PWs. In normal use this shouldn't be necessary as credentials are cached upstream, // but if loading many files in parallel (e.g. from sessions) calls to this method can queue up before the // user enters their credentials, causing needless reentry. String pKey = type.toString() + getRequestingProtocol() + getRequestingHost(); PasswordAuthentication pw = pwCache.get(pKey); if (pw != null) { // Prevents infinite loop if credentials are incorrect if (cacheAttempts.contains(urlString)) { cacheAttempts.remove(urlString); } else { cacheAttempts.add(urlString); return pw; } } if (isProxyChallenge) { if (proxySettings.auth && proxySettings.user != null && proxySettings.pw != null) { return new PasswordAuthentication(proxySettings.user, proxySettings.pw.toCharArray()); } } if (defaultUserName != null && defaultPassword != null) { return new PasswordAuthentication(defaultUserName, defaultPassword); } Frame owner = IGV.hasInstance() ? IGV.getMainFrame() : null; boolean isGenomeSpace = GSUtils.isGenomeSpace(getRequestingURL()); if (isGenomeSpace) { // If we are being challenged by GS the token must be bad/expired GSUtils.logout(); } LoginDialog dlg = new LoginDialog(owner, isGenomeSpace, urlString, isProxyChallenge); dlg.setVisible(true); if (dlg.isCanceled()) { return null; } else { final String userString = dlg.getUsername(); final char[] userPass = dlg.getPassword(); if (isProxyChallenge) { proxySettings.user = userString; proxySettings.pw = new String(userPass); } pw = new PasswordAuthentication(userString, userPass); pwCache.put(pKey, pw); return pw; } } } /** * Runnable for downloading a file from a URL. * Downloading is buffered, and can be cancelled (between buffers) * via {@link #cancel(boolean)} */ public class URLDownloader implements Runnable { private ProgressMonitor monitor = null; private final URL srcUrl; private final File outputFile; private volatile boolean started = false; private volatile boolean done = false; private volatile boolean cancelled = false; private volatile RunnableResult result; public URLDownloader(String url, File outputFile) throws MalformedURLException { this.srcUrl = new URL(url); this.outputFile = outputFile; } @Override public void run() { if (this.cancelled) { return; } started = true; try { this.result = doDownload(); } catch (IOException e) { log.error(e.getMessage(), e); } finally { this.done(); } } /** * Return the result. Must be called after run is complete * * @return */ public RunnableResult getResult() { if (!this.done) throw new IllegalStateException("Must wait for run to finish before getting result"); return this.result; } private RunnableResult doDownload() throws IOException { log.info("Downloading " + srcUrl + " to " + outputFile.getAbsolutePath()); HttpURLConnection conn = openConnection(this.srcUrl, null); long contentLength = -1; String contentLengthString = conn.getHeaderField("Content-Length"); if (contentLengthString != null) { contentLength = Long.parseLong(contentLengthString); } InputStream is = null; OutputStream out = null; long downloaded = 0; long downSinceLast = 0; String curStatus; String msg1 = String.format("downloaded of %s total", contentLength >= 0 ? bytesToByteCountString(contentLength) : "unknown"); int perc = 0; try { is = conn.getInputStream(); out = new FileOutputStream(outputFile); byte[] buf = new byte[64 * 1024]; int counter = 0; int interval = 100; int bytesRead = 0; while (!this.cancelled && (bytesRead = is.read(buf)) != -1) { out.write(buf, 0, bytesRead); downloaded += bytesRead; downSinceLast += bytesRead; counter = (counter + 1) % interval; if (counter == 0 && this.monitor != null) { curStatus = String.format("%s %s", bytesToByteCountString(downloaded), msg1); this.monitor.updateStatus(curStatus); if (contentLength >= 0) { perc = (int) ((downSinceLast * 100) / contentLength); this.monitor.fireProgressChange(perc); if (perc >= 1) downSinceLast = 0; } } } log.info("Download complete. Total bytes downloaded = " + downloaded); } finally { if (is != null) is.close(); if (out != null) { out.flush(); out.close(); } } long fileLength = outputFile.length(); if (this.cancelled) return RunnableResult.CANCELLED; boolean knownComplete = contentLength == fileLength; //Assume success if file length not known if (knownComplete || contentLength < 0) { if (this.monitor != null) { this.monitor.fireProgressChange(100); this.monitor.updateStatus("Done"); } return RunnableResult.SUCCESS; } else { return RunnableResult.FAILURE; } } protected void done() { this.done = true; } public boolean isDone() { return this.done; } /** * See {@link java.util.concurrent.FutureTask#cancel(boolean)} * * @param mayInterruptIfRunning * @return */ public boolean cancel(boolean mayInterruptIfRunning) { if (this.started && !mayInterruptIfRunning) { return false; } this.cancelled = true; return true; } public void setMonitor(ProgressMonitor monitor) { this.monitor = monitor; } /** * Convert bytes to human-readable string. * e.g. 102894 -> 102.89 KB. If too big or too small, * doesn't append a prefix just returns {@code bytes + " B"} * * @param bytes * @return */ public String bytesToByteCountString(long bytes) { int unit = 1000; String prefs = "KMGT"; if (bytes < unit) return bytes + " B"; int exp = (int) (Math.log(bytes) / Math.log(unit)); if (exp <= 0 || exp >= prefs.length()) return bytes + " B"; String pre = (prefs).charAt(exp - 1) + ""; return String.format("%.2f %sB", bytes / Math.pow(unit, exp), pre); } } /** * Provide override for unit tests */ public void setAuthenticator(Authenticator authenticator) { Authenticator.setDefault(authenticator); } /** * For unit tests */ public void resetAuthenticator() { Authenticator.setDefault(new IGVAuthenticator()); } /** * Extension of CookieManager that grabs cookies from the GenomeSpace identity server to store locally. * This is to support the GenomeSpace "single sign-on". Examples ... * gs-username=igvtest; Domain=.genomespace.org; Expires=Mon, 21-Jul-2031 03:27:23 GMT; Path=/ * gs-token=HnR9rBShNO4dTXk8cKXVJT98Oe0jWVY+; Domain=.genomespace.org; Expires=Mon, 21-Jul-2031 03:27:23 GMT; Path=/ */ static class IGVCookieManager extends CookieManager { @Override public Map<String, List<String>> get(URI uri, Map<String, List<String>> requestHeaders) throws IOException { Map<String, List<String>> headers = new HashMap<String, List<String>>(); headers.putAll(super.get(uri, requestHeaders)); if (GSUtils.isGenomeSpace(uri.toURL())) { String token = GSUtils.getGSToken(); if (token != null) { List<String> cookieList = headers.get("Cookie"); boolean needsTokenCookie = true; if (cookieList == null) { cookieList = new ArrayList<String>(1); headers.put("Cookie", cookieList); } for (String cookie : cookieList) { if (cookie.startsWith("gs-token")) { needsTokenCookie = false; break; } } if (needsTokenCookie) { cookieList.add("gs-token=" + token); } } } return Collections.unmodifiableMap(headers); } @Override public void put(URI uri, Map<String, List<String>> responseHeaders) throws IOException { String urilc = uri.toString().toLowerCase(); if (urilc.contains("identity") && urilc.contains("genomespace")) { List<String> cookies = responseHeaders.get("Set-Cookie"); if (cookies != null) { for (String cstring : cookies) { List<HttpCookie> cookieList = HttpCookie.parse(cstring); for (HttpCookie cookie : cookieList) { String cookieName = cookie.getName(); String value = cookie.getValue(); if (cookieName.equals("gs-token")) { //log.debug("gs-token: " + value); GSUtils.setGSToken(value); } else if (cookieName.equals("gs-username")) { //log.debug("gs-username: " + value); GSUtils.setGSUser(value); } } } } } super.put(uri, responseHeaders); } } }