Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.brooklyn.util.http; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkState; import java.io.InputStream; import java.net.HttpURLConnection; import java.net.URI; import java.net.URL; import java.net.URLConnection; import java.security.cert.CertificateException; import java.security.cert.X509Certificate; import java.util.ArrayList; import java.util.Collection; import java.util.Map; import java.util.Map.Entry; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicReference; import com.google.common.base.Throwables; import org.apache.brooklyn.util.crypto.SslTrustUtils; import org.apache.brooklyn.util.exceptions.Exceptions; import org.apache.brooklyn.util.net.URLParamEncoder; import org.apache.brooklyn.util.stream.Streams; import org.apache.brooklyn.util.text.Strings; import org.apache.brooklyn.util.time.Duration; import org.apache.brooklyn.util.time.Time; import org.apache.commons.codec.binary.Base64; import org.apache.http.ConnectionReuseStrategy; import org.apache.http.HttpEntity; import org.apache.http.HttpRequest; import org.apache.http.HttpResponse; import org.apache.http.NameValuePair; import org.apache.http.auth.AuthScope; import org.apache.http.auth.Credentials; import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.HttpClient; import org.apache.http.client.entity.UrlEncodedFormEntity; import org.apache.http.client.methods.HttpDelete; import org.apache.http.client.methods.HttpEntityEnclosingRequestBase; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpHead; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpPut; import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.conn.ClientConnectionManager; import org.apache.http.conn.scheme.Scheme; import org.apache.http.conn.scheme.SchemeSocketFactory; import org.apache.http.conn.ssl.SSLSocketFactory; import org.apache.http.conn.ssl.TrustSelfSignedStrategy; import org.apache.http.conn.ssl.TrustStrategy; import org.apache.http.conn.ssl.X509HostnameVerifier; import org.apache.http.entity.ByteArrayEntity; import org.apache.http.impl.client.DefaultHttpClient; import org.apache.http.impl.client.LaxRedirectStrategy; import org.apache.http.message.BasicNameValuePair; import org.apache.http.params.BasicHttpParams; import org.apache.http.params.HttpConnectionParams; import org.apache.http.params.HttpParams; import org.apache.http.util.EntityUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.base.Function; import com.google.common.base.Joiner; import com.google.common.base.Optional; import com.google.common.collect.Iterables; import com.google.common.collect.Multimap; import javax.net.ssl.HostnameVerifier; import javax.net.ssl.HttpsURLConnection; import javax.net.ssl.SSLSession; /** * A utility tool for HTTP operations. */ public class HttpTool { private static final Logger LOG = LoggerFactory.getLogger(HttpTool.class); static final ExecutorService executor = Executors.newCachedThreadPool(); /** * Connects to the given url and returns the connection. * Caller should {@code connection.getInputStream().close()} the result of this * (especially if they are making heavy use of this method). */ public static URLConnection connectToUrl(String u) throws Exception { final URL url = new URL(u); final AtomicReference<Exception> exception = new AtomicReference<Exception>(); // sometimes openConnection hangs, so run in background Future<URLConnection> f = executor.submit(new Callable<URLConnection>() { public URLConnection call() { try { HttpsURLConnection.setDefaultHostnameVerifier(new HostnameVerifier() { @Override public boolean verify(String s, SSLSession sslSession) { return true; } }); URLConnection connection = url.openConnection(); TrustingSslSocketFactory.configure(connection); connection.connect(); connection.getContentLength(); // Make sure the connection is made. return connection; } catch (Exception e) { exception.set(e); LOG.debug("Error connecting to url " + url + " (propagating): " + e, e); } return null; } }); try { URLConnection result = null; try { result = f.get(60, TimeUnit.SECONDS); } catch (InterruptedException e) { throw e; } catch (Exception e) { LOG.debug("Error connecting to url " + url + ", probably timed out (rethrowing): " + e); throw new IllegalStateException( "Connect to URL not complete within 60 seconds, for url " + url + ": " + e); } if (exception.get() != null) { LOG.debug("Error connecting to url " + url + ", thread caller of " + exception, new Throwable("source of rethrown error " + exception)); throw exception.get(); } else { return result; } } finally { f.cancel(true); } } public static int getHttpStatusCode(String url) throws Exception { URLConnection connection = connectToUrl(url); long startTime = System.currentTimeMillis(); int status = ((HttpURLConnection) connection).getResponseCode(); // read fully if possible, then close everything, trying to prevent cached threads at server consumeAndCloseQuietly((HttpURLConnection) connection); if (LOG.isDebugEnabled()) LOG.debug("connection to {} ({}ms) gives {}", new Object[] { url, (System.currentTimeMillis() - startTime), status }); return status; } public static String getContent(String url) { try { return Streams.readFullyString(SslTrustUtils.trustAll(new URL(url).openConnection()).getInputStream()); } catch (Exception e) { throw Throwables.propagate(e); } } public static String getErrorContent(String url) { try { HttpURLConnection connection = (HttpURLConnection) connectToUrl(url); long startTime = System.currentTimeMillis(); String err; int status; try { InputStream errStream = connection.getErrorStream(); err = Streams.readFullyString(errStream); status = connection.getResponseCode(); } finally { closeQuietly(connection); } if (LOG.isDebugEnabled()) LOG.debug("read of err {} ({}ms) complete; http code {}", new Object[] { url, Time.makeTimeStringRounded(System.currentTimeMillis() - startTime), status }); return err; } catch (Exception e) { throw Exceptions.propagate(e); } } /** * Consumes the input stream entirely and then cleanly closes the connection. * Ignores all exceptions completely, not even logging them! * * Consuming the stream fully is useful for preventing idle TCP connections. * @see <a href="http://docs.oracle.com/javase/8/docs/technotes/guides/net/http-keepalive.html">Persistent Connections</a> */ public static void consumeAndCloseQuietly(HttpURLConnection connection) { try { Streams.readFully(connection.getInputStream()); } catch (Exception e) { } closeQuietly(connection); } /** * Closes all streams of the connection, and disconnects it. Ignores all exceptions completely, * not even logging them! */ public static void closeQuietly(HttpURLConnection connection) { try { connection.disconnect(); } catch (Exception e) { } try { connection.getInputStream().close(); } catch (Exception e) { } try { connection.getOutputStream().close(); } catch (Exception e) { } try { connection.getErrorStream().close(); } catch (Exception e) { } } /** Apache HTTP commons utility for trusting all. * <p> * For generic java HTTP usage, see {@link SslTrustUtils#trustAll(java.net.URLConnection)} * and static constants in the same class. */ public static class TrustAllStrategy implements TrustStrategy { @Override public boolean isTrusted(X509Certificate[] chain, String authType) throws CertificateException { return true; } } public static HttpClientBuilder httpClientBuilder() { return new HttpClientBuilder(); } public static class HttpClientBuilder { private ClientConnectionManager clientConnectionManager; private HttpParams httpParams; private URI uri; private Integer port; private Credentials credentials; private boolean laxRedirect; private Boolean https; private SchemeSocketFactory socketFactory; private ConnectionReuseStrategy reuseStrategy; private boolean trustAll; private boolean trustSelfSigned; public HttpClientBuilder clientConnectionManager(ClientConnectionManager val) { this.clientConnectionManager = checkNotNull(val, "clientConnectionManager"); return this; } public HttpClientBuilder httpParams(HttpParams val) { checkState(httpParams == null, "Must not call httpParams multiple times, or after other methods like connectionTimeout"); this.httpParams = checkNotNull(val, "httpParams"); return this; } public HttpClientBuilder connectionTimeout(Duration val) { if (httpParams == null) httpParams = new BasicHttpParams(); long millis = checkNotNull(val, "connectionTimeout").toMilliseconds(); if (millis > Integer.MAX_VALUE) throw new IllegalStateException( "HttpClient only accepts upto max-int millis for connectionTimeout, but given " + val); HttpConnectionParams.setConnectionTimeout(httpParams, (int) millis); return this; } public HttpClientBuilder socketTimeout(Duration val) { if (httpParams == null) httpParams = new BasicHttpParams(); long millis = checkNotNull(val, "socketTimeout").toMilliseconds(); if (millis > Integer.MAX_VALUE) throw new IllegalStateException( "HttpClient only accepts upto max-int millis for socketTimeout, but given " + val); HttpConnectionParams.setSoTimeout(httpParams, (int) millis); return this; } public HttpClientBuilder reuseStrategy(ConnectionReuseStrategy val) { this.reuseStrategy = checkNotNull(val, "reuseStrategy"); return this; } public HttpClientBuilder uri(String val) { return uri(URI.create(checkNotNull(val, "uri"))); } public HttpClientBuilder uri(URI val) { this.uri = checkNotNull(val, "uri"); if (https == null) https = ("https".equalsIgnoreCase(uri.getScheme())); return this; } public HttpClientBuilder port(int val) { this.port = val; return this; } public HttpClientBuilder credentials(Credentials val) { this.credentials = checkNotNull(val, "credentials"); return this; } public void credential(Optional<Credentials> val) { if (val.isPresent()) credentials = val.get(); } /** similar to curl --post301 -L` */ public HttpClientBuilder laxRedirect(boolean val) { this.laxRedirect = val; return this; } public HttpClientBuilder https(boolean val) { this.https = val; return this; } public HttpClientBuilder socketFactory(SchemeSocketFactory val) { this.socketFactory = checkNotNull(val, "socketFactory"); return this; } public HttpClientBuilder trustAll() { this.trustAll = true; return this; } public HttpClientBuilder trustSelfSigned() { this.trustSelfSigned = true; return this; } public HttpClient build() { final DefaultHttpClient httpClient = new DefaultHttpClient(clientConnectionManager); httpClient.setParams(httpParams); // support redirects for POST (similar to `curl --post301 -L`) // http://stackoverflow.com/questions/3658721/httpclient-4-error-302-how-to-redirect if (laxRedirect) { httpClient.setRedirectStrategy(new LaxRedirectStrategy()); } if (reuseStrategy != null) { httpClient.setReuseStrategy(reuseStrategy); } if (https == Boolean.TRUE || (uri != null && uri.toString().startsWith("https:"))) { try { if (port == null) { port = (uri != null && uri.getPort() >= 0) ? uri.getPort() : 443; } if (socketFactory == null) { if (trustAll) { TrustStrategy trustStrategy = new TrustAllStrategy(); X509HostnameVerifier hostnameVerifier = SSLSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER; socketFactory = new SSLSocketFactory(trustStrategy, hostnameVerifier); } else if (trustSelfSigned) { TrustStrategy trustStrategy = new TrustSelfSignedStrategy(); X509HostnameVerifier hostnameVerifier = SSLSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER; socketFactory = new SSLSocketFactory(trustStrategy, hostnameVerifier); } else { // Using default https scheme: based on default java truststore, which is pretty strict! } } if (socketFactory != null) { Scheme sch = new Scheme("https", port, socketFactory); httpClient.getConnectionManager().getSchemeRegistry().register(sch); } } catch (Exception e) { LOG.warn("Error setting trust for uri {}", uri); throw Exceptions.propagate(e); } } // Set credentials if (uri != null && credentials != null) { String hostname = uri.getHost(); int port = uri.getPort(); httpClient.getCredentialsProvider().setCredentials(new AuthScope(hostname, port), credentials); } if (uri == null && credentials != null) { LOG.warn("credentials have no effect in builder unless URI for host is specified"); } return httpClient; } } protected static abstract class HttpRequestBuilder<B extends HttpRequestBuilder<B, R>, R extends HttpRequest> { protected R req; protected HttpRequestBuilder(R req) { this.req = req; } @SuppressWarnings("unchecked") protected B self() { return (B) this; } public B headers(Map<String, String> headers) { if (headers != null) { for (Map.Entry<String, String> entry : headers.entrySet()) { req.addHeader(entry.getKey(), entry.getValue()); } } return self(); } public B headers(Multimap<String, String> headers) { if (headers != null) { for (Map.Entry<String, String> entry : headers.entries()) { req.addHeader(entry.getKey(), entry.getValue()); } } return self(); } public R build() { return req; } } protected static abstract class HttpEntityEnclosingRequestBaseBuilder<B extends HttpEntityEnclosingRequestBaseBuilder<B, R>, R extends HttpEntityEnclosingRequestBase> extends HttpRequestBuilder<B, R> { protected HttpEntityEnclosingRequestBaseBuilder(R req) { super(req); } public B body(byte[] body) { if (body != null) { HttpEntity httpEntity = new ByteArrayEntity(body); req.setEntity(httpEntity); } return self(); } } public static class HttpGetBuilder extends HttpRequestBuilder<HttpGetBuilder, HttpGet> { public HttpGetBuilder(URI uri) { super(new HttpGet(uri)); } } public static class HttpHeadBuilder extends HttpRequestBuilder<HttpHeadBuilder, HttpHead> { public HttpHeadBuilder(URI uri) { super(new HttpHead(uri)); } } public static class HttpDeleteBuilder extends HttpRequestBuilder<HttpDeleteBuilder, HttpDelete> { public HttpDeleteBuilder(URI uri) { super(new HttpDelete(uri)); } } public static class HttpPostBuilder extends HttpEntityEnclosingRequestBaseBuilder<HttpPostBuilder, HttpPost> { HttpPostBuilder(URI uri) { super(new HttpPost(uri)); } } public static class HttpFormPostBuilder extends HttpRequestBuilder<HttpFormPostBuilder, HttpPost> { HttpFormPostBuilder(URI uri) { super(new HttpPost(uri)); } public HttpFormPostBuilder params(Map<String, String> params) { if (params != null) { Collection<NameValuePair> httpParams = new ArrayList<NameValuePair>(params.size()); for (Entry<String, String> param : params.entrySet()) { httpParams.add(new BasicNameValuePair(param.getKey(), param.getValue())); } req.setEntity(new UrlEncodedFormEntity(httpParams)); } return self(); } } public static class HttpPutBuilder extends HttpEntityEnclosingRequestBaseBuilder<HttpPutBuilder, HttpPut> { public HttpPutBuilder(URI uri) { super(new HttpPut(uri)); } } public static HttpToolResponse httpGet(HttpClient httpClient, URI uri, Map<String, String> headers) { HttpGet req = new HttpGetBuilder(uri).headers(headers).build(); return execAndConsume(httpClient, req); } public static HttpToolResponse httpPost(HttpClient httpClient, URI uri, Map<String, String> headers, byte[] body) { HttpPost req = new HttpPostBuilder(uri).headers(headers).body(body).build(); return execAndConsume(httpClient, req); } public static HttpToolResponse httpPut(HttpClient httpClient, URI uri, Map<String, String> headers, byte[] body) { HttpPut req = new HttpPutBuilder(uri).headers(headers).body(body).build(); return execAndConsume(httpClient, req); } public static HttpToolResponse httpPost(HttpClient httpClient, URI uri, Map<String, String> headers, Map<String, String> params) { HttpPost req = new HttpFormPostBuilder(uri).headers(headers).params(params).build(); return execAndConsume(httpClient, req); } public static HttpToolResponse httpDelete(HttpClient httpClient, URI uri, Map<String, String> headers) { HttpDelete req = new HttpDeleteBuilder(uri).headers(headers).build(); return execAndConsume(httpClient, req); } public static HttpToolResponse httpHead(HttpClient httpClient, URI uri, Map<String, String> headers) { HttpHead req = new HttpHeadBuilder(uri).headers(headers).build(); return execAndConsume(httpClient, req); } public static HttpToolResponse execAndConsume(HttpClient httpClient, HttpUriRequest req) { long startTime = System.currentTimeMillis(); try { HttpResponse httpResponse = httpClient.execute(req); try { return new HttpToolResponse(httpResponse, startTime); } finally { EntityUtils.consume(httpResponse.getEntity()); } } catch (Exception e) { throw Exceptions.propagate(e); } } public static boolean isStatusCodeHealthy(int code) { return (code >= 200 && code <= 299); } public static String toBasicAuthorizationValue(UsernamePasswordCredentials credentials) { return "Basic " + Base64 .encodeBase64String((credentials.getUserName() + ":" + credentials.getPassword()).getBytes()); } public static String encodeUrlParams(Map<?, ?> data) { if (data == null) return ""; Iterable<String> args = Iterables.transform(data.entrySet(), new Function<Map.Entry<?, ?>, String>() { @Override public String apply(Map.Entry<?, ?> entry) { Object k = entry.getKey(); Object v = entry.getValue(); return URLParamEncoder.encode(Strings.toString(k)) + (v != null ? "=" + URLParamEncoder.encode(Strings.toString(v)) : ""); } }); return Joiner.on("&").join(args); } }