List of usage examples for org.apache.http.impl.client HttpClientBuilder setConnectionManager
public final HttpClientBuilder setConnectionManager(final HttpClientConnectionManager connManager)
From source file:lucee.runtime.tag.Http.java
private void ssl(HttpClientBuilder builder) throws PageException { try {//from ww w. j a v a 2s .com // SSLContext sslcontext = SSLContexts.createSystemDefault(); SSLContext sslcontext = SSLContext.getInstance("TLSv1.2"); if (!StringUtil.isEmpty(this.clientCert)) { if (this.clientCertPassword == null) this.clientCertPassword = ""; File ksFile = new File(this.clientCert); KeyStore clientStore = KeyStore.getInstance("PKCS12"); clientStore.load(new FileInputStream(ksFile), this.clientCertPassword.toCharArray()); KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); kmf.init(clientStore, this.clientCertPassword.toCharArray()); sslcontext.init(kmf.getKeyManagers(), null, new java.security.SecureRandom()); } else { sslcontext.init(null, null, new java.security.SecureRandom()); } final SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactoryImpl(sslcontext, new DefaultHostnameVerifierImpl()); builder.setSSLSocketFactory(sslsf); Registry<ConnectionSocketFactory> reg = RegistryBuilder.<ConnectionSocketFactory>create() .register("http", PlainConnectionSocketFactory.getSocketFactory()).register("https", sslsf) .build(); PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager( new DefaultHttpClientConnectionOperatorImpl(reg), null, -1, TimeUnit.MILLISECONDS); // TODO review -1 setting builder.setConnectionManager(cm); } catch (Exception e) { throw Caster.toPageException(e); } }
From source file:org.codelibs.fess.crawler.client.http.HcHttpClient.java
public synchronized void init() { if (httpClient != null) { return;/*from w w w . j a v a2 s .c om*/ } if (logger.isDebugEnabled()) { logger.debug("Initializing " + HcHttpClient.class.getName()); } super.init(); // robots.txt parser final Boolean robotsTxtEnabled = getInitParameter(ROBOTS_TXT_ENABLED_PROPERTY, Boolean.TRUE, Boolean.class); if (robotsTxtHelper != null) { robotsTxtHelper.setEnabled(robotsTxtEnabled.booleanValue()); } // httpclient final org.apache.http.client.config.RequestConfig.Builder requestConfigBuilder = RequestConfig.custom(); final HttpClientBuilder httpClientBuilder = HttpClientBuilder.create(); final Integer connectionTimeoutParam = getInitParameter(CONNECTION_TIMEOUT_PROPERTY, connectionTimeout, Integer.class); if (connectionTimeoutParam != null) { requestConfigBuilder.setConnectTimeout(connectionTimeoutParam); } final Integer soTimeoutParam = getInitParameter(SO_TIMEOUT_PROPERTY, soTimeout, Integer.class); if (soTimeoutParam != null) { requestConfigBuilder.setSocketTimeout(soTimeoutParam); } // AuthSchemeFactory final RegistryBuilder<AuthSchemeProvider> authSchemeProviderBuilder = RegistryBuilder.create(); @SuppressWarnings("unchecked") final Map<String, AuthSchemeProvider> factoryMap = getInitParameter(AUTH_SCHEME_PROVIDERS_PROPERTY, authSchemeProviderMap, Map.class); if (factoryMap != null) { for (final Map.Entry<String, AuthSchemeProvider> entry : factoryMap.entrySet()) { authSchemeProviderBuilder.register(entry.getKey(), entry.getValue()); } } // user agent userAgent = getInitParameter(USER_AGENT_PROPERTY, userAgent, String.class); if (StringUtil.isNotBlank(userAgent)) { httpClientBuilder.setUserAgent(userAgent); } final HttpRoutePlanner planner = buildRoutePlanner(); if (planner != null) { httpClientBuilder.setRoutePlanner(planner); } // Authentication final Authentication[] siteCredentialList = getInitParameter(BASIC_AUTHENTICATIONS_PROPERTY, new Authentication[0], Authentication[].class); for (final Authentication authentication : siteCredentialList) { final AuthScope authScope = authentication.getAuthScope(); credentialsProvider.setCredentials(authScope, authentication.getCredentials()); final AuthScheme authScheme = authentication.getAuthScheme(); if (authScope.getHost() != null && authScheme != null) { final HttpHost targetHost = new HttpHost(authScope.getHost(), authScope.getPort()); authCache.put(targetHost, authScheme); } } httpClientContext.setAuthCache(authCache); httpClientContext.setCredentialsProvider(credentialsProvider); // Request Header final RequestHeader[] requestHeaders = getInitParameter(REQUERT_HEADERS_PROPERTY, new RequestHeader[0], RequestHeader[].class); for (final RequestHeader requestHeader : requestHeaders) { if (requestHeader.isValid()) { requestHeaderList.add(new BasicHeader(requestHeader.getName(), requestHeader.getValue())); } } // do not redirect requestConfigBuilder .setRedirectsEnabled(getInitParameter(REDIRECTS_ENABLED, redirectsEnabled, Boolean.class)); // cookie if (cookieSpec != null) { requestConfigBuilder.setCookieSpec(cookieSpec); } // cookie store httpClientBuilder.setDefaultCookieStore(cookieStore); if (cookieStore != null) { final Cookie[] cookies = getInitParameter(COOKIES_PROPERTY, new Cookie[0], Cookie[].class); for (final Cookie cookie : cookies) { cookieStore.addCookie(cookie); } } connectionMonitorTask = TimeoutManager.getInstance().addTimeoutTarget( new HcConnectionMonitorTarget(clientConnectionManager, idleConnectionTimeout), connectionCheckInterval, true); final CloseableHttpClient closeableHttpClient = httpClientBuilder .setConnectionManager(clientConnectionManager).setDefaultRequestConfig(requestConfigBuilder.build()) .build(); if (!httpClientPropertyMap.isEmpty()) { final BeanDesc beanDesc = BeanDescFactory.getBeanDesc(closeableHttpClient.getClass()); for (final Map.Entry<String, Object> entry : httpClientPropertyMap.entrySet()) { final String propertyName = entry.getKey(); if (beanDesc.hasPropertyDesc(propertyName)) { final PropertyDesc propertyDesc = beanDesc.getPropertyDesc(propertyName); propertyDesc.setValue(closeableHttpClient, entry.getValue()); } else { logger.warn("DefaultHttpClient does not have " + propertyName + "."); } } } httpClient = closeableHttpClient; }
From source file:org.seasar.robot.client.http.HcHttpClient.java
public synchronized void init() { if (httpClient != null) { return;// w w w.ja va 2 s. co m } if (logger.isDebugEnabled()) { logger.debug("Initializing " + HcHttpClient.class.getName()); } // access timeout final Integer accessTimeoutParam = getInitParameter(ACCESS_TIMEOUT_PROPERTY, accessTimeout); if (accessTimeoutParam != null) { accessTimeout = accessTimeoutParam; } // robots.txt parser final Boolean robotsTxtEnabled = getInitParameter(ROBOTS_TXT_ENABLED_PROPERTY, Boolean.TRUE); if (robotsTxtHelper != null) { robotsTxtHelper.setEnabled(robotsTxtEnabled.booleanValue()); } // httpclient final org.apache.http.client.config.RequestConfig.Builder requestConfigBuilder = RequestConfig.custom(); final HttpClientBuilder httpClientBuilder = HttpClientBuilder.create(); final Integer connectionTimeoutParam = getInitParameter(CONNECTION_TIMEOUT_PROPERTY, connectionTimeout); if (connectionTimeoutParam != null) { requestConfigBuilder.setConnectTimeout(connectionTimeoutParam); } final Boolean staleCheckingEnabledParam = getInitParameter(STALE_CHECKING_ENABLED_PROPERTY, staleCheckingEnabled); if (staleCheckingEnabledParam != null) { requestConfigBuilder.setStaleConnectionCheckEnabled(staleCheckingEnabledParam); } final Integer soTimeoutParam = getInitParameter(SO_TIMEOUT_PROPERTY, soTimeout); if (soTimeoutParam != null) { requestConfigBuilder.setSocketTimeout(soTimeoutParam); } // AuthSchemeFactory final RegistryBuilder<AuthSchemeProvider> authSchemeProviderBuilder = RegistryBuilder.create(); final Map<String, AuthSchemeProvider> factoryMap = getInitParameter(AUTH_SCHEME_PROVIDERS_PROPERTY, authSchemeProviderMap); if (factoryMap != null) { for (final Map.Entry<String, AuthSchemeProvider> entry : factoryMap.entrySet()) { authSchemeProviderBuilder.register(entry.getKey(), entry.getValue()); } } // user agent userAgent = getInitParameter(USER_AGENT_PROPERTY, userAgent); if (StringUtil.isNotBlank(userAgent)) { httpClientBuilder.setUserAgent(userAgent); } CredentialsProvider credsProvider = null; AuthCache authCache = null; // proxy final String proxyHost = getInitParameter(PROXY_HOST_PROPERTY, this.proxyHost); final Integer proxyPort = getInitParameter(PROXY_PORT_PROPERTY, this.proxyPort); if (proxyHost != null && proxyPort != null) { final HttpHost proxy = new HttpHost(proxyHost, proxyPort); final DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy); httpClientBuilder.setRoutePlanner(routePlanner); final Credentials credentials = getInitParameter(PROXY_CREDENTIALS_PROPERTY, proxyCredentials); if (credentials != null) { authCache = new BasicAuthCache(); credsProvider = new BasicCredentialsProvider(); credsProvider.setCredentials(new AuthScope(proxyHost, proxyPort), credentials); final AuthScheme authScheme = getInitParameter(PROXY_AUTH_SCHEME_PROPERTY, proxyAuthScheme); if (authScheme != null) { authCache.put(proxy, authScheme); } } } // Authentication final Authentication[] siteCredentialList = getInitParameter(BASIC_AUTHENTICATIONS_PROPERTY, new Authentication[0]); if (siteCredentialList != null && siteCredentialList.length > 0 && authCache == null) { authCache = new BasicAuthCache(); credsProvider = new BasicCredentialsProvider(); } for (final Authentication authentication : siteCredentialList) { final AuthScope authScope = authentication.getAuthScope(); credsProvider.setCredentials(authScope, authentication.getCredentials()); final AuthScheme authScheme = authentication.getAuthScheme(); if (authScope.getHost() != null && authScheme != null) { final HttpHost targetHost = new HttpHost(authScope.getHost(), authScope.getPort()); authCache.put(targetHost, authScheme); } } if (authCache != null) { httpClientContext.setAuthCache(authCache); httpClientContext.setCredentialsProvider(credsProvider); } // Request Header final RequestHeader[] requestHeaders = getInitParameter(REQUERT_HEADERS_PROPERTY, new RequestHeader[0]); for (final RequestHeader requestHeader : requestHeaders) { if (requestHeader.isValid()) { requestHeaderList.add(new BasicHeader(requestHeader.getName(), requestHeader.getValue())); } } // do not redirect requestConfigBuilder.setRedirectsEnabled(false); // cookie if (cookieSpec != null) { requestConfigBuilder.setCookieSpec(cookieSpec); } // cookie store httpClientBuilder.setDefaultCookieStore(cookieStore); if (cookieStore != null) { final Cookie[] cookies = getInitParameter(COOKIES_PROPERTY, new Cookie[0]); for (final Cookie cookie : cookies) { cookieStore.addCookie(cookie); } } connectionMonitorTask = TimeoutManager.getInstance().addTimeoutTarget( new HcConnectionMonitorTarget(clientConnectionManager, idleConnectionTimeout), connectionCheckInterval, true); final CloseableHttpClient closeableHttpClient = httpClientBuilder .setConnectionManager(clientConnectionManager).setDefaultRequestConfig(requestConfigBuilder.build()) .build(); if (!httpClientPropertyMap.isEmpty()) { final BeanDesc beanDesc = BeanDescFactory.getBeanDesc(closeableHttpClient.getClass()); for (final Map.Entry<String, Object> entry : httpClientPropertyMap.entrySet()) { final String propertyName = entry.getKey(); if (beanDesc.hasPropertyDesc(propertyName)) { final PropertyDesc propertyDesc = beanDesc.getPropertyDesc(propertyName); propertyDesc.setValue(closeableHttpClient, entry.getValue()); } else { logger.warn("DefaultHttpClient does not have " + propertyName + "."); } } } httpClient = closeableHttpClient; }
From source file:org.apache.ambari.view.hive.client.Connection.java
private CloseableHttpClient getHttpClient(Boolean useSsl) throws SQLException { boolean isCookieEnabled = authParams.get(Utils.HiveAuthenticationParams.COOKIE_AUTH) == null || (!Utils.HiveAuthenticationParams.COOKIE_AUTH_FALSE .equalsIgnoreCase(authParams.get(Utils.HiveAuthenticationParams.COOKIE_AUTH))); String cookieName = authParams.get(Utils.HiveAuthenticationParams.COOKIE_NAME) == null ? Utils.HiveAuthenticationParams.DEFAULT_COOKIE_NAMES_HS2 : authParams.get(Utils.HiveAuthenticationParams.COOKIE_NAME); CookieStore cookieStore = isCookieEnabled ? new BasicCookieStore() : null; HttpClientBuilder httpClientBuilder; // Request interceptor for any request pre-processing logic HttpRequestInterceptor requestInterceptor; Map<String, String> additionalHttpHeaders = new HashMap<String, String>(); // Retrieve the additional HttpHeaders for (Map.Entry<String, String> entry : authParams.entrySet()) { String key = entry.getKey(); if (key.startsWith(Utils.HiveAuthenticationParams.HTTP_HEADER_PREFIX)) { additionalHttpHeaders.put(key.substring(Utils.HiveAuthenticationParams.HTTP_HEADER_PREFIX.length()), entry.getValue());//w ww . j a v a 2 s . c om } } // Configure http client for kerberos/password based authentication if (isKerberosAuthMode()) { /** * Add an interceptor which sets the appropriate header in the request. * It does the kerberos authentication and get the final service ticket, * for sending to the server before every request. * In https mode, the entire information is encrypted */ Boolean assumeSubject = Utils.HiveAuthenticationParams.AUTH_KERBEROS_AUTH_TYPE_FROM_SUBJECT .equals(authParams.get(Utils.HiveAuthenticationParams.AUTH_KERBEROS_AUTH_TYPE)); requestInterceptor = new HttpKerberosRequestInterceptor( authParams.get(Utils.HiveAuthenticationParams.AUTH_PRINCIPAL), host, getServerHttpUrl(useSsl), assumeSubject, cookieStore, cookieName, useSsl, additionalHttpHeaders); } else { /** * Add an interceptor to pass username/password in the header. * In https mode, the entire information is encrypted */ requestInterceptor = new HttpBasicAuthInterceptor( getAuthParamDefault(Utils.HiveAuthenticationParams.AUTH_USER, getUsername()), getPassword(), cookieStore, cookieName, useSsl, additionalHttpHeaders); } // Configure http client for cookie based authentication if (isCookieEnabled) { // Create a http client with a retry mechanism when the server returns a status code of 401. httpClientBuilder = HttpClients.custom() .setServiceUnavailableRetryStrategy(new ServiceUnavailableRetryStrategy() { @Override public boolean retryRequest(final HttpResponse response, final int executionCount, final HttpContext context) { int statusCode = response.getStatusLine().getStatusCode(); boolean ret = statusCode == 401 && executionCount <= 1; // Set the context attribute to true which will be interpreted by the request interceptor if (ret) { context.setAttribute(Utils.HIVE_SERVER2_RETRY_KEY, Utils.HIVE_SERVER2_RETRY_TRUE); } return ret; } @Override public long getRetryInterval() { // Immediate retry return 0; } }); } else { httpClientBuilder = HttpClientBuilder.create(); } // Add the request interceptor to the client builder httpClientBuilder.addInterceptorFirst(requestInterceptor); // Configure http client for SSL if (useSsl) { String useTwoWaySSL = authParams.get(Utils.HiveAuthenticationParams.USE_TWO_WAY_SSL); String sslTrustStorePath = authParams.get(Utils.HiveAuthenticationParams.SSL_TRUST_STORE); String sslTrustStorePassword = authParams.get(Utils.HiveAuthenticationParams.SSL_TRUST_STORE_PASSWORD); KeyStore sslTrustStore; SSLSocketFactory socketFactory; /** * The code within the try block throws: * 1. SSLInitializationException * 2. KeyStoreException * 3. IOException * 4. NoSuchAlgorithmException * 5. CertificateException * 6. KeyManagementException * 7. UnrecoverableKeyException * We don't want the client to retry on any of these, hence we catch all * and throw a SQLException. */ try { if (useTwoWaySSL != null && useTwoWaySSL.equalsIgnoreCase(Utils.HiveAuthenticationParams.TRUE)) { socketFactory = getTwoWaySSLSocketFactory(); } else if (sslTrustStorePath == null || sslTrustStorePath.isEmpty()) { // Create a default socket factory based on standard JSSE trust material socketFactory = SSLSocketFactory.getSocketFactory(); } else { // Pick trust store config from the given path sslTrustStore = KeyStore.getInstance(Utils.HiveAuthenticationParams.SSL_TRUST_STORE_TYPE); try (FileInputStream fis = new FileInputStream(sslTrustStorePath)) { sslTrustStore.load(fis, sslTrustStorePassword.toCharArray()); } socketFactory = new SSLSocketFactory(sslTrustStore); } socketFactory.setHostnameVerifier(SSLSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER); final Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create() .register("https", socketFactory).build(); httpClientBuilder.setConnectionManager(new BasicHttpClientConnectionManager(registry)); } catch (Exception e) { String msg = "Could not create an https connection to " + getServerHttpUrl(useSsl) + ". " + e.getMessage(); throw new SQLException(msg, " 08S01", e); } } return httpClientBuilder.build(); }
From source file:crawlercommons.fetcher.http.SimpleHttpFetcher.java
private void init() { if (_httpClient == null) { synchronized (SimpleHttpFetcher.class) { if (_httpClient != null) return; final HttpClientBuilder httpClientBuilder = HttpClientBuilder.create(); final RequestConfig.Builder requestConfigBuilder = RequestConfig.custom(); // Set the socket and connection timeout to be something // reasonable. requestConfigBuilder.setSocketTimeout(_socketTimeout); requestConfigBuilder.setConnectTimeout(_connectionTimeout); requestConfigBuilder.setConnectionRequestTimeout(_connectionRequestTimeout); /*//from ww w .j av a 2 s . co m * CoreConnectionPNames.TCP_NODELAY='http.tcp.nodelay': * determines whether Nagle's algorithm is to be used. Nagle's * algorithm tries to conserve bandwidth by minimizing the * number of segments that are sent. When applications wish to * decrease network latency and increase performance, they can * disable Nagle's algorithm (that is enable TCP_NODELAY. Data * will be sent earlier, at the cost of an increase in bandwidth * consumption. This parameter expects a value of type * java.lang.Boolean. If this parameter is not set, TCP_NODELAY * will be enabled (no delay). */ // FIXME Could not find this parameter in http-client version // 4.5 // HttpConnectionParams.setTcpNoDelay(params, true); // HttpProtocolParams.setVersion(params, _httpVersion); httpClientBuilder.setUserAgent(_userAgent.getUserAgentString()); // HttpProtocolParams.setContentCharset(params, "UTF-8"); // HttpProtocolParams.setHttpElementCharset(params, "UTF-8"); /* * CoreProtocolPNames.USE_EXPECT_CONTINUE= * 'http.protocol.expect-continue': activates the Expect: * 100-Continue handshake for the entity enclosing methods. The * purpose of the Expect: 100-Continue handshake is to allow the * client that is sending a request message with a request body * to determine if the origin server is willing to accept the * request (based on the request headers) before the client * sends the request body. The use of the Expect: 100-continue * handshake can result in a noticeable performance improvement * for entity enclosing requests (such as POST and PUT) that * require the target server's authentication. The Expect: * 100-continue handshake should be used with caution, as it may * cause problems with HTTP servers and proxies that do not * support HTTP/1.1 protocol. This parameter expects a value of * type java.lang.Boolean. If this parameter is not set, * HttpClient will not attempt to use the handshake. */ requestConfigBuilder.setExpectContinueEnabled(true); /* * CoreProtocolPNames.WAIT_FOR_CONTINUE= * 'http.protocol.wait-for-continue': defines the maximum period * of time in milliseconds the client should spend waiting for a * 100-continue response. This parameter expects a value of type * java.lang.Integer. If this parameter is not set HttpClient * will wait 3 seconds for a confirmation before resuming the * transmission of the request body. */ // FIXME Could not find this parameter in http-client version // 4.5 // params.setIntParameter(CoreProtocolPNames.WAIT_FOR_CONTINUE, // 5000); // FIXME Could not find this parameter in http-client version // 4.5 // CookieSpecParamBean cookieParams = new // CookieSpecParamBean(params); // cookieParams.setSingleHeader(false); // Create and initialize connection socket factory registry RegistryBuilder<ConnectionSocketFactory> registry = RegistryBuilder .<ConnectionSocketFactory>create(); registry.register("http", PlainConnectionSocketFactory.getSocketFactory()); SSLConnectionSocketFactory sf = createSSLConnectionSocketFactory(); if (sf != null) { registry.register("https", sf); } else { LOGGER.warn("No valid SSLContext found for https"); } _connectionManager = new PoolingHttpClientConnectionManager(registry.build()); _connectionManager.setMaxTotal(_maxThreads); _connectionManager.setDefaultMaxPerRoute(getMaxConnectionsPerHost()); /* * CoreConnectionPNames.STALE_CONNECTION_CHECK= * 'http.connection.stalecheck': determines whether stale * connection check is to be used. Disabling stale connection * check may result in a noticeable performance improvement (the * check can cause up to 30 millisecond overhead per request) at * the risk of getting an I/O error when executing a request * over a connection that has been closed at the server side. * This parameter expects a value of type java.lang.Boolean. For * performance critical operations the check should be disabled. * If this parameter is not set, the stale connection check will * be performed before each request execution. * * We don't need I/O exceptions in case if Server doesn't * support Kee-Alive option; our client by default always tries * keep-alive. */ // Even with stale checking enabled, a connection can "go stale" // between the check and the next request. So we still need to // handle the case of a closed socket (from the server side), // and disabling this check improves performance. // Stale connections will be checked in a separate monitor // thread _connectionManager.setValidateAfterInactivity(-1); httpClientBuilder.setConnectionManager(_connectionManager); httpClientBuilder.setRetryHandler(new MyRequestRetryHandler(_maxRetryCount)); httpClientBuilder.setRedirectStrategy(new MyRedirectStrategy(getRedirectMode())); httpClientBuilder.setRequestExecutor(new MyHttpRequestExecutor()); // FUTURE KKr - support authentication // FIXME Could not find this parameter in http-client version // 4.5 // HttpClientParams.setAuthenticating(params, false); requestConfigBuilder.setCookieSpec(CookieSpecs.DEFAULT); if (getMaxRedirects() == 0) { requestConfigBuilder.setRedirectsEnabled(false); } else { requestConfigBuilder.setRedirectsEnabled(true); requestConfigBuilder.setMaxRedirects(getMaxRedirects()); } // Set up default headers. This helps us get back from servers // what we want. HashSet<Header> defaultHeaders = new HashSet<Header>(); defaultHeaders.add(new BasicHeader(HttpHeaders.ACCEPT_LANGUAGE, getAcceptLanguage())); defaultHeaders.add(new BasicHeader(HttpHeaders.ACCEPT_CHARSET, DEFAULT_ACCEPT_CHARSET)); defaultHeaders.add(new BasicHeader(HttpHeaders.ACCEPT_ENCODING, DEFAULT_ACCEPT_ENCODING)); defaultHeaders.add(new BasicHeader(HttpHeaders.ACCEPT, DEFAULT_ACCEPT)); httpClientBuilder.setDefaultHeaders(defaultHeaders); httpClientBuilder.setKeepAliveStrategy(new MyConnectionKeepAliveStrategy()); monitor = new IdleConnectionMonitorThread(_connectionManager); monitor.start(); httpClientBuilder.setDefaultRequestConfig(requestConfigBuilder.build()); _httpClient = httpClientBuilder.build(); } } }
From source file:org.apache.hive.jdbc.HiveConnection.java
private CloseableHttpClient getHttpClient(Boolean useSsl) throws SQLException { boolean isCookieEnabled = sessConfMap.get(JdbcConnectionParams.COOKIE_AUTH) == null || (!JdbcConnectionParams.COOKIE_AUTH_FALSE .equalsIgnoreCase(sessConfMap.get(JdbcConnectionParams.COOKIE_AUTH))); String cookieName = sessConfMap.get(JdbcConnectionParams.COOKIE_NAME) == null ? JdbcConnectionParams.DEFAULT_COOKIE_NAMES_HS2 : sessConfMap.get(JdbcConnectionParams.COOKIE_NAME); CookieStore cookieStore = isCookieEnabled ? new BasicCookieStore() : null; HttpClientBuilder httpClientBuilder; // Request interceptor for any request pre-processing logic HttpRequestInterceptor requestInterceptor; Map<String, String> additionalHttpHeaders = new HashMap<String, String>(); // Retrieve the additional HttpHeaders for (Map.Entry<String, String> entry : sessConfMap.entrySet()) { String key = entry.getKey(); if (key.startsWith(JdbcConnectionParams.HTTP_HEADER_PREFIX)) { additionalHttpHeaders.put(key.substring(JdbcConnectionParams.HTTP_HEADER_PREFIX.length()), entry.getValue());/* ww w .j a v a 2s. co m*/ } } // Configure http client for kerberos/password based authentication if (isKerberosAuthMode()) { /** * Add an interceptor which sets the appropriate header in the request. * It does the kerberos authentication and get the final service ticket, * for sending to the server before every request. * In https mode, the entire information is encrypted */ requestInterceptor = new HttpKerberosRequestInterceptor( sessConfMap.get(JdbcConnectionParams.AUTH_PRINCIPAL), host, getServerHttpUrl(useSsl), assumeSubject, cookieStore, cookieName, useSsl, additionalHttpHeaders); } else { // Check for delegation token, if present add it in the header String tokenStr = getClientDelegationToken(sessConfMap); if (tokenStr != null) { requestInterceptor = new HttpTokenAuthInterceptor(tokenStr, cookieStore, cookieName, useSsl, additionalHttpHeaders); } else { /** * Add an interceptor to pass username/password in the header. * In https mode, the entire information is encrypted */ requestInterceptor = new HttpBasicAuthInterceptor(getUserName(), getPassword(), cookieStore, cookieName, useSsl, additionalHttpHeaders); } } // Configure http client for cookie based authentication if (isCookieEnabled) { // Create a http client with a retry mechanism when the server returns a status code of 401. httpClientBuilder = HttpClients.custom() .setServiceUnavailableRetryStrategy(new ServiceUnavailableRetryStrategy() { @Override public boolean retryRequest(final HttpResponse response, final int executionCount, final HttpContext context) { int statusCode = response.getStatusLine().getStatusCode(); boolean ret = statusCode == 401 && executionCount <= 1; // Set the context attribute to true which will be interpreted by the request // interceptor if (ret) { context.setAttribute(Utils.HIVE_SERVER2_RETRY_KEY, Utils.HIVE_SERVER2_RETRY_TRUE); } return ret; } @Override public long getRetryInterval() { // Immediate retry return 0; } }); } else { httpClientBuilder = HttpClientBuilder.create(); } // In case the server's idletimeout is set to a lower value, it might close it's side of // connection. However we retry one more time on NoHttpResponseException httpClientBuilder.setRetryHandler(new HttpRequestRetryHandler() { @Override public boolean retryRequest(IOException exception, int executionCount, HttpContext context) { if (executionCount > 1) { LOG.info("Retry attempts to connect to server exceeded."); return false; } if (exception instanceof org.apache.http.NoHttpResponseException) { LOG.info("Could not connect to the server. Retrying one more time."); return true; } return false; } }); // Add the request interceptor to the client builder httpClientBuilder.addInterceptorFirst(requestInterceptor); // Add an interceptor to add in an XSRF header httpClientBuilder.addInterceptorLast(new XsrfHttpRequestInterceptor()); // Configure http client for SSL if (useSsl) { String useTwoWaySSL = sessConfMap.get(JdbcConnectionParams.USE_TWO_WAY_SSL); String sslTrustStorePath = sessConfMap.get(JdbcConnectionParams.SSL_TRUST_STORE); String sslTrustStorePassword = sessConfMap.get(JdbcConnectionParams.SSL_TRUST_STORE_PASSWORD); KeyStore sslTrustStore; SSLConnectionSocketFactory socketFactory; SSLContext sslContext; /** * The code within the try block throws: SSLInitializationException, KeyStoreException, * IOException, NoSuchAlgorithmException, CertificateException, KeyManagementException & * UnrecoverableKeyException. We don't want the client to retry on any of these, * hence we catch all and throw a SQLException. */ try { if (useTwoWaySSL != null && useTwoWaySSL.equalsIgnoreCase(JdbcConnectionParams.TRUE)) { socketFactory = getTwoWaySSLSocketFactory(); } else if (sslTrustStorePath == null || sslTrustStorePath.isEmpty()) { // Create a default socket factory based on standard JSSE trust material socketFactory = SSLConnectionSocketFactory.getSocketFactory(); } else { // Pick trust store config from the given path sslTrustStore = KeyStore.getInstance(JdbcConnectionParams.SSL_TRUST_STORE_TYPE); try (FileInputStream fis = new FileInputStream(sslTrustStorePath)) { sslTrustStore.load(fis, sslTrustStorePassword.toCharArray()); } sslContext = SSLContexts.custom().loadTrustMaterial(sslTrustStore, null).build(); socketFactory = new SSLConnectionSocketFactory(sslContext, new DefaultHostnameVerifier(null)); } final Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create() .register("https", socketFactory).build(); httpClientBuilder.setConnectionManager(new BasicHttpClientConnectionManager(registry)); } catch (Exception e) { String msg = "Could not create an https connection to " + jdbcUriString + ". " + e.getMessage(); throw new SQLException(msg, " 08S01", e); } } return httpClientBuilder.build(); }
From source file:org.apache.marmotta.client.util.HTTPUtil.java
public static HttpClient createClient(ClientConfiguration config, String context) { final HttpClientBuilder httpClientBuilder = HttpClients.custom(); httpClientBuilder.setUserAgent("Marmotta Client Library/" + MetaUtil.getVersion()); httpClientBuilder.setRedirectStrategy(new MarmottaRedirectStrategy()); httpClientBuilder.setRetryHandler(new MarmottaHttpRequestRetryHandler()); final RequestConfig.Builder requestConfigBuilder = RequestConfig.custom(); requestConfigBuilder.setSocketTimeout(config.getSoTimeout()); requestConfigBuilder.setConnectTimeout(config.getConnectionTimeout()); requestConfigBuilder.setRedirectsEnabled(true); requestConfigBuilder.setMaxRedirects(3); httpClientBuilder.setDefaultRequestConfig(requestConfigBuilder.build()); if (config.getConectionManager() != null) { httpClientBuilder.setConnectionManager(config.getConectionManager()); } else {/* ww w . j a v a 2s . co m*/ final Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create() .register("http", PlainConnectionSocketFactory.getSocketFactory()) //.register("https", ) .build(); final PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(registry); cm.setMaxTotal(100); httpClientBuilder.setConnectionManager(cm); } return httpClientBuilder.build(); }
From source file:org.apache.nifi.processors.standard.GetHTTP.java
@Override public void onTrigger(final ProcessContext context, final ProcessSessionFactory sessionFactory) throws ProcessException { final ComponentLog logger = getLogger(); final ProcessSession session = sessionFactory.createSession(); final FlowFile incomingFlowFile = session.get(); if (incomingFlowFile != null) { session.transfer(incomingFlowFile, REL_SUCCESS); logger.warn("found FlowFile {} in input queue; transferring to success", new Object[] { incomingFlowFile }); }// w w w . j av a 2 s . co m // get the URL final String url = context.getProperty(URL).evaluateAttributeExpressions().getValue(); final URI uri; String source = url; try { uri = new URI(url); source = uri.getHost(); } catch (final URISyntaxException swallow) { // this won't happen as the url has already been validated } // get the ssl context service final SSLContextService sslContextService = context.getProperty(SSL_CONTEXT_SERVICE) .asControllerService(SSLContextService.class); // create the connection manager final HttpClientConnectionManager conMan; if (sslContextService == null) { conMan = new BasicHttpClientConnectionManager(); } else { final SSLContext sslContext; try { sslContext = createSSLContext(sslContextService); } catch (final Exception e) { throw new ProcessException(e); } final SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext); // Also include a plain socket factory for regular http connections (especially proxies) final Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder .<ConnectionSocketFactory>create().register("https", sslsf) .register("http", PlainConnectionSocketFactory.getSocketFactory()).build(); conMan = new BasicHttpClientConnectionManager(socketFactoryRegistry); } try { // build the request configuration final RequestConfig.Builder requestConfigBuilder = RequestConfig.custom(); requestConfigBuilder.setConnectionRequestTimeout( context.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue()); requestConfigBuilder.setConnectTimeout( context.getProperty(CONNECTION_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue()); requestConfigBuilder.setSocketTimeout( context.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue()); requestConfigBuilder.setRedirectsEnabled(context.getProperty(FOLLOW_REDIRECTS).asBoolean()); switch (context.getProperty(REDIRECT_COOKIE_POLICY).getValue()) { case STANDARD_COOKIE_POLICY_STR: requestConfigBuilder.setCookieSpec(CookieSpecs.STANDARD); break; case STRICT_COOKIE_POLICY_STR: requestConfigBuilder.setCookieSpec(CookieSpecs.STANDARD_STRICT); break; case NETSCAPE_COOKIE_POLICY_STR: requestConfigBuilder.setCookieSpec(CookieSpecs.NETSCAPE); break; case IGNORE_COOKIE_POLICY_STR: requestConfigBuilder.setCookieSpec(CookieSpecs.IGNORE_COOKIES); break; case DEFAULT_COOKIE_POLICY_STR: default: requestConfigBuilder.setCookieSpec(CookieSpecs.DEFAULT); } // build the http client final HttpClientBuilder clientBuilder = HttpClientBuilder.create(); clientBuilder.setConnectionManager(conMan); // include the user agent final String userAgent = context.getProperty(USER_AGENT).getValue(); if (userAgent != null) { clientBuilder.setUserAgent(userAgent); } // set the ssl context if necessary if (sslContextService != null) { clientBuilder.setSslcontext(sslContextService.createSSLContext(ClientAuth.REQUIRED)); } final String username = context.getProperty(USERNAME).getValue(); final String password = context.getProperty(PASSWORD).getValue(); // set the credentials if appropriate if (username != null) { final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); if (password == null) { credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username)); } else { credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password)); } clientBuilder.setDefaultCredentialsProvider(credentialsProvider); } // Set the proxy if specified if (context.getProperty(PROXY_HOST).isSet() && context.getProperty(PROXY_PORT).isSet()) { final String host = context.getProperty(PROXY_HOST).getValue(); final int port = context.getProperty(PROXY_PORT).asInteger(); clientBuilder.setProxy(new HttpHost(host, port)); } // create request final HttpGet get = new HttpGet(url); get.setConfig(requestConfigBuilder.build()); final StateMap beforeStateMap; try { beforeStateMap = context.getStateManager().getState(Scope.LOCAL); final String lastModified = beforeStateMap.get(LAST_MODIFIED + ":" + url); if (lastModified != null) { get.addHeader(HEADER_IF_MODIFIED_SINCE, parseStateValue(lastModified).getValue()); } final String etag = beforeStateMap.get(ETAG + ":" + url); if (etag != null) { get.addHeader(HEADER_IF_NONE_MATCH, parseStateValue(etag).getValue()); } } catch (final IOException ioe) { throw new ProcessException(ioe); } final String accept = context.getProperty(ACCEPT_CONTENT_TYPE).getValue(); if (accept != null) { get.addHeader(HEADER_ACCEPT, accept); } // Add dynamic headers PropertyValue customHeaderValue; for (PropertyDescriptor customProperty : customHeaders) { customHeaderValue = context.getProperty(customProperty).evaluateAttributeExpressions(); if (StringUtils.isNotBlank(customHeaderValue.getValue())) { get.addHeader(customProperty.getName(), customHeaderValue.getValue()); } } // create the http client try (final CloseableHttpClient client = clientBuilder.build()) { // NOTE: including this inner try in order to swallow exceptions on close try { final StopWatch stopWatch = new StopWatch(true); final HttpResponse response = client.execute(get); final int statusCode = response.getStatusLine().getStatusCode(); if (statusCode == NOT_MODIFIED) { logger.info( "content not retrieved because server returned HTTP Status Code {}: Not Modified", new Object[] { NOT_MODIFIED }); context.yield(); // doing a commit in case there were flow files in the input queue session.commit(); return; } final String statusExplanation = response.getStatusLine().getReasonPhrase(); if ((statusCode >= 300) || (statusCode == 204)) { logger.error("received status code {}:{} from {}", new Object[] { statusCode, statusExplanation, url }); // doing a commit in case there were flow files in the input queue session.commit(); return; } FlowFile flowFile = session.create(); flowFile = session.putAttribute(flowFile, CoreAttributes.FILENAME.key(), context.getProperty(FILENAME).evaluateAttributeExpressions().getValue()); flowFile = session.putAttribute(flowFile, this.getClass().getSimpleName().toLowerCase() + ".remote.source", source); flowFile = session.importFrom(response.getEntity().getContent(), flowFile); final Header contentTypeHeader = response.getFirstHeader("Content-Type"); if (contentTypeHeader != null) { final String contentType = contentTypeHeader.getValue(); if (!contentType.trim().isEmpty()) { flowFile = session.putAttribute(flowFile, CoreAttributes.MIME_TYPE.key(), contentType.trim()); } } final long flowFileSize = flowFile.getSize(); stopWatch.stop(); final String dataRate = stopWatch.calculateDataRate(flowFileSize); session.getProvenanceReporter().receive(flowFile, url, stopWatch.getDuration(TimeUnit.MILLISECONDS)); session.transfer(flowFile, REL_SUCCESS); logger.info("Successfully received {} from {} at a rate of {}; transferred to success", new Object[] { flowFile, url, dataRate }); session.commit(); updateStateMap(context, response, beforeStateMap, url); } catch (final IOException e) { context.yield(); session.rollback(); logger.error("Failed to retrieve file from {} due to {}; rolling back session", new Object[] { url, e.getMessage() }, e); throw new ProcessException(e); } catch (final Throwable t) { context.yield(); session.rollback(); logger.error("Failed to process due to {}; rolling back session", new Object[] { t.getMessage() }, t); throw t; } } catch (final IOException e) { logger.debug("Error closing client due to {}, continuing.", new Object[] { e.getMessage() }); } } finally { conMan.shutdown(); } }