List of usage examples for org.apache.http.impl.client HttpClientBuilder setProxy
public final HttpClientBuilder setProxy(final HttpHost proxy)
From source file:com.stormpath.sdk.impl.http.httpclient.HttpClientRequestExecutor.java
/** * Creates a new {@code HttpClientRequestExecutor} using the specified {@code ApiKey} and optional {@code Proxy} * configuration.//from w w w. j ava2 s. c o m * @param clientCredentials the Stormpath account API Key that will be used to authenticate the client with Stormpath's API sever * @param proxy the HTTP proxy to be used when communicating with the Stormpath API server (can be null) * @param authenticationScheme the HTTP authentication scheme to be used when communicating with the Stormpath API server. * If null, then Sauthc1 will be used. */ public HttpClientRequestExecutor(ClientCredentials clientCredentials, Proxy proxy, AuthenticationScheme authenticationScheme, RequestAuthenticatorFactory requestAuthenticatorFactory, Integer connectionTimeout) { Assert.notNull(clientCredentials, "clientCredentials argument is required."); Assert.isTrue(connectionTimeout >= 0, "Timeout cannot be a negative number."); RequestAuthenticatorFactory factory = (requestAuthenticatorFactory != null) ? requestAuthenticatorFactory : new DefaultRequestAuthenticatorFactory(); this.requestAuthenticator = factory.create(authenticationScheme, clientCredentials); PoolingHttpClientConnectionManager connMgr = new PoolingHttpClientConnectionManager(); if (MAX_CONNECTIONS_TOTAL >= MAX_CONNECTIONS_PER_ROUTE) { connMgr.setDefaultMaxPerRoute(MAX_CONNECTIONS_PER_ROUTE); connMgr.setMaxTotal(MAX_CONNECTIONS_TOTAL); } else { connMgr.setDefaultMaxPerRoute(DEFAULT_MAX_CONNECTIONS_PER_ROUTE); connMgr.setMaxTotal(DEFAULT_MAX_CONNECTIONS_TOTAL); log.warn( "{} ({}) is less than {} ({}). " + "Reverting to defaults: connectionMaxTotal ({}) and connectionMaxPerRoute ({}).", MAX_CONNECTIONS_TOTAL_PROPERTY_KEY, MAX_CONNECTIONS_TOTAL, MAX_CONNECTIONS_PER_ROUTE_PROPERTY_KEY, MAX_CONNECTIONS_PER_ROUTE, DEFAULT_MAX_CONNECTIONS_TOTAL, DEFAULT_MAX_CONNECTIONS_PER_ROUTE); } // The connectionTimeout value is specified in seconds in Stormpath configuration settings. // Therefore, multiply it by 1000 to be milliseconds since RequestConfig expects milliseconds. int connectionTimeoutAsMilliseconds = connectionTimeout * 1000; RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(connectionTimeoutAsMilliseconds) .setSocketTimeout(connectionTimeoutAsMilliseconds).setRedirectsEnabled(false).build(); ConnectionConfig connectionConfig = ConnectionConfig.custom().setCharset(Consts.UTF_8).build(); HttpClientBuilder httpClientBuilder = HttpClientBuilder.create().setDefaultRequestConfig(requestConfig) .disableCookieManagement().setDefaultConnectionConfig(connectionConfig) .setConnectionManager(connMgr); this.httpClientRequestFactory = new HttpClientRequestFactory(requestConfig); if (proxy != null) { //We have some proxy setting to use! HttpHost httpProxyHost = new HttpHost(proxy.getHost(), proxy.getPort()); httpClientBuilder.setProxy(httpProxyHost); if (proxy.isAuthenticationRequired()) { AuthScope authScope = new AuthScope(proxy.getHost(), proxy.getPort()); Credentials credentials = new UsernamePasswordCredentials(proxy.getUsername(), proxy.getPassword()); CredentialsProvider credentialsProviderProvider = new BasicCredentialsProvider(); credentialsProviderProvider.setCredentials(authScope, credentials); httpClientBuilder.setDefaultCredentialsProvider(credentialsProviderProvider); } } this.httpClient = httpClientBuilder.build(); }
From source file:com.joyent.manta.http.MantaConnectionFactory.java
/** * Configures the builder class with all of the settings needed to connect to * Manta./*from w w w. ja v a 2s . c om*/ * * @param metricConfig nullable configuration for client metrics tracking * @return configured instance */ protected HttpClientBuilder createStandardBuilder(final MantaClientMetricConfiguration metricConfig) { final int maxConns = ObjectUtils.firstNonNull(config.getMaximumConnections(), DefaultsConfigContext.DEFAULT_MAX_CONNS); final int timeout = ObjectUtils.firstNonNull(config.getTimeout(), DefaultsConfigContext.DEFAULT_HTTP_TIMEOUT); final int connectionRequestTimeout = ObjectUtils.firstNonNull(config.getConnectionRequestTimeout(), DefaultsConfigContext.DEFAULT_CONNECTION_REQUEST_TIMEOUT); final Integer expectContinueTimeout = config.getExpectContinueTimeout(); final boolean expectContinueEnabled = expectContinueTimeout != null; final RequestConfig requestConfig = RequestConfig.custom().setAuthenticationEnabled(false) .setSocketTimeout(timeout).setConnectionRequestTimeout(connectionRequestTimeout) .setContentCompressionEnabled(true).setExpectContinueEnabled(expectContinueEnabled).build(); final MantaHttpRequestExecutor requestExecutor = MantaHttpRequestExecutor.Builder.create() .setMetricConfiguration(metricConfig).setWaitForContinue(expectContinueTimeout).build(); final HttpClientBuilder builder = HttpClients.custom().disableAuthCaching().disableCookieManagement() .setUserAgent(USER_AGENT).setConnectionReuseStrategy(new DefaultConnectionReuseStrategy()) .setMaxConnTotal(maxConns).setKeepAliveStrategy(new DefaultConnectionKeepAliveStrategy()) .setDefaultRequestConfig(requestConfig).setConnectionManagerShared(false) .setRequestExecutor(requestExecutor).setConnectionBackoffStrategy(new DefaultBackoffStrategy()); final HttpHost proxyHost = findProxyServer(); if (proxyHost != null) { builder.setProxy(proxyHost); } return builder; }
From source file:de.vanita5.twittnuker.util.net.TwidereHttpClientImpl.java
public TwidereHttpClientImpl(final Context context, final HttpClientConfiguration conf) { this.conf = conf; final HttpClientBuilder clientBuilder = HttpClients.custom(); final LayeredConnectionSocketFactory factory = TwidereSSLSocketFactory.getSocketFactory(context, conf.isSSLErrorIgnored());//from w w w . j av a2 s . c o m clientBuilder.setSSLSocketFactory(factory); final RequestConfig.Builder requestConfigBuilder = RequestConfig.custom(); requestConfigBuilder.setConnectionRequestTimeout(conf.getHttpConnectionTimeout()); requestConfigBuilder.setConnectTimeout(conf.getHttpConnectionTimeout()); requestConfigBuilder.setSocketTimeout(conf.getHttpReadTimeout()); requestConfigBuilder.setRedirectsEnabled(false); clientBuilder.setDefaultRequestConfig(requestConfigBuilder.build()); if (conf.isProxyConfigured()) { final HttpHost proxy = new HttpHost(conf.getHttpProxyHost(), conf.getHttpProxyPort()); clientBuilder.setProxy(proxy); if (!TextUtils.isEmpty(conf.getHttpProxyUser())) { if (logger.isDebugEnabled()) { logger.debug("Proxy AuthUser: " + conf.getHttpProxyUser()); logger.debug( "Proxy AuthPassword: " + InternalStringUtil.maskString(conf.getHttpProxyPassword())); } final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); credentialsProvider.setCredentials(new AuthScope(conf.getHttpProxyHost(), conf.getHttpProxyPort()), new UsernamePasswordCredentials(conf.getHttpProxyUser(), conf.getHttpProxyPassword())); clientBuilder.setDefaultCredentialsProvider(credentialsProvider); } } client = clientBuilder.build(); }
From source file:io.github.cidisk.indexcrawler.fetcher.PageFetcher.java
public PageFetcher(CrawlConfig config) { super(config); RequestConfig requestConfig = RequestConfig.custom().setExpectContinueEnabled(false) .setCookieSpec(CookieSpecs.BROWSER_COMPATIBILITY).setRedirectsEnabled(false) .setSocketTimeout(config.getSocketTimeout()).setConnectTimeout(config.getConnectionTimeout()) .build();//from ww w .j a v a 2 s . com RegistryBuilder<ConnectionSocketFactory> connRegistryBuilder = RegistryBuilder.create(); connRegistryBuilder.register("http", PlainConnectionSocketFactory.INSTANCE); if (config.isIncludeHttpsPages()) { try { // Fixing: https://code.google.com/p/crawler4j/issues/detail?id=174 // By always trusting the ssl certificate SSLContext sslContext = SSLContexts.custom().loadTrustMaterial(null, new TrustStrategy() { @Override public boolean isTrusted(final X509Certificate[] chain, String authType) { return true; } }).build(); SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER); connRegistryBuilder.register("https", sslsf); } catch (Exception e) { logger.warn("Exception thrown while trying to register https"); logger.debug("Stacktrace", e); } } Registry<ConnectionSocketFactory> connRegistry = connRegistryBuilder.build(); connectionManager = new PoolingHttpClientConnectionManager(connRegistry); connectionManager.setMaxTotal(config.getMaxTotalConnections()); connectionManager.setDefaultMaxPerRoute(config.getMaxConnectionsPerHost()); HttpClientBuilder clientBuilder = HttpClientBuilder.create(); clientBuilder.setDefaultRequestConfig(requestConfig); clientBuilder.setConnectionManager(connectionManager); clientBuilder.setUserAgent(config.getUserAgentString()); if (config.getProxyHost() != null) { if (config.getProxyUsername() != null) { BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider(); credentialsProvider.setCredentials(new AuthScope(config.getProxyHost(), config.getProxyPort()), new UsernamePasswordCredentials(config.getProxyUsername(), config.getProxyPassword())); clientBuilder.setDefaultCredentialsProvider(credentialsProvider); } HttpHost proxy = new HttpHost(config.getProxyHost(), config.getProxyPort()); clientBuilder.setProxy(proxy); logger.debug("Working through Proxy: {}", proxy.getHostName()); } httpClient = clientBuilder.build(); if (config.getAuthInfos() != null && !config.getAuthInfos().isEmpty()) { doAuthetication(config.getAuthInfos()); } if (connectionMonitorThread == null) { connectionMonitorThread = new IdleConnectionMonitorThread(connectionManager); } connectionMonitorThread.start(); }
From source file:com.crawler.app.fetcher.PageFetcher.java
public PageFetcher(CrawlConfig config) { super(config); RequestConfig requestConfig = RequestConfig.custom().setExpectContinueEnabled(false) .setCookieSpec(CookieSpecs.BROWSER_COMPATIBILITY).setRedirectsEnabled(false) //.setRelativeRedirectsAllowed(true) .setSocketTimeout(config.getSocketTimeout()).setConnectTimeout(config.getConnectionTimeout()) .build();// w ww . java 2 s .c o m RegistryBuilder<ConnectionSocketFactory> connRegistryBuilder = RegistryBuilder.create(); connRegistryBuilder.register("http", PlainConnectionSocketFactory.INSTANCE); if (config.isIncludeHttpsPages()) { try { // Fixing: https://code.google.com/p/crawler4j/issues/detail?id=174 // By always trusting the ssl certificate SSLContext sslContext = SSLContexts.custom().loadTrustMaterial(null, new TrustStrategy() { //@Override public boolean isTrusted(final X509Certificate[] chain, String authType) { return true; } }).build(); SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER); connRegistryBuilder.register("https", sslsf); } catch (Exception e) { logger.warn("Exception thrown while trying to register https"); logger.debug("Stacktrace", e); } } Registry<ConnectionSocketFactory> connRegistry = connRegistryBuilder.build(); connectionManager = new PoolingHttpClientConnectionManager(connRegistry); connectionManager.setMaxTotal(config.getMaxTotalConnections()); connectionManager.setDefaultMaxPerRoute(config.getMaxConnectionsPerHost()); HttpClientBuilder clientBuilder = HttpClientBuilder.create(); clientBuilder.setDefaultRequestConfig(requestConfig); clientBuilder.setConnectionManager(connectionManager); clientBuilder.setUserAgent(config.getUserAgentString()); if (config.getProxyHost() != null) { if (config.getProxyUsername() != null) { BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider(); credentialsProvider.setCredentials(new AuthScope(config.getProxyHost(), config.getProxyPort()), new UsernamePasswordCredentials(config.getProxyUsername(), config.getProxyPassword())); clientBuilder.setDefaultCredentialsProvider(credentialsProvider); } HttpHost proxy = new HttpHost(config.getProxyHost(), config.getProxyPort()); clientBuilder.setProxy(proxy); logger.debug("Working through Proxy: {}", proxy.getHostName()); } httpClient = clientBuilder.build(); if (config.getAuthInfos() != null && !config.getAuthInfos().isEmpty()) { doAuthetication(config.getAuthInfos()); } if (connectionMonitorThread == null) { connectionMonitorThread = new IdleConnectionMonitorThread(connectionManager); } connectionMonitorThread.start(); }
From source file:info.bonjean.beluga.connection.BelugaHTTPClient.java
private BelugaHTTPClient() { BelugaConfiguration configuration = BelugaConfiguration.getInstance(); HttpClientBuilder clientBuilder = HttpClients.custom(); // timeout/* w w w.j a va 2s. c o m*/ RequestConfig config = RequestConfig.custom().setConnectTimeout(TIMEOUT).setSocketTimeout(TIMEOUT) .setConnectionRequestTimeout(TIMEOUT).build(); clientBuilder.setDefaultRequestConfig(config); switch (configuration.getConnectionType()) { case PROXY_DNS: Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create() .register("http", PlainConnectionSocketFactory.getSocketFactory()) .register("https", SSLConnectionSocketFactory.getSocketFactory()).build(); BelugaDNSResolver dnsOverrider = new BelugaDNSResolver(DNSProxy.PROXY_DNS); connectionManager = new PoolingHttpClientConnectionManager(registry, dnsOverrider); break; case HTTP_PROXY: HttpHost proxy = new HttpHost(configuration.getProxyHost(), configuration.getProxyPort(), "http"); clientBuilder.setProxy(proxy); break; default: } // limit the pool size connectionManager.setDefaultMaxPerRoute(2); // add interceptor, currently for debugging only clientBuilder.addInterceptorFirst(new HttpResponseInterceptor() { @Override public void process(HttpResponse response, HttpContext context) throws HttpException, IOException { HttpInetConnection connection = (HttpInetConnection) context .getAttribute(HttpCoreContext.HTTP_CONNECTION); log.debug("Remote address: " + connection.getRemoteAddress()); // TODO: reimplement blacklisting for DNS proxy by maintaining a // map [DNS IP,RESOLVED IP] in the DNS resolver for reverse // lookup } }); // finally create the HTTP client clientBuilder.setConnectionManager(connectionManager); httpClient = clientBuilder.build(); }
From source file:com.norconex.collector.http.client.impl.GenericHttpClientFactory.java
@Override public HttpClient createHTTPClient(String userAgent) { HttpClientBuilder builder = HttpClientBuilder.create(); builder.setSslcontext(createSSLContext()); builder.setSchemePortResolver(createSchemePortResolver()); builder.setDefaultRequestConfig(createRequestConfig()); builder.setProxy(createProxy()); builder.setDefaultCredentialsProvider(createCredentialsProvider()); builder.setDefaultConnectionConfig(createConnectionConfig()); builder.setUserAgent(userAgent);/*from w w w. ja v a2 s. c om*/ builder.setMaxConnTotal(maxConnections); //builder.setMaxConnPerRoute(maxConnPerRoute) buildCustomHttpClient(builder); //TODO Put in place a more permanent solution to the following //Fix GitHub #17 start RedirectStrategy strategy = createRedirectStrategy(); if (strategy == null) { strategy = LaxRedirectStrategy.INSTANCE; } builder.setRedirectStrategy(new TargetURLRedirectStrategy(strategy)); //Fix end HttpClient httpClient = builder.build(); if (AUTH_METHOD_FORM.equalsIgnoreCase(authMethod)) { authenticateUsingForm(httpClient); } return httpClient; }
From source file:de.comlineag.snc.webcrawler.fetcher.PageFetcher.java
public PageFetcher(CrawlConfig config) { super(config); RequestConfig requestConfig = RequestConfig.custom().setExpectContinueEnabled(false) .setCookieSpec(CookieSpecs.BROWSER_COMPATIBILITY).setRedirectsEnabled(false) .setSocketTimeout(config.getSocketTimeout()).setConnectTimeout(config.getConnectionTimeout()) .build();/*from ww w. jav a 2 s . c o m*/ RegistryBuilder<ConnectionSocketFactory> connRegistryBuilder = RegistryBuilder.create(); connRegistryBuilder.register("http", PlainConnectionSocketFactory.INSTANCE); if (config.isIncludeHttpsPages()) { try { // Fixing: https://code.google.com/p/crawler4j/issues/detail?id=174 // By always trusting the ssl certificate SSLContext sslContext = SSLContexts.custom().loadTrustMaterial(null, new TrustStrategy() { @Override public boolean isTrusted(final X509Certificate[] chain, String authType) { return true; } }).build(); SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER); connRegistryBuilder.register("https", sslsf); } catch (Exception e) { logger.debug("Exception thrown while trying to register https:", e); } } Registry<ConnectionSocketFactory> connRegistry = connRegistryBuilder.build(); connectionManager = new PoolingHttpClientConnectionManager(connRegistry); connectionManager.setMaxTotal(config.getMaxTotalConnections()); connectionManager.setDefaultMaxPerRoute(config.getMaxConnectionsPerHost()); HttpClientBuilder clientBuilder = HttpClientBuilder.create(); clientBuilder.setDefaultRequestConfig(requestConfig); clientBuilder.setConnectionManager(connectionManager); clientBuilder.setUserAgent(config.getUserAgentString()); if (config.getProxyHost() != null) { if (config.getProxyUsername() != null) { BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider(); credentialsProvider.setCredentials(new AuthScope(config.getProxyHost(), config.getProxyPort()), new UsernamePasswordCredentials(config.getProxyUsername(), config.getProxyPassword())); clientBuilder.setDefaultCredentialsProvider(credentialsProvider); } HttpHost proxy = new HttpHost(config.getProxyHost(), config.getProxyPort()); clientBuilder.setProxy(proxy); } clientBuilder.addInterceptorLast(new HttpResponseInterceptor() { @Override public void process(final HttpResponse response, final HttpContext context) throws HttpException, IOException { HttpEntity entity = response.getEntity(); Header contentEncoding = entity.getContentEncoding(); if (contentEncoding != null) { HeaderElement[] codecs = contentEncoding.getElements(); for (HeaderElement codec : codecs) { if (codec.getName().equalsIgnoreCase("gzip")) { response.setEntity(new GzipDecompressingEntity(response.getEntity())); return; } } } } }); httpClient = clientBuilder.build(); if (connectionMonitorThread == null) { connectionMonitorThread = new IdleConnectionMonitorThread(connectionManager); } connectionMonitorThread.start(); }
From source file:crawler.PageFetcher.java
public PageFetcher(CrawlConfig config) { super(config); RequestConfig requestConfig = RequestConfig.custom().setExpectContinueEnabled(false) .setCookieSpec(config.getCookiePolicy()).setRedirectsEnabled(false) .setSocketTimeout(config.getSocketTimeout()).setConnectTimeout(config.getConnectionTimeout()) .build();/*from ww w . ja v a2 s . co m*/ RegistryBuilder<ConnectionSocketFactory> connRegistryBuilder = RegistryBuilder.create(); connRegistryBuilder.register("http", PlainConnectionSocketFactory.INSTANCE); if (config.isIncludeHttpsPages()) { try { // Fixing: https://code.google.com/p/crawler4j/issues/detail?id=174 // By always trusting the ssl certificate SSLContext sslContext = SSLContexts.custom().loadTrustMaterial(null, new TrustStrategy() { @Override public boolean isTrusted(final X509Certificate[] chain, String authType) { return true; } }).build(); SSLConnectionSocketFactory sslsf = new SniSSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE); connRegistryBuilder.register("https", sslsf); } catch (Exception e) { logger.warn("Exception thrown while trying to register https"); logger.debug("Stacktrace", e); } } Registry<ConnectionSocketFactory> connRegistry = connRegistryBuilder.build(); connectionManager = new SniPoolingHttpClientConnectionManager(connRegistry); connectionManager.setMaxTotal(config.getMaxTotalConnections()); connectionManager.setDefaultMaxPerRoute(config.getMaxConnectionsPerHost()); HttpClientBuilder clientBuilder = HttpClientBuilder.create(); clientBuilder.setDefaultRequestConfig(requestConfig); clientBuilder.setConnectionManager(connectionManager); clientBuilder.setUserAgent(config.getUserAgentString()); clientBuilder.setDefaultHeaders(config.getDefaultHeaders()); if (config.getProxyHost() != null) { if (config.getProxyUsername() != null) { BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider(); credentialsProvider.setCredentials(new AuthScope(config.getProxyHost(), config.getProxyPort()), new UsernamePasswordCredentials(config.getProxyUsername(), config.getProxyPassword())); clientBuilder.setDefaultCredentialsProvider(credentialsProvider); } HttpHost proxy = new HttpHost(config.getProxyHost(), config.getProxyPort()); clientBuilder.setProxy(proxy); logger.debug("Working through Proxy: {}", proxy.getHostName()); } httpClient = clientBuilder.build(); if ((config.getAuthInfos() != null) && !config.getAuthInfos().isEmpty()) { doAuthetication(config.getAuthInfos()); } if (connectionMonitorThread == null) { connectionMonitorThread = new IdleConnectionMonitorThread(connectionManager); } connectionMonitorThread.start(); }