List of usage examples for org.apache.http.impl.client HttpClientBuilder setProxy
public final HttpClientBuilder setProxy(final HttpHost proxy)
From source file:eu.europa.ec.markt.dss.validation102853.https.CommonDataLoader.java
/** * Configure the proxy with the required credential if needed * * @param httpClientBuilder//from w ww . j a v a 2 s. co m * @param credentialsProvider * @param uri * @return */ private HttpClientBuilder configureProxy(final HttpClientBuilder httpClientBuilder, final CredentialsProvider credentialsProvider, final URI uri) throws DSSException { if (proxyPreferenceManager == null) { return httpClientBuilder; } final String protocol = uri.getScheme(); final boolean proxyHTTPS = Protocol.isHttps(protocol) && proxyPreferenceManager.isHttpsEnabled(); final boolean proxyHTTP = Protocol.isHttp(protocol) && proxyPreferenceManager.isHttpEnabled(); if (!proxyHTTPS && !proxyHTTP) { return httpClientBuilder; } String proxyHost = null; int proxyPort = 0; String proxyUser = null; String proxyPassword = null; if (proxyHTTPS) { LOG.debug("Use proxy https parameters"); final Long port = proxyPreferenceManager.getHttpsPort(); proxyPort = port != null ? port.intValue() : 0; proxyHost = proxyPreferenceManager.getHttpsHost(); proxyUser = proxyPreferenceManager.getHttpsUser(); proxyPassword = proxyPreferenceManager.getHttpsPassword(); } else if (proxyHTTP) { // noinspection ConstantConditions LOG.debug("Use proxy http parameters"); final Long port = proxyPreferenceManager.getHttpPort(); proxyPort = port != null ? port.intValue() : 0; proxyHost = proxyPreferenceManager.getHttpHost(); proxyUser = proxyPreferenceManager.getHttpUser(); proxyPassword = proxyPreferenceManager.getHttpPassword(); } if (DSSUtils.isNotEmpty(proxyUser) && DSSUtils.isNotEmpty(proxyPassword)) { AuthScope proxyAuth = new AuthScope(proxyHost, proxyPort); UsernamePasswordCredentials proxyCredentials = new UsernamePasswordCredentials(proxyUser, proxyPassword); credentialsProvider.setCredentials(proxyAuth, proxyCredentials); } LOG.debug("proxy host/port: " + proxyHost + ":" + proxyPort); // TODO SSL peer shut down incorrectly when protocol is https final HttpHost proxy = new HttpHost(proxyHost, proxyPort, Protocol.HTTP.getName()); httpClientBuilder.setProxy(proxy); updated = false; return httpClientBuilder; }
From source file:org.glassfish.jersey.apache.connector.ApacheConnector.java
/** * Create the new Apache HTTP Client connector. * * @param client JAX-RS client instance for which the connector is being created. * @param config client configuration./* www . j av a 2 s . c o m*/ */ ApacheConnector(final Client client, final Configuration config) { final Object connectionManager = config.getProperties().get(ApacheClientProperties.CONNECTION_MANAGER); if (connectionManager != null) { if (!(connectionManager instanceof HttpClientConnectionManager)) { LOGGER.log(Level.WARNING, LocalizationMessages.IGNORING_VALUE_OF_PROPERTY(ApacheClientProperties.CONNECTION_MANAGER, connectionManager.getClass().getName(), HttpClientConnectionManager.class.getName())); } } Object reqConfig = config.getProperties().get(ApacheClientProperties.REQUEST_CONFIG); if (reqConfig != null) { if (!(reqConfig instanceof RequestConfig)) { LOGGER.log(Level.WARNING, LocalizationMessages.IGNORING_VALUE_OF_PROPERTY(ApacheClientProperties.REQUEST_CONFIG, reqConfig.getClass().getName(), RequestConfig.class.getName())); reqConfig = null; } } final SSLContext sslContext = client.getSslContext(); final HttpClientBuilder clientBuilder = HttpClientBuilder.create(); clientBuilder.setConnectionManager(getConnectionManager(client, config, sslContext)); clientBuilder.setConnectionManagerShared(PropertiesHelper.getValue(config.getProperties(), ApacheClientProperties.CONNECTION_MANAGER_SHARED, false, null)); clientBuilder.setSslcontext(sslContext); final RequestConfig.Builder requestConfigBuilder = RequestConfig.custom(); final Object credentialsProvider = config.getProperty(ApacheClientProperties.CREDENTIALS_PROVIDER); if (credentialsProvider != null && (credentialsProvider instanceof CredentialsProvider)) { clientBuilder.setDefaultCredentialsProvider((CredentialsProvider) credentialsProvider); } final Object retryHandler = config.getProperties().get(ApacheClientProperties.RETRY_HANDLER); if (retryHandler != null && (retryHandler instanceof HttpRequestRetryHandler)) { clientBuilder.setRetryHandler((HttpRequestRetryHandler) retryHandler); } final Object proxyUri; proxyUri = config.getProperty(ClientProperties.PROXY_URI); if (proxyUri != null) { final URI u = getProxyUri(proxyUri); final HttpHost proxy = new HttpHost(u.getHost(), u.getPort(), u.getScheme()); final String userName; userName = ClientProperties.getValue(config.getProperties(), ClientProperties.PROXY_USERNAME, String.class); if (userName != null) { final String password; password = ClientProperties.getValue(config.getProperties(), ClientProperties.PROXY_PASSWORD, String.class); if (password != null) { final CredentialsProvider credsProvider = new BasicCredentialsProvider(); credsProvider.setCredentials(new AuthScope(u.getHost(), u.getPort()), new UsernamePasswordCredentials(userName, password)); clientBuilder.setDefaultCredentialsProvider(credsProvider); } } clientBuilder.setProxy(proxy); } final Boolean preemptiveBasicAuthProperty = (Boolean) config.getProperties() .get(ApacheClientProperties.PREEMPTIVE_BASIC_AUTHENTICATION); this.preemptiveBasicAuth = (preemptiveBasicAuthProperty != null) ? preemptiveBasicAuthProperty : false; final boolean ignoreCookies = PropertiesHelper.isProperty(config.getProperties(), ApacheClientProperties.DISABLE_COOKIES); if (reqConfig != null) { final RequestConfig.Builder reqConfigBuilder = RequestConfig.copy((RequestConfig) reqConfig); if (ignoreCookies) { reqConfigBuilder.setCookieSpec(CookieSpecs.IGNORE_COOKIES); } requestConfig = reqConfigBuilder.build(); } else { if (ignoreCookies) { requestConfigBuilder.setCookieSpec(CookieSpecs.IGNORE_COOKIES); } requestConfig = requestConfigBuilder.build(); } if (requestConfig.getCookieSpec() == null || !requestConfig.getCookieSpec().equals(CookieSpecs.IGNORE_COOKIES)) { this.cookieStore = new BasicCookieStore(); clientBuilder.setDefaultCookieStore(cookieStore); } else { this.cookieStore = null; } clientBuilder.setDefaultRequestConfig(requestConfig); this.client = clientBuilder.build(); }
From source file:com.github.dockerjava.jaxrs.connector.ApacheConnector.java
/** * Create the new Apache HTTP Client connector. * * @param config client configuration.//from w w w.java2 s.c om */ ApacheConnector(Configuration config) { Object reqConfig = null; if (config != null) { final Object connectionManager = config.getProperties().get(ApacheClientProperties.CONNECTION_MANAGER); if (connectionManager != null) { if (!(connectionManager instanceof HttpClientConnectionManager)) { LOGGER.log(Level.WARNING, LocalizationMessages.IGNORING_VALUE_OF_PROPERTY( ApacheClientProperties.CONNECTION_MANAGER, connectionManager.getClass().getName(), HttpClientConnectionManager.class.getName())); } } reqConfig = config.getProperties().get(ApacheClientProperties.REQUEST_CONFIG); if (reqConfig != null) { if (!(reqConfig instanceof RequestConfig)) { LOGGER.log(Level.WARNING, LocalizationMessages.IGNORING_VALUE_OF_PROPERTY(ApacheClientProperties.REQUEST_CONFIG, reqConfig.getClass().getName(), RequestConfig.class.getName())); reqConfig = null; } } } final SSLContext sslContext = getSslContext(config); final HttpClientBuilder clientBuilder = HttpClientBuilder.create(); clientBuilder.setConnectionManager(getConnectionManager(config, sslContext)); clientBuilder.setSslcontext(sslContext); final RequestConfig.Builder requestConfigBuilder = RequestConfig.custom(); int connectTimeout = 0; int socketTimeout = 0; boolean ignoreCookies = false; if (config != null) { connectTimeout = ClientProperties.getValue(config.getProperties(), ClientProperties.CONNECT_TIMEOUT, 0); socketTimeout = ClientProperties.getValue(config.getProperties(), ClientProperties.READ_TIMEOUT, 0); ignoreCookies = PropertiesHelper.isProperty(config.getProperties(), ApacheClientProperties.DISABLE_COOKIES); final Object credentialsProvider = config.getProperty(ApacheClientProperties.CREDENTIALS_PROVIDER); if (credentialsProvider != null && (credentialsProvider instanceof CredentialsProvider)) { clientBuilder.setDefaultCredentialsProvider((CredentialsProvider) credentialsProvider); } Object proxyUri; proxyUri = config.getProperty(ClientProperties.PROXY_URI); if (proxyUri != null) { final URI u = getProxyUri(proxyUri); final HttpHost proxy = new HttpHost(u.getHost(), u.getPort(), u.getScheme()); String userName; userName = ClientProperties.getValue(config.getProperties(), ClientProperties.PROXY_USERNAME, String.class); if (userName != null) { String password; password = ClientProperties.getValue(config.getProperties(), ClientProperties.PROXY_PASSWORD, String.class); if (password != null) { final CredentialsProvider credsProvider = new BasicCredentialsProvider(); credsProvider.setCredentials(new AuthScope(u.getHost(), u.getPort()), new UsernamePasswordCredentials(userName, password)); clientBuilder.setDefaultCredentialsProvider(credsProvider); } } clientBuilder.setProxy(proxy); } final Boolean preemptiveBasicAuthProperty = (Boolean) config.getProperties() .get(ApacheClientProperties.PREEMPTIVE_BASIC_AUTHENTICATION); this.preemptiveBasicAuth = (preemptiveBasicAuthProperty != null) ? preemptiveBasicAuthProperty : false; } else { this.preemptiveBasicAuth = false; } if (reqConfig != null) { RequestConfig.Builder reqConfigBuilder = RequestConfig.copy((RequestConfig) reqConfig); if (connectTimeout > 0) { reqConfigBuilder.setConnectTimeout(connectTimeout); } if (socketTimeout > 0) { reqConfigBuilder.setSocketTimeout(socketTimeout); } if (ignoreCookies) { reqConfigBuilder.setCookieSpec(CookieSpecs.IGNORE_COOKIES); } requestConfig = reqConfigBuilder.build(); } else { requestConfigBuilder.setConnectTimeout(connectTimeout); requestConfigBuilder.setSocketTimeout(socketTimeout); if (ignoreCookies) { requestConfigBuilder.setCookieSpec(CookieSpecs.IGNORE_COOKIES); } requestConfig = requestConfigBuilder.build(); } if (requestConfig.getCookieSpec() == null || !requestConfig.getCookieSpec().equals(CookieSpecs.IGNORE_COOKIES)) { this.cookieStore = new BasicCookieStore(); clientBuilder.setDefaultCookieStore(cookieStore); } else { this.cookieStore = null; } clientBuilder.setDefaultRequestConfig(requestConfig); this.client = clientBuilder.build(); }
From source file:org.jenkinsci.plugins.bitbucketNotifier.BitbucketNotifier.java
/** * Returns the HttpClient through which the REST call is made. Uses an * unsafe TrustStrategy in case the user specified a HTTPS URL and * set the ignoreUnverifiedSSLPeer flag. * * @param logger the logger to log messages to * @param build/* w w w. j av a 2 s. co m*/ * @return the HttpClient */ private HttpClient getHttpClient(PrintStream logger, AbstractBuild<?, ?> build) throws Exception { boolean ignoreUnverifiedSSL = ignoreUnverifiedSSLPeer; String bitbucketServer = bitbucketServerBaseUrl; DescriptorImpl descriptor = getDescriptor(); // Determine if we are using the local or global settings String credentialsId = getCredentialsId(); if (StringUtils.isBlank(credentialsId)) { credentialsId = descriptor.getCredentialsId(); } Credentials credentials = CredentialsMatchers.firstOrNull(CredentialsProvider .lookupCredentials(CertificateCredentials.class, Jenkins.getInstance(), ACL.SYSTEM), CredentialsMatchers.withId(credentialsId)); if ("".equals(bitbucketServer) || bitbucketServer == null) { bitbucketServer = descriptor.getBitbucketRootUrl(); } if (!ignoreUnverifiedSSL) { ignoreUnverifiedSSL = descriptor.isIgnoreUnverifiedSsl(); } URL url = new URL(bitbucketServer); HttpClientBuilder builder = HttpClientBuilder.create(); if (url.getProtocol().equals("https") && (ignoreUnverifiedSSL || credentials instanceof CertificateCredentials)) { // add unsafe trust manager to avoid thrown // SSLPeerUnverifiedException try { SSLConnectionSocketFactory sslConnSocketFactory = new SSLConnectionSocketFactory( buildSslContext(ignoreUnverifiedSSL, credentials), ignoreUnverifiedSSL ? SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER : null); builder.setSSLSocketFactory(sslConnSocketFactory); Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create() .register("https", sslConnSocketFactory).build(); HttpClientConnectionManager ccm = new BasicHttpClientConnectionManager(registry); builder.setConnectionManager(ccm); } catch (NoSuchAlgorithmException nsae) { logger.println("Couldn't establish SSL context:"); nsae.printStackTrace(logger); } catch (KeyManagementException kme) { logger.println("Couldn't initialize SSL context:"); kme.printStackTrace(logger); } catch (KeyStoreException kse) { logger.println("Couldn't initialize SSL context:"); kse.printStackTrace(logger); } } // Configure the proxy, if needed // Using the Jenkins methods handles the noProxyHost settings ProxyConfiguration proxyConfig = Jenkins.getInstance().proxy; if (proxyConfig != null) { Proxy proxy = proxyConfig.createProxy(url.getHost()); if (proxy != null && proxy.type() == Proxy.Type.HTTP) { SocketAddress addr = proxy.address(); if (addr != null && addr instanceof InetSocketAddress) { InetSocketAddress proxyAddr = (InetSocketAddress) addr; HttpHost proxyHost = new HttpHost(proxyAddr.getAddress().getHostAddress(), proxyAddr.getPort()); builder = builder.setProxy(proxyHost); String proxyUser = proxyConfig.getUserName(); if (proxyUser != null) { String proxyPass = proxyConfig.getPassword(); BasicCredentialsProvider cred = new BasicCredentialsProvider(); cred.setCredentials(new AuthScope(proxyHost), new UsernamePasswordCredentials(proxyUser, proxyPass)); builder = builder.setDefaultCredentialsProvider(cred) .setProxyAuthenticationStrategy(new ProxyAuthenticationStrategy()); } } } } return builder.build(); }
From source file:eu.europa.ec.markt.dss.validation102853.https.CommonsDataLoader.java
/** * Configure the proxy with the required credential if needed * * @param httpClientBuilder/*w w w .ja va 2 s . c om*/ * @param credsProvider * @param url * @return * @throws java.net.MalformedURLException */ private HttpClientBuilder configureProxy(HttpClientBuilder httpClientBuilder, CredentialsProvider credsProvider, String url) throws DSSException { try { if (proxyPreferenceManager == null) { return httpClientBuilder; } final String protocol = new URL(url).getProtocol(); final boolean proxyHTTPS = Protocol.isHttps(protocol) && proxyPreferenceManager.isHttpsEnabled(); final boolean proxyHTTP = Protocol.isHttp(protocol) && proxyPreferenceManager.isHttpEnabled(); if (!proxyHTTPS && !proxyHTTP) { return httpClientBuilder; } String proxyHost = null; int proxyPort = 0; String proxyUser = null; String proxyPassword = null; if (proxyHTTPS) { LOG.debug("Use proxy https parameters"); final Long port = proxyPreferenceManager.getHttpsPort(); proxyPort = port != null ? port.intValue() : 0; proxyHost = proxyPreferenceManager.getHttpsHost(); proxyUser = proxyPreferenceManager.getHttpsUser(); proxyPassword = proxyPreferenceManager.getHttpsPassword(); } else // noinspection ConstantConditions if (proxyHTTP) { LOG.debug("Use proxy http parameters"); final Long port = proxyPreferenceManager.getHttpPort(); proxyPort = port != null ? port.intValue() : 0; proxyHost = proxyPreferenceManager.getHttpHost(); proxyUser = proxyPreferenceManager.getHttpUser(); proxyPassword = proxyPreferenceManager.getHttpPassword(); } if (DSSUtils.isNotEmpty(proxyUser) && DSSUtils.isNotEmpty(proxyPassword)) { LOG.debug("proxy user: " + proxyUser + ":" + proxyPassword); AuthScope proxyAuth = new AuthScope(proxyHost, proxyPort); UsernamePasswordCredentials proxyCredentials = new UsernamePasswordCredentials(proxyUser, proxyPassword); credsProvider.setCredentials(proxyAuth, proxyCredentials); } LOG.debug("proxy host/port: " + proxyHost + ":" + proxyPort); // TODO SSL peer shut down incorrectly when protocol is https HttpHost proxy = new HttpHost(proxyHost, proxyPort, Protocol.HTTP.getName()); return httpClientBuilder.setProxy(proxy); } catch (MalformedURLException e) { throw new DSSException(e); } }
From source file:eu.europa.esig.dss.client.http.commons.CommonsDataLoader.java
/** * Configure the proxy with the required credential if needed * * @param httpClientBuilder//w w w. j a v a 2s .c om * @param credentialsProvider * @param url * @return */ private HttpClientBuilder configureProxy(HttpClientBuilder httpClientBuilder, CredentialsProvider credentialsProvider, String url) throws DSSException { if (proxyPreferenceManager == null) { return httpClientBuilder; } try { final String protocol = new URL(url).getProtocol(); final boolean proxyHTTPS = Protocol.isHttps(protocol) && proxyPreferenceManager.isHttpsEnabled(); final boolean proxyHTTP = Protocol.isHttp(protocol) && proxyPreferenceManager.isHttpEnabled(); if (!proxyHTTPS && !proxyHTTP) { return httpClientBuilder; } String proxyHost = null; int proxyPort = 0; String proxyUser = null; String proxyPassword = null; String proxyExcludedHosts = null; if (proxyHTTPS) { LOG.debug("Use proxy https parameters"); final Long port = proxyPreferenceManager.getHttpsPort(); proxyPort = port != null ? port.intValue() : 0; proxyHost = proxyPreferenceManager.getHttpsHost(); proxyUser = proxyPreferenceManager.getHttpsUser(); proxyPassword = proxyPreferenceManager.getHttpsPassword(); proxyExcludedHosts = proxyPreferenceManager.getHttpsExcludedHosts(); } else if (proxyHTTP) { // noinspection ConstantConditions LOG.debug("Use proxy http parameters"); final Long port = proxyPreferenceManager.getHttpPort(); proxyPort = port != null ? port.intValue() : 0; proxyHost = proxyPreferenceManager.getHttpHost(); proxyUser = proxyPreferenceManager.getHttpUser(); proxyPassword = proxyPreferenceManager.getHttpPassword(); proxyExcludedHosts = proxyPreferenceManager.getHttpExcludedHosts(); } if (StringUtils.isNotEmpty(proxyUser) && StringUtils.isNotEmpty(proxyPassword)) { AuthScope proxyAuth = new AuthScope(proxyHost, proxyPort); UsernamePasswordCredentials proxyCredentials = new UsernamePasswordCredentials(proxyUser, proxyPassword); credentialsProvider.setCredentials(proxyAuth, proxyCredentials); } LOG.debug("proxy host/port: " + proxyHost + ":" + proxyPort); // TODO SSL peer shut down incorrectly when protocol is https final HttpHost proxy = new HttpHost(proxyHost, proxyPort, Protocol.HTTP.getName()); if (StringUtils.isNotEmpty(proxyExcludedHosts)) { final String[] hosts = proxyExcludedHosts.split("[,; ]"); HttpRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy) { @Override public HttpRoute determineRoute(final HttpHost host, final HttpRequest request, final HttpContext context) throws HttpException { String hostname = (host != null ? host.getHostName() : null); if ((hosts != null) && (hostname != null)) { for (String h : hosts) { if (hostname.equalsIgnoreCase(h)) { // bypass proxy for that hostname return new HttpRoute(host); } } } return super.determineRoute(host, request, context); } }; httpClientBuilder.setRoutePlanner(routePlanner); } final HttpClientBuilder httpClientBuilder1 = httpClientBuilder.setProxy(proxy); updated = false; return httpClientBuilder1; } catch (MalformedURLException e) { throw new DSSException(e); } }
From source file:org.apache.nifi.processors.standard.GetHTTP.java
@Override public void onTrigger(final ProcessContext context, final ProcessSessionFactory sessionFactory) throws ProcessException { final ComponentLog logger = getLogger(); final ProcessSession session = sessionFactory.createSession(); final FlowFile incomingFlowFile = session.get(); if (incomingFlowFile != null) { session.transfer(incomingFlowFile, REL_SUCCESS); logger.warn("found FlowFile {} in input queue; transferring to success", new Object[] { incomingFlowFile }); }// w w w. ja v a 2s . c om // get the URL final String url = context.getProperty(URL).evaluateAttributeExpressions().getValue(); final URI uri; String source = url; try { uri = new URI(url); source = uri.getHost(); } catch (final URISyntaxException swallow) { // this won't happen as the url has already been validated } // get the ssl context service final SSLContextService sslContextService = context.getProperty(SSL_CONTEXT_SERVICE) .asControllerService(SSLContextService.class); // create the connection manager final HttpClientConnectionManager conMan; if (sslContextService == null) { conMan = new BasicHttpClientConnectionManager(); } else { final SSLContext sslContext; try { sslContext = createSSLContext(sslContextService); } catch (final Exception e) { throw new ProcessException(e); } final SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext); // Also include a plain socket factory for regular http connections (especially proxies) final Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder .<ConnectionSocketFactory>create().register("https", sslsf) .register("http", PlainConnectionSocketFactory.getSocketFactory()).build(); conMan = new BasicHttpClientConnectionManager(socketFactoryRegistry); } try { // build the request configuration final RequestConfig.Builder requestConfigBuilder = RequestConfig.custom(); requestConfigBuilder.setConnectionRequestTimeout( context.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue()); requestConfigBuilder.setConnectTimeout( context.getProperty(CONNECTION_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue()); requestConfigBuilder.setSocketTimeout( context.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue()); requestConfigBuilder.setRedirectsEnabled(context.getProperty(FOLLOW_REDIRECTS).asBoolean()); switch (context.getProperty(REDIRECT_COOKIE_POLICY).getValue()) { case STANDARD_COOKIE_POLICY_STR: requestConfigBuilder.setCookieSpec(CookieSpecs.STANDARD); break; case STRICT_COOKIE_POLICY_STR: requestConfigBuilder.setCookieSpec(CookieSpecs.STANDARD_STRICT); break; case NETSCAPE_COOKIE_POLICY_STR: requestConfigBuilder.setCookieSpec(CookieSpecs.NETSCAPE); break; case IGNORE_COOKIE_POLICY_STR: requestConfigBuilder.setCookieSpec(CookieSpecs.IGNORE_COOKIES); break; case DEFAULT_COOKIE_POLICY_STR: default: requestConfigBuilder.setCookieSpec(CookieSpecs.DEFAULT); } // build the http client final HttpClientBuilder clientBuilder = HttpClientBuilder.create(); clientBuilder.setConnectionManager(conMan); // include the user agent final String userAgent = context.getProperty(USER_AGENT).getValue(); if (userAgent != null) { clientBuilder.setUserAgent(userAgent); } // set the ssl context if necessary if (sslContextService != null) { clientBuilder.setSslcontext(sslContextService.createSSLContext(ClientAuth.REQUIRED)); } final String username = context.getProperty(USERNAME).getValue(); final String password = context.getProperty(PASSWORD).getValue(); // set the credentials if appropriate if (username != null) { final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); if (password == null) { credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username)); } else { credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password)); } clientBuilder.setDefaultCredentialsProvider(credentialsProvider); } // Set the proxy if specified if (context.getProperty(PROXY_HOST).isSet() && context.getProperty(PROXY_PORT).isSet()) { final String host = context.getProperty(PROXY_HOST).getValue(); final int port = context.getProperty(PROXY_PORT).asInteger(); clientBuilder.setProxy(new HttpHost(host, port)); } // create request final HttpGet get = new HttpGet(url); get.setConfig(requestConfigBuilder.build()); final StateMap beforeStateMap; try { beforeStateMap = context.getStateManager().getState(Scope.LOCAL); final String lastModified = beforeStateMap.get(LAST_MODIFIED + ":" + url); if (lastModified != null) { get.addHeader(HEADER_IF_MODIFIED_SINCE, parseStateValue(lastModified).getValue()); } final String etag = beforeStateMap.get(ETAG + ":" + url); if (etag != null) { get.addHeader(HEADER_IF_NONE_MATCH, parseStateValue(etag).getValue()); } } catch (final IOException ioe) { throw new ProcessException(ioe); } final String accept = context.getProperty(ACCEPT_CONTENT_TYPE).getValue(); if (accept != null) { get.addHeader(HEADER_ACCEPT, accept); } // Add dynamic headers PropertyValue customHeaderValue; for (PropertyDescriptor customProperty : customHeaders) { customHeaderValue = context.getProperty(customProperty).evaluateAttributeExpressions(); if (StringUtils.isNotBlank(customHeaderValue.getValue())) { get.addHeader(customProperty.getName(), customHeaderValue.getValue()); } } // create the http client try (final CloseableHttpClient client = clientBuilder.build()) { // NOTE: including this inner try in order to swallow exceptions on close try { final StopWatch stopWatch = new StopWatch(true); final HttpResponse response = client.execute(get); final int statusCode = response.getStatusLine().getStatusCode(); if (statusCode == NOT_MODIFIED) { logger.info( "content not retrieved because server returned HTTP Status Code {}: Not Modified", new Object[] { NOT_MODIFIED }); context.yield(); // doing a commit in case there were flow files in the input queue session.commit(); return; } final String statusExplanation = response.getStatusLine().getReasonPhrase(); if ((statusCode >= 300) || (statusCode == 204)) { logger.error("received status code {}:{} from {}", new Object[] { statusCode, statusExplanation, url }); // doing a commit in case there were flow files in the input queue session.commit(); return; } FlowFile flowFile = session.create(); flowFile = session.putAttribute(flowFile, CoreAttributes.FILENAME.key(), context.getProperty(FILENAME).evaluateAttributeExpressions().getValue()); flowFile = session.putAttribute(flowFile, this.getClass().getSimpleName().toLowerCase() + ".remote.source", source); flowFile = session.importFrom(response.getEntity().getContent(), flowFile); final Header contentTypeHeader = response.getFirstHeader("Content-Type"); if (contentTypeHeader != null) { final String contentType = contentTypeHeader.getValue(); if (!contentType.trim().isEmpty()) { flowFile = session.putAttribute(flowFile, CoreAttributes.MIME_TYPE.key(), contentType.trim()); } } final long flowFileSize = flowFile.getSize(); stopWatch.stop(); final String dataRate = stopWatch.calculateDataRate(flowFileSize); session.getProvenanceReporter().receive(flowFile, url, stopWatch.getDuration(TimeUnit.MILLISECONDS)); session.transfer(flowFile, REL_SUCCESS); logger.info("Successfully received {} from {} at a rate of {}; transferred to success", new Object[] { flowFile, url, dataRate }); session.commit(); updateStateMap(context, response, beforeStateMap, url); } catch (final IOException e) { context.yield(); session.rollback(); logger.error("Failed to retrieve file from {} due to {}; rolling back session", new Object[] { url, e.getMessage() }, e); throw new ProcessException(e); } catch (final Throwable t) { context.yield(); session.rollback(); logger.error("Failed to process due to {}; rolling back session", new Object[] { t.getMessage() }, t); throw t; } } catch (final IOException e) { logger.debug("Error closing client due to {}, continuing.", new Object[] { e.getMessage() }); } } finally { conMan.shutdown(); } }
From source file:org.apache.nifi.processors.standard.PostHTTP.java
@Override public void onTrigger(final ProcessContext context, final ProcessSession session) { final boolean sendAsFlowFile = context.getProperty(SEND_AS_FLOWFILE).asBoolean(); final int compressionLevel = context.getProperty(COMPRESSION_LEVEL).asInteger(); final String userAgent = context.getProperty(USER_AGENT).getValue(); final RequestConfig.Builder requestConfigBuilder = RequestConfig.custom(); requestConfigBuilder.setConnectionRequestTimeout( context.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue()); requestConfigBuilder.setConnectTimeout( context.getProperty(CONNECTION_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue()); requestConfigBuilder.setRedirectsEnabled(false); requestConfigBuilder/*from w w w. j a v a2 s . c o m*/ .setSocketTimeout(context.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue()); final RequestConfig requestConfig = requestConfigBuilder.build(); final StreamThrottler throttler = throttlerRef.get(); final ComponentLog logger = getLogger(); final Double maxBatchBytes = context.getProperty(MAX_BATCH_SIZE).asDataSize(DataUnit.B); String lastUrl = null; long bytesToSend = 0L; final List<FlowFile> toSend = new ArrayList<>(); DestinationAccepts destinationAccepts = null; CloseableHttpClient client = null; final String transactionId = UUID.randomUUID().toString(); final AtomicReference<String> dnHolder = new AtomicReference<>("none"); while (true) { FlowFile flowFile = session.get(); if (flowFile == null) { break; } final String url = context.getProperty(URL).evaluateAttributeExpressions(flowFile).getValue(); try { new java.net.URL(url); } catch (final MalformedURLException e) { logger.error( "After substituting attribute values for {}, URL is {}; this is not a valid URL, so routing to failure", new Object[] { flowFile, url }); flowFile = session.penalize(flowFile); session.transfer(flowFile, REL_FAILURE); continue; } // If this FlowFile doesn't have the same url, throw it back on the queue and stop grabbing FlowFiles if (lastUrl != null && !lastUrl.equals(url)) { session.transfer(flowFile); break; } lastUrl = url; toSend.add(flowFile); if (client == null || destinationAccepts == null) { final Config config = getConfig(url, context); final HttpClientConnectionManager conMan = config.getConnectionManager(); final HttpClientBuilder clientBuilder = HttpClientBuilder.create(); clientBuilder.setConnectionManager(conMan); clientBuilder.setUserAgent(userAgent); clientBuilder.addInterceptorFirst(new HttpResponseInterceptor() { @Override public void process(final HttpResponse response, final HttpContext httpContext) throws HttpException, IOException { final HttpCoreContext coreContext = HttpCoreContext.adapt(httpContext); final ManagedHttpClientConnection conn = coreContext .getConnection(ManagedHttpClientConnection.class); if (!conn.isOpen()) { return; } final SSLSession sslSession = conn.getSSLSession(); if (sslSession != null) { final Certificate[] certChain = sslSession.getPeerCertificates(); if (certChain == null || certChain.length == 0) { throw new SSLPeerUnverifiedException("No certificates found"); } try { final X509Certificate cert = CertificateUtils .convertAbstractX509Certificate(certChain[0]); dnHolder.set(cert.getSubjectDN().getName().trim()); } catch (CertificateException e) { final String msg = "Could not extract subject DN from SSL session peer certificate"; logger.warn(msg); throw new SSLPeerUnverifiedException(msg); } } } }); clientBuilder.disableAutomaticRetries(); clientBuilder.disableContentCompression(); final String username = context.getProperty(USERNAME).getValue(); final String password = context.getProperty(PASSWORD).getValue(); // set the credentials if appropriate if (username != null) { final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); if (password == null) { credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username)); } else { credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password)); } clientBuilder.setDefaultCredentialsProvider(credentialsProvider); } // Set the proxy if specified if (context.getProperty(PROXY_HOST).isSet() && context.getProperty(PROXY_PORT).isSet()) { final String host = context.getProperty(PROXY_HOST).getValue(); final int port = context.getProperty(PROXY_PORT).asInteger(); clientBuilder.setProxy(new HttpHost(host, port)); } client = clientBuilder.build(); // determine whether or not destination accepts flowfile/gzip destinationAccepts = config.getDestinationAccepts(); if (destinationAccepts == null) { try { destinationAccepts = getDestinationAcceptance(sendAsFlowFile, client, url, getLogger(), transactionId); config.setDestinationAccepts(destinationAccepts); } catch (final IOException e) { flowFile = session.penalize(flowFile); session.transfer(flowFile, REL_FAILURE); logger.error( "Unable to communicate with destination {} to determine whether or not it can accept " + "flowfiles/gzip; routing {} to failure due to {}", new Object[] { url, flowFile, e }); context.yield(); return; } } } bytesToSend += flowFile.getSize(); if (bytesToSend > maxBatchBytes.longValue()) { break; } // if we are not sending as flowfile, or if the destination doesn't accept V3 or V2 (streaming) format, // then only use a single FlowFile if (!sendAsFlowFile || !destinationAccepts.isFlowFileV3Accepted() && !destinationAccepts.isFlowFileV2Accepted()) { break; } } if (toSend.isEmpty()) { return; } final String url = lastUrl; final HttpPost post = new HttpPost(url); final List<FlowFile> flowFileList = toSend; final DestinationAccepts accepts = destinationAccepts; final boolean isDestinationLegacyNiFi = accepts.getProtocolVersion() == null; final EntityTemplate entity = new EntityTemplate(new ContentProducer() { @Override public void writeTo(final OutputStream rawOut) throws IOException { final OutputStream throttled = throttler == null ? rawOut : throttler.newThrottledOutputStream(rawOut); OutputStream wrappedOut = new BufferedOutputStream(throttled); if (compressionLevel > 0 && accepts.isGzipAccepted()) { wrappedOut = new GZIPOutputStream(wrappedOut, compressionLevel); } try (final OutputStream out = wrappedOut) { for (final FlowFile flowFile : flowFileList) { session.read(flowFile, new InputStreamCallback() { @Override public void process(final InputStream rawIn) throws IOException { try (final InputStream in = new BufferedInputStream(rawIn)) { FlowFilePackager packager = null; if (!sendAsFlowFile) { packager = null; } else if (accepts.isFlowFileV3Accepted()) { packager = new FlowFilePackagerV3(); } else if (accepts.isFlowFileV2Accepted()) { packager = new FlowFilePackagerV2(); } else if (accepts.isFlowFileV1Accepted()) { packager = new FlowFilePackagerV1(); } // if none of the above conditions is met, we should never get here, because // we will have already verified that at least 1 of the FlowFile packaging // formats is acceptable if sending as FlowFile. if (packager == null) { StreamUtils.copy(in, out); } else { final Map<String, String> flowFileAttributes; if (isDestinationLegacyNiFi) { // Old versions of NiFi expect nf.file.name and nf.file.path to indicate filename & path; // in order to maintain backward compatibility, we copy the filename & path to those attribute keys. flowFileAttributes = new HashMap<>(flowFile.getAttributes()); flowFileAttributes.put("nf.file.name", flowFile.getAttribute(CoreAttributes.FILENAME.key())); flowFileAttributes.put("nf.file.path", flowFile.getAttribute(CoreAttributes.PATH.key())); } else { flowFileAttributes = flowFile.getAttributes(); } packager.packageFlowFile(in, out, flowFileAttributes, flowFile.getSize()); } } } }); } out.flush(); } } }) { @Override public long getContentLength() { if (compressionLevel == 0 && !sendAsFlowFile && !context.getProperty(CHUNKED_ENCODING).asBoolean()) { return toSend.get(0).getSize(); } else { return -1; } } }; if (context.getProperty(CHUNKED_ENCODING).isSet()) { entity.setChunked(context.getProperty(CHUNKED_ENCODING).asBoolean()); } post.setEntity(entity); post.setConfig(requestConfig); final String contentType; if (sendAsFlowFile) { if (accepts.isFlowFileV3Accepted()) { contentType = APPLICATION_FLOW_FILE_V3; } else if (accepts.isFlowFileV2Accepted()) { contentType = APPLICATION_FLOW_FILE_V2; } else if (accepts.isFlowFileV1Accepted()) { contentType = APPLICATION_FLOW_FILE_V1; } else { logger.error( "Cannot send data to {} because the destination does not accept FlowFiles and this processor is " + "configured to deliver FlowFiles; rolling back session", new Object[] { url }); session.rollback(); context.yield(); IOUtils.closeQuietly(client); return; } } else { final String contentTypeValue = context.getProperty(CONTENT_TYPE) .evaluateAttributeExpressions(toSend.get(0)).getValue(); contentType = StringUtils.isBlank(contentTypeValue) ? DEFAULT_CONTENT_TYPE : contentTypeValue; } final String attributeHeaderRegex = context.getProperty(ATTRIBUTES_AS_HEADERS_REGEX).getValue(); if (attributeHeaderRegex != null && !sendAsFlowFile && flowFileList.size() == 1) { final Pattern pattern = Pattern.compile(attributeHeaderRegex); final Map<String, String> attributes = flowFileList.get(0).getAttributes(); for (final Map.Entry<String, String> entry : attributes.entrySet()) { final String key = entry.getKey(); if (pattern.matcher(key).matches()) { post.setHeader(entry.getKey(), entry.getValue()); } } } post.setHeader(CONTENT_TYPE_HEADER, contentType); post.setHeader(FLOWFILE_CONFIRMATION_HEADER, "true"); post.setHeader(PROTOCOL_VERSION_HEADER, PROTOCOL_VERSION); post.setHeader(TRANSACTION_ID_HEADER, transactionId); if (compressionLevel > 0 && accepts.isGzipAccepted()) { if (sendAsFlowFile) { post.setHeader(GZIPPED_HEADER, "true"); } else { post.setHeader(CONTENT_ENCODING_HEADER, CONTENT_ENCODING_GZIP_VALUE); } } // Do the actual POST final String flowFileDescription = toSend.size() <= 10 ? toSend.toString() : toSend.size() + " FlowFiles"; final String uploadDataRate; final long uploadMillis; CloseableHttpResponse response = null; try { final StopWatch stopWatch = new StopWatch(true); response = client.execute(post); // consume input stream entirely, ignoring its contents. If we // don't do this, the Connection will not be returned to the pool EntityUtils.consume(response.getEntity()); stopWatch.stop(); uploadDataRate = stopWatch.calculateDataRate(bytesToSend); uploadMillis = stopWatch.getDuration(TimeUnit.MILLISECONDS); } catch (final IOException e) { logger.error("Failed to Post {} due to {}; transferring to failure", new Object[] { flowFileDescription, e }); context.yield(); for (FlowFile flowFile : toSend) { flowFile = session.penalize(flowFile); session.transfer(flowFile, REL_FAILURE); } return; } finally { if (response != null) { try { response.close(); } catch (final IOException e) { getLogger().warn("Failed to close HTTP Response due to {}", new Object[] { e }); } } } // If we get a 'SEE OTHER' status code and an HTTP header that indicates that the intent // of the Location URI is a flowfile hold, we will store this holdUri. This prevents us // from posting to some other webservice and then attempting to delete some resource to which // we are redirected final int responseCode = response.getStatusLine().getStatusCode(); final String responseReason = response.getStatusLine().getReasonPhrase(); String holdUri = null; if (responseCode == HttpServletResponse.SC_SEE_OTHER) { final Header locationUriHeader = response.getFirstHeader(LOCATION_URI_INTENT_NAME); if (locationUriHeader != null) { if (LOCATION_URI_INTENT_VALUE.equals(locationUriHeader.getValue())) { final Header holdUriHeader = response.getFirstHeader(LOCATION_HEADER_NAME); if (holdUriHeader != null) { holdUri = holdUriHeader.getValue(); } } } if (holdUri == null) { for (FlowFile flowFile : toSend) { flowFile = session.penalize(flowFile); logger.error( "Failed to Post {} to {}: sent content and received status code {}:{} but no Hold URI", new Object[] { flowFile, url, responseCode, responseReason }); session.transfer(flowFile, REL_FAILURE); } return; } } if (holdUri == null) { if (responseCode == HttpServletResponse.SC_SERVICE_UNAVAILABLE) { for (FlowFile flowFile : toSend) { flowFile = session.penalize(flowFile); logger.error( "Failed to Post {} to {}: response code was {}:{}; will yield processing, " + "since the destination is temporarily unavailable", new Object[] { flowFile, url, responseCode, responseReason }); session.transfer(flowFile, REL_FAILURE); } context.yield(); return; } if (responseCode >= 300) { for (FlowFile flowFile : toSend) { flowFile = session.penalize(flowFile); logger.error("Failed to Post {} to {}: response code was {}:{}", new Object[] { flowFile, url, responseCode, responseReason }); session.transfer(flowFile, REL_FAILURE); } return; } logger.info("Successfully Posted {} to {} in {} at a rate of {}", new Object[] { flowFileDescription, url, FormatUtils.formatMinutesSeconds(uploadMillis, TimeUnit.MILLISECONDS), uploadDataRate }); for (final FlowFile flowFile : toSend) { session.getProvenanceReporter().send(flowFile, url, "Remote DN=" + dnHolder.get(), uploadMillis, true); session.transfer(flowFile, REL_SUCCESS); } return; } // // the response indicated a Hold URI; delete the Hold. // // determine the full URI of the Flow File's Hold; Unfortunately, the responses that are returned have // changed over the past, so we have to take into account a few different possibilities. String fullHoldUri = holdUri; if (holdUri.startsWith("/contentListener")) { // If the Hold URI that we get starts with /contentListener, it may not really be /contentListener, // as this really indicates that it should be whatever we posted to -- if posting directly to the // ListenHTTP component, it will be /contentListener, but if posting to a proxy/load balancer, we may // be posting to some other URL. fullHoldUri = url + holdUri.substring(16); } else if (holdUri.startsWith("/")) { // URL indicates the full path but not hostname or port; use the same hostname & port that we posted // to but use the full path indicated by the response. int firstSlash = url.indexOf("/", 8); if (firstSlash < 0) { firstSlash = url.length(); } final String beforeSlash = url.substring(0, firstSlash); fullHoldUri = beforeSlash + holdUri; } else if (!holdUri.startsWith("http")) { // Absolute URL fullHoldUri = url + (url.endsWith("/") ? "" : "/") + holdUri; } final HttpDelete delete = new HttpDelete(fullHoldUri); delete.setHeader(TRANSACTION_ID_HEADER, transactionId); while (true) { try { final HttpResponse holdResponse = client.execute(delete); EntityUtils.consume(holdResponse.getEntity()); final int holdStatusCode = holdResponse.getStatusLine().getStatusCode(); final String holdReason = holdResponse.getStatusLine().getReasonPhrase(); if (holdStatusCode >= 300) { logger.error( "Failed to delete Hold that destination placed on {}: got response code {}:{}; routing to failure", new Object[] { flowFileDescription, holdStatusCode, holdReason }); for (FlowFile flowFile : toSend) { flowFile = session.penalize(flowFile); session.transfer(flowFile, REL_FAILURE); } return; } logger.info("Successfully Posted {} to {} in {} milliseconds at a rate of {}", new Object[] { flowFileDescription, url, uploadMillis, uploadDataRate }); for (final FlowFile flowFile : toSend) { session.getProvenanceReporter().send(flowFile, url); session.transfer(flowFile, REL_SUCCESS); } return; } catch (final IOException e) { logger.warn("Failed to delete Hold that destination placed on {} due to {}", new Object[] { flowFileDescription, e }); } if (!isScheduled()) { context.yield(); logger.warn( "Failed to delete Hold that destination placed on {}; Processor has been stopped so routing FlowFile(s) to failure", new Object[] { flowFileDescription }); for (FlowFile flowFile : toSend) { flowFile = session.penalize(flowFile); session.transfer(flowFile, REL_FAILURE); } return; } } }
From source file:org.opensextant.xtext.collectors.web.WebClient.java
/** * TODO: Update to use HTTP client "HttpClients....build()" method of creating and tailoring HttpClient * using the proxy and cookie settings, as well as any other tuning. * * Override if your context requires a different style of HTTP client. * /*from ww w .j a va 2 s . com*/ * @return HttpClient 4.x object */ public HttpClient getClient() { HttpClientBuilder clientHelper = null; if (this.useSystemProperties) { clientHelper = HttpClientBuilder.create().useSystemProperties(); } else { clientHelper = HttpClientBuilder.create(); if (proxyHost != null) { clientHelper.setProxy(proxyHost); } } RequestConfig globalConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.BROWSER_COMPATIBILITY) .build(); HttpClient httpClient = clientHelper.setDefaultRequestConfig(globalConfig).build(); return httpClient; }
From source file:org.tanaguru.util.http.HttpRequestHandler.java
private CloseableHttpClient getHttpClient(String url) { RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(socketTimeout) .setConnectTimeout(connectionTimeout).build(); HttpClientBuilder httpClientBuilder = HttpClientBuilder.create(); httpClientBuilder.setDefaultRequestConfig(requestConfig); httpClientBuilder.setConnectionManager(new PoolingHttpClientConnectionManager()); httpClientBuilder.setUserAgent(TANAGURU_USER_AGENT); if (isProxySet(url)) { LOGGER.debug(("Set proxy with " + proxyHost + " and " + proxyPort)); httpClientBuilder.setProxy(new HttpHost(proxyHost, Integer.valueOf(proxyPort))); if (isProxyCredentialSet()) { CredentialsProvider credsProvider = new BasicCredentialsProvider(); credsProvider.setCredentials(new AuthScope(proxyHost, Integer.valueOf(proxyPort)), new UsernamePasswordCredentials(proxyUser, proxyPassword)); httpClientBuilder.setDefaultCredentialsProvider(credsProvider); LOGGER.debug(("Set proxy credentials " + proxyHost + " and " + proxyPort + " and " + proxyUser + " and " + proxyPassword)); }/*from w w w . j a va 2s. c o m*/ } return httpClientBuilder.build(); }