List of usage examples for org.apache.http.impl.client HttpClientBuilder build
public CloseableHttpClient build()
From source file:com.petalmd.armor.AbstractUnitTest.java
protected final HeaderAwareJestHttpClient getJestClient(final String serverUri, final String username, final String password) throws Exception {// http://hc.apache.org/httpcomponents-client-ga/tutorial/html/authentication.html final CredentialsProvider credsProvider = new BasicCredentialsProvider(); final HttpClientConfig clientConfig1 = new HttpClientConfig.Builder(serverUri).multiThreaded(true).build(); // Construct a new Jest client according to configuration via factory final HeaderAwareJestClientFactory factory1 = new HeaderAwareJestClientFactory(); factory1.setHttpClientConfig(clientConfig1); final HeaderAwareJestHttpClient c = factory1.getObject(); final HttpClientBuilder hcb = HttpClients.custom(); if (username != null) { credsProvider.setCredentials(new AuthScope(AuthScope.ANY), new UsernamePasswordCredentials(username, password)); }//from w w w .j a v a 2 s. c o m if (useSpnego) { //SPNEGO/Kerberos setup log.debug("SPNEGO activated"); final AuthSchemeProvider nsf = new SPNegoSchemeFactory(true, false);// new NegotiateSchemeProvider(); final Credentials jaasCreds = new JaasCredentials(); credsProvider.setCredentials(new AuthScope(null, -1, null, AuthSchemes.SPNEGO), jaasCreds); credsProvider.setCredentials(new AuthScope(null, -1, null, AuthSchemes.NTLM), new NTCredentials("Guest", "Guest", "Guest", "Guest")); final Registry<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create() .register(AuthSchemes.SPNEGO, nsf).register(AuthSchemes.NTLM, new NTLMSchemeFactory()).build(); hcb.setDefaultAuthSchemeRegistry(authSchemeRegistry); } hcb.setDefaultCredentialsProvider(credsProvider); if (serverUri.startsWith("https")) { log.debug("Configure Jest with SSL"); final KeyStore myTrustStore = KeyStore.getInstance("JKS"); myTrustStore.load(new FileInputStream(SecurityUtil.getAbsoluteFilePathFromClassPath("ArmorTS.jks")), "changeit".toCharArray()); final KeyStore keyStore = KeyStore.getInstance("JKS"); keyStore.load(new FileInputStream(SecurityUtil.getAbsoluteFilePathFromClassPath("ArmorKS.jks")), "changeit".toCharArray()); final SSLContext sslContext = SSLContexts.custom().useTLS() .loadKeyMaterial(keyStore, "changeit".toCharArray()).loadTrustMaterial(myTrustStore).build(); String[] protocols = null; if (enableSSLv3Only) { protocols = new String[] { "SSLv3" }; } else { protocols = SecurityUtil.ENABLED_SSL_PROTOCOLS; } final SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext, protocols, SecurityUtil.ENABLED_SSL_CIPHERS, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER); hcb.setSSLSocketFactory(sslsf); } hcb.setDefaultSocketConfig(SocketConfig.custom().setSoTimeout(60 * 1000).build()); final CloseableHttpClient httpClient = hcb.build(); c.setHttpClient(httpClient); return c; }
From source file:net.www_eee.portal.channels.ProxyChannel.java
/** * Construct and configure the {@link HttpClient} to be used to * {@linkplain #doProxyRequest(Page.Request, Channel.Mode) proxy} content while fulfilling the specified * <code>pageRequest</code>. This method will also configure {@linkplain ProxyRequestInterceptor request} and * {@linkplain ProxyResponseInterceptor response} interceptors on the created client. * //from w w w. j a v a 2 s.c om * @param pageRequest The {@link net.www_eee.portal.Page.Request Request} currently being processed. * @return The created {@link HttpClient}. * @throws WWWEEEPortal.Exception If a problem occurred while determining the result. * @see #doProxyRequest(Page.Request, Channel.Mode) * @see #PROXY_CLIENT_HOOK * @see ProxyRequestInterceptor * @see ProxyResponseInterceptor */ protected CloseableHttpClient createProxyClient(final Page.Request pageRequest) throws WWWEEEPortal.Exception { final org.apache.http.client.CookieStore proxyClientCookieStore = createProxyClientCookieStore(pageRequest); Object[] context = new Object[] { proxyClientManager, proxyClientCookieStore }; HttpClientBuilder proxyClientBuilder = PROXY_CLIENT_HOOK.value(plugins, context, pageRequest); if (proxyClientBuilder == null) { final HttpClientBuilder pcb = HttpClientBuilder.create(); Optional.ofNullable(proxyClientManager) .ifPresent((proxyClientManager) -> pcb.setConnectionManager(proxyClientManager)); proxyClientBuilder = pcb; proxyClientBuilder.setDefaultCookieStore(proxyClientCookieStore); proxyClientBuilder = proxyClientBuilder.addInterceptorLast(new ProxyRequestInterceptor(pageRequest)); proxyClientBuilder = proxyClientBuilder.addInterceptorLast(new ProxyResponseInterceptor(pageRequest)); } proxyClientBuilder = PROXY_CLIENT_HOOK .requireFilteredResult(PROXY_CLIENT_HOOK.filter(plugins, context, pageRequest, proxyClientBuilder)); return proxyClientBuilder.build(); }
From source file:microsoft.exchange.webservices.data.HttpClientWebRequest.java
/** * Prepare connection//w w w .ja va 2s. c o m * * @throws microsoft.exchange.webservices.data.EWSHttpException the eWS http exception */ @Override public void prepareConnection() throws EWSHttpException { try { HttpClientBuilder builder = HttpClients.custom(); builder.setConnectionManager(this.httpClientConnMng); //create the cookie store if (cookieStore == null) { cookieStore = new BasicCookieStore(); } builder.setDefaultCookieStore(cookieStore); if (getProxy() != null) { HttpHost proxy = new HttpHost(getProxy().getHost(), getProxy().getPort()); builder.setProxy(proxy); if (HttpProxyCredentials.isProxySet()) { NTCredentials cred = new NTCredentials(HttpProxyCredentials.getUserName(), HttpProxyCredentials.getPassword(), "", HttpProxyCredentials.getDomain()); CredentialsProvider credsProvider = new BasicCredentialsProvider(); credsProvider.setCredentials(new AuthScope(proxy), cred); builder.setDefaultCredentialsProvider(credsProvider); } } if (getUserName() != null) { CredentialsProvider credsProvider = new BasicCredentialsProvider(); credsProvider.setCredentials(AuthScope.ANY, new NTCredentials(getUserName(), getPassword(), "", getDomain())); builder.setDefaultCredentialsProvider(credsProvider); } //fix socket config SocketConfig sc = SocketConfig.custom().setSoTimeout(getTimeout()).build(); builder.setDefaultSocketConfig(sc); RequestConfig.Builder rcBuilder = RequestConfig.custom(); rcBuilder.setAuthenticationEnabled(true); rcBuilder.setConnectionRequestTimeout(getTimeout()); rcBuilder.setConnectTimeout(getTimeout()); rcBuilder.setRedirectsEnabled(isAllowAutoRedirect()); rcBuilder.setSocketTimeout(getTimeout()); // fix issue #144 + #160: if we used NTCredentials from above: these are NT credentials if (getUserName() != null) { ArrayList<String> authPrefs = new ArrayList<String>(); authPrefs.add(AuthSchemes.NTLM); rcBuilder.setTargetPreferredAuthSchemes(authPrefs); } // builder.setDefaultRequestConfig(rcBuilder.build()); httpPostReq = new HttpPost(getUrl().toString()); httpPostReq.addHeader("Content-type", getContentType()); //httpPostReq.setDoAuthentication(true); httpPostReq.addHeader("User-Agent", getUserAgent()); httpPostReq.addHeader("Accept", getAccept()); httpPostReq.addHeader("Keep-Alive", "300"); httpPostReq.addHeader("Connection", "Keep-Alive"); if (isAcceptGzipEncoding()) { httpPostReq.addHeader("Accept-Encoding", "gzip,deflate"); } if (getHeaders().size() > 0) { for (Map.Entry<String, String> httpHeader : getHeaders().entrySet()) { httpPostReq.addHeader(httpHeader.getKey(), httpHeader.getValue()); } } //create the client client = builder.build(); } catch (Exception er) { er.printStackTrace(); } }
From source file:com.floragunn.searchguard.httpclient.HttpClient.java
private final CloseableHttpClient createHTTPClient() throws NoSuchAlgorithmException, KeyStoreException, CertificateException, FileNotFoundException, IOException, UnrecoverableKeyException, KeyManagementException { // basic auth // pki auth/*from w w w.j a v a2 s . c om*/ // kerberos auth final org.apache.http.impl.client.HttpClientBuilder hcb = HttpClients.custom(); if (ssl) { final SSLContextBuilder sslContextbBuilder = SSLContexts.custom().useTLS(); if (log.isTraceEnabled()) { log.trace("Configure HTTP client with SSL"); } if (trustStore != null) { final KeyStore myTrustStore = KeyStore .getInstance(trustStore.getName().endsWith("jks") ? "JKS" : "PKCS12"); myTrustStore.load(new FileInputStream(trustStore), truststorePassword == null || truststorePassword.isEmpty() ? null : truststorePassword.toCharArray()); sslContextbBuilder.loadTrustMaterial(myTrustStore); } if (keystore != null) { final KeyStore keyStore = KeyStore .getInstance(keystore.getName().endsWith("jks") ? "JKS" : "PKCS12"); keyStore.load(new FileInputStream(keystore), keystorePassword == null || keystorePassword.isEmpty() ? null : keystorePassword.toCharArray()); sslContextbBuilder.loadKeyMaterial(keyStore, keystorePassword == null || keystorePassword.isEmpty() ? null : keystorePassword.toCharArray()); } final SSLContext sslContext = sslContextbBuilder.build(); final SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext, new String[] { "TLSv1.1", "TLSv1.2" }, null, verifyHostnames ? SSLConnectionSocketFactory.STRICT_HOSTNAME_VERIFIER : SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER); hcb.setSSLSocketFactory(sslsf); } /*if (keytab != null) { //System.setProperty("java.security.auth.login.config", "login.conf"); //System.setProperty("java.security.krb5.conf", "krb5.conf"); final CredentialsProvider credsProvider = new BasicCredentialsProvider(); //SPNEGO/Kerberos setup log.debug("SPNEGO activated"); final AuthSchemeProvider nsf = new LoginSPNegoSchemeFactory(true); final Credentials jaasCreds = new JaasCredentials(); credsProvider.setCredentials(new AuthScope(null, -1, null, AuthSchemes.SPNEGO), jaasCreds); credsProvider.setCredentials(new AuthScope(null, -1, null, AuthSchemes.NTLM), new NTCredentials("Guest", "Guest", "Guest", "Guest")); final Registry<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider> create() .register(AuthSchemes.SPNEGO, nsf).register(AuthSchemes.NTLM, new NTLMSchemeFactory()).build(); hcb.setDefaultAuthSchemeRegistry(authSchemeRegistry); hcb.setDefaultCredentialsProvider(credsProvider); }*/ if (basicCredentials != null) { hcb.setDefaultHeaders( Lists.newArrayList(new BasicHeader(HttpHeaders.AUTHORIZATION, "Basic " + basicCredentials))); } return hcb.build(); }
From source file:org.archive.modules.recrawl.wbm.WbmPersistLoadProcessor.java
public synchronized HttpClient getHttpClient() { if (client == null) { if (conman == null) { conman = new PoolingHttpClientConnectionManager(); conman.setDefaultMaxPerRoute(maxConnections); conman.setMaxTotal(Math.max(conman.getMaxTotal(), maxConnections)); }//from ww w . j a v a 2 s . c o m HttpClientBuilder builder = HttpClientBuilder.create().disableCookieManagement() .setConnectionManager(conman); builder.useSystemProperties(); // TODO: use setDefaultHeaders for adding requestHeaders? It's a bit painful // because we need to convert it to a Collection of Header objects. // config code for older version of httpclient. // builder.setDefaultSocketConfig(SocketConfig.custom().setSoTimeout(socketTimeout).build()); // HttpParams params = client.getParams(); // params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT, socketTimeout); // params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, connectionTimeout); // setup request/response intercepter for handling gzip-compressed response. // Disabled because httpclient 4.3.3 sends "Accept-Encoding: gzip,deflate" by // default. Response parsing will fail If gzip-decompression ResponseInterceptor // is installed. builder.addInterceptorLast(new HttpRequestInterceptor() { @Override public void process(final HttpRequest request, final HttpContext context) { // System.err.println("RequestInterceptor"); // if (request.containsHeader("Accept-Encoding")) { // System.err.println("already has Accept-Encoding: " + request.getHeaders("Accept-Encoding")[0]); // } // if (gzipAccepted) { // if (!request.containsHeader("Accept-Encoding")) { // request.addHeader("Accept-Encoding", "gzip"); // } // } // add extra headers configured. if (requestHeaders != null) { for (Entry<String, String> ent : requestHeaders.entrySet()) { request.addHeader(ent.getKey(), ent.getValue()); } } } }); // builder.addInterceptorFirst(new HttpResponseInterceptor() { // @Override // public void process(final HttpResponse response, final HttpContext context) // throws HttpException, IOException { // HttpEntity entity = response.getEntity(); // Header ceheader = entity.getContentEncoding(); // if (ceheader != null && contains(ceheader.getElements(), "gzip")) { // response.setEntity(new GzipInflatingHttpEntityWrapper(response.getEntity())); // } // } // }); this.client = builder.build(); } return client; }
From source file:org.jenkinsci.plugins.bitbucketNotifier.BitbucketNotifier.java
/** * Returns the HttpClient through which the REST call is made. Uses an * unsafe TrustStrategy in case the user specified a HTTPS URL and * set the ignoreUnverifiedSSLPeer flag. * * @param logger the logger to log messages to * @param build/* w w w . j a v a 2s . c o m*/ * @return the HttpClient */ private HttpClient getHttpClient(PrintStream logger, AbstractBuild<?, ?> build) throws Exception { boolean ignoreUnverifiedSSL = ignoreUnverifiedSSLPeer; String bitbucketServer = bitbucketServerBaseUrl; DescriptorImpl descriptor = getDescriptor(); // Determine if we are using the local or global settings String credentialsId = getCredentialsId(); if (StringUtils.isBlank(credentialsId)) { credentialsId = descriptor.getCredentialsId(); } Credentials credentials = CredentialsMatchers.firstOrNull(CredentialsProvider .lookupCredentials(CertificateCredentials.class, Jenkins.getInstance(), ACL.SYSTEM), CredentialsMatchers.withId(credentialsId)); if ("".equals(bitbucketServer) || bitbucketServer == null) { bitbucketServer = descriptor.getBitbucketRootUrl(); } if (!ignoreUnverifiedSSL) { ignoreUnverifiedSSL = descriptor.isIgnoreUnverifiedSsl(); } URL url = new URL(bitbucketServer); HttpClientBuilder builder = HttpClientBuilder.create(); if (url.getProtocol().equals("https") && (ignoreUnverifiedSSL || credentials instanceof CertificateCredentials)) { // add unsafe trust manager to avoid thrown // SSLPeerUnverifiedException try { SSLConnectionSocketFactory sslConnSocketFactory = new SSLConnectionSocketFactory( buildSslContext(ignoreUnverifiedSSL, credentials), ignoreUnverifiedSSL ? SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER : null); builder.setSSLSocketFactory(sslConnSocketFactory); Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create() .register("https", sslConnSocketFactory).build(); HttpClientConnectionManager ccm = new BasicHttpClientConnectionManager(registry); builder.setConnectionManager(ccm); } catch (NoSuchAlgorithmException nsae) { logger.println("Couldn't establish SSL context:"); nsae.printStackTrace(logger); } catch (KeyManagementException kme) { logger.println("Couldn't initialize SSL context:"); kme.printStackTrace(logger); } catch (KeyStoreException kse) { logger.println("Couldn't initialize SSL context:"); kse.printStackTrace(logger); } } // Configure the proxy, if needed // Using the Jenkins methods handles the noProxyHost settings ProxyConfiguration proxyConfig = Jenkins.getInstance().proxy; if (proxyConfig != null) { Proxy proxy = proxyConfig.createProxy(url.getHost()); if (proxy != null && proxy.type() == Proxy.Type.HTTP) { SocketAddress addr = proxy.address(); if (addr != null && addr instanceof InetSocketAddress) { InetSocketAddress proxyAddr = (InetSocketAddress) addr; HttpHost proxyHost = new HttpHost(proxyAddr.getAddress().getHostAddress(), proxyAddr.getPort()); builder = builder.setProxy(proxyHost); String proxyUser = proxyConfig.getUserName(); if (proxyUser != null) { String proxyPass = proxyConfig.getPassword(); BasicCredentialsProvider cred = new BasicCredentialsProvider(); cred.setCredentials(new AuthScope(proxyHost), new UsernamePasswordCredentials(proxyUser, proxyPass)); builder = builder.setDefaultCredentialsProvider(cred) .setProxyAuthenticationStrategy(new ProxyAuthenticationStrategy()); } } } } return builder.build(); }
From source file:com.cognitivemedicine.nifi.http.PostHTTP2.java
@Override public void onTrigger(final ProcessContext context, final ProcessSession session) { final boolean sendAsFlowFile = context.getProperty(SEND_AS_FLOWFILE).asBoolean(); final int compressionLevel = context.getProperty(COMPRESSION_LEVEL).asInteger(); final String userAgent = context.getProperty(USER_AGENT).getValue(); final RequestConfig.Builder requestConfigBuilder = RequestConfig.custom(); requestConfigBuilder.setConnectionRequestTimeout( context.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue()); requestConfigBuilder.setConnectTimeout( context.getProperty(CONNECTION_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue()); requestConfigBuilder.setRedirectsEnabled(false); requestConfigBuilder//from w w w.j a v a 2 s . com .setSocketTimeout(context.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue()); final RequestConfig requestConfig = requestConfigBuilder.build(); final StreamThrottler throttler = throttlerRef.get(); final ProcessorLog logger = getLogger(); final Double maxBatchBytes = context.getProperty(MAX_BATCH_SIZE).asDataSize(DataUnit.B); String lastUrl = null; long bytesToSend = 0L; final List<FlowFile> toSend = new ArrayList<>(); DestinationAccepts destinationAccepts = null; CloseableHttpClient client = null; final String transactionId = UUID.randomUUID().toString(); final ObjectHolder<String> dnHolder = new ObjectHolder<>("none"); while (true) { FlowFile flowFile = session.get(); if (flowFile == null) { break; } final String url = context.getProperty(URL).evaluateAttributeExpressions(flowFile).getValue(); try { new java.net.URL(url); } catch (final MalformedURLException e) { logger.error( "After substituting attribute values for {}, URL is {}; this is not a valid URL, so routing to failure", new Object[] { flowFile, url }); flowFile = session.penalize(flowFile); session.transfer(flowFile, REL_FAILURE); continue; } // If this FlowFile doesn't have the same url, throw it back on the queue and stop grabbing FlowFiles if (lastUrl != null && !lastUrl.equals(url)) { session.transfer(flowFile); break; } lastUrl = url; toSend.add(flowFile); if (client == null || destinationAccepts == null) { final Config config = getConfig(url, context); final HttpClientConnectionManager conMan = config.getConnectionManager(); final HttpClientBuilder clientBuilder = HttpClientBuilder.create(); clientBuilder.setConnectionManager(conMan); clientBuilder.setUserAgent(userAgent); clientBuilder.addInterceptorFirst(new HttpResponseInterceptor() { @Override public void process(final HttpResponse response, final HttpContext httpContext) throws HttpException, IOException { HttpCoreContext coreContext = HttpCoreContext.adapt(httpContext); ManagedHttpClientConnection conn = coreContext .getConnection(ManagedHttpClientConnection.class); if (!conn.isOpen()) { return; } SSLSession sslSession = conn.getSSLSession(); if (sslSession != null) { final X509Certificate[] certChain = sslSession.getPeerCertificateChain(); if (certChain == null || certChain.length == 0) { throw new SSLPeerUnverifiedException("No certificates found"); } final X509Certificate cert = certChain[0]; dnHolder.set(cert.getSubjectDN().getName().trim()); } } }); clientBuilder.disableAutomaticRetries(); clientBuilder.disableContentCompression(); final String username = context.getProperty(USERNAME).getValue(); final String password = context.getProperty(PASSWORD).getValue(); // set the credentials if appropriate if (username != null) { final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); if (password == null) { credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username)); } else { credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password)); } ; clientBuilder.setDefaultCredentialsProvider(credentialsProvider); } client = clientBuilder.build(); // determine whether or not destination accepts flowfile/gzip destinationAccepts = config.getDestinationAccepts(); if (destinationAccepts == null) { try { if (sendAsFlowFile) { destinationAccepts = getDestinationAcceptance(client, url, getLogger(), transactionId); } else { destinationAccepts = new DestinationAccepts(false, false, false, false, null); } config.setDestinationAccepts(destinationAccepts); } catch (IOException e) { flowFile = session.penalize(flowFile); session.transfer(flowFile, REL_FAILURE); logger.error( "Unable to communicate with destination {} to determine whether or not it can accept flowfiles/gzip; routing {} to failure due to {}", new Object[] { url, flowFile, e }); context.yield(); return; } } } // if we are not sending as flowfile, or if the destination doesn't accept V3 or V2 (streaming) format, // then only use a single FlowFile if (!sendAsFlowFile || (!destinationAccepts.isFlowFileV3Accepted() && !destinationAccepts.isFlowFileV2Accepted())) { break; } bytesToSend += flowFile.getSize(); if (bytesToSend > maxBatchBytes.longValue()) { break; } } if (toSend.isEmpty()) { return; } final String url = lastUrl; final HttpPost post = new HttpPost(url); final List<FlowFile> flowFileList = toSend; final DestinationAccepts accepts = destinationAccepts; final boolean isDestinationLegacyNiFi = accepts.getProtocolVersion() == null; final EntityTemplate entity = new EntityTemplate(new ContentProducer() { @Override public void writeTo(final OutputStream rawOut) throws IOException { final OutputStream throttled = (throttler == null) ? rawOut : throttler.newThrottledOutputStream(rawOut); OutputStream wrappedOut = new BufferedOutputStream(throttled); if (compressionLevel > 0 && accepts.isGzipAccepted()) { wrappedOut = new GZIPOutputStream(wrappedOut, compressionLevel); } try (final OutputStream out = wrappedOut) { for (final FlowFile flowFile : flowFileList) { session.read(flowFile, new InputStreamCallback() { @Override public void process(final InputStream rawIn) throws IOException { try (final InputStream in = new BufferedInputStream(rawIn)) { FlowFilePackager packager = null; if (!sendAsFlowFile) { packager = null; } else if (accepts.isFlowFileV3Accepted()) { packager = new FlowFilePackagerV3(); } else if (accepts.isFlowFileV2Accepted()) { packager = new FlowFilePackagerV2(); } else if (accepts.isFlowFileV1Accepted()) { packager = new FlowFilePackagerV1(); } // if none of the above conditions is met, we should never get here, because // we will have already verified that at least 1 of the FlowFile packaging // formats is acceptable if sending as FlowFile. if (packager == null) { StreamUtils.copy(in, out); } else { final Map<String, String> flowFileAttributes; if (isDestinationLegacyNiFi) { // Old versions of NiFi expect nf.file.name and nf.file.path to indicate filename & path; // in order to maintain backward compatibility, we copy the filename & path to those attribute keys. flowFileAttributes = new HashMap<>(flowFile.getAttributes()); flowFileAttributes.put("nf.file.name", flowFile.getAttribute(CoreAttributes.FILENAME.key())); flowFileAttributes.put("nf.file.path", flowFile.getAttribute(CoreAttributes.PATH.key())); } else { flowFileAttributes = flowFile.getAttributes(); } packager.packageFlowFile(in, out, flowFileAttributes, flowFile.getSize()); } } } }); } out.flush(); } } }); entity.setChunked(context.getProperty(CHUNKED_ENCODING).asBoolean()); post.setEntity(entity); post.setConfig(requestConfig); final String contentType; if (sendAsFlowFile) { if (accepts.isFlowFileV3Accepted()) { contentType = APPLICATION_FLOW_FILE_V3; } else if (accepts.isFlowFileV2Accepted()) { contentType = APPLICATION_FLOW_FILE_V2; } else if (accepts.isFlowFileV1Accepted()) { contentType = APPLICATION_FLOW_FILE_V1; } else { logger.error( "Cannot send data to {} because the destination does not accept FlowFiles and this processor is configured to deliver FlowFiles; rolling back session", new Object[] { url }); session.rollback(); context.yield(); return; } } else { final String attributeValue = toSend.get(0).getAttribute(CoreAttributes.MIME_TYPE.key()); contentType = (attributeValue == null) ? DEFAULT_CONTENT_TYPE : attributeValue; } final String attributeHeaderRegex = context.getProperty(ATTRIBUTES_AS_HEADERS_REGEX).getValue(); if (attributeHeaderRegex != null && !sendAsFlowFile && flowFileList.size() == 1) { final Pattern pattern = Pattern.compile(attributeHeaderRegex); final Map<String, String> attributes = flowFileList.get(0).getAttributes(); for (final Map.Entry<String, String> entry : attributes.entrySet()) { final String key = entry.getKey(); if (pattern.matcher(key).matches()) { post.setHeader(entry.getKey(), entry.getValue()); } } } post.setHeader(CONTENT_TYPE, contentType); post.setHeader(FLOWFILE_CONFIRMATION_HEADER, "true"); post.setHeader(PROTOCOL_VERSION_HEADER, PROTOCOL_VERSION); post.setHeader(TRANSACTION_ID_HEADER, transactionId); if (compressionLevel > 0 && accepts.isGzipAccepted()) { post.setHeader(GZIPPED_HEADER, "true"); } // Do the actual POST final String flowFileDescription = toSend.size() <= 10 ? toSend.toString() : toSend.size() + " FlowFiles"; final String uploadDataRate; final long uploadMillis; CloseableHttpResponse response = null; try { final StopWatch stopWatch = new StopWatch(true); response = client.execute(post); // consume input stream entirely, ignoring its contents. If we // don't do this, the Connection will not be returned to the pool EntityUtils.consume(response.getEntity()); stopWatch.stop(); uploadDataRate = stopWatch.calculateDataRate(bytesToSend); uploadMillis = stopWatch.getDuration(TimeUnit.MILLISECONDS); } catch (final IOException e) { logger.error("Failed to Post {} due to {}; transferring to failure", new Object[] { flowFileDescription, e }); context.yield(); for (FlowFile flowFile : toSend) { flowFile = session.penalize(flowFile); session.transfer(flowFile, REL_FAILURE); } return; } finally { if (response != null) { try { response.close(); } catch (IOException e) { getLogger().warn("Failed to close HTTP Response due to {}", new Object[] { e }); } } } // If we get a 'SEE OTHER' status code and an HTTP header that indicates that the intent // of the Location URI is a flowfile hold, we will store this holdUri. This prevents us // from posting to some other webservice and then attempting to delete some resource to which // we are redirected final int responseCode = response.getStatusLine().getStatusCode(); final String responseReason = response.getStatusLine().getReasonPhrase(); String holdUri = null; if (responseCode == HttpServletResponse.SC_SEE_OTHER) { final Header locationUriHeader = response.getFirstHeader(LOCATION_URI_INTENT_NAME); if (locationUriHeader != null) { if (LOCATION_URI_INTENT_VALUE.equals(locationUriHeader.getValue())) { final Header holdUriHeader = response.getFirstHeader(LOCATION_HEADER_NAME); if (holdUriHeader != null) { holdUri = holdUriHeader.getValue(); } } } if (holdUri == null) { for (FlowFile flowFile : toSend) { flowFile = session.penalize(flowFile); logger.error( "Failed to Post {} to {}: sent content and received status code {}:{} but no Hold URI", new Object[] { flowFile, url, responseCode, responseReason }); session.transfer(flowFile, REL_FAILURE); } return; } } if (holdUri == null) { if (responseCode == HttpServletResponse.SC_SERVICE_UNAVAILABLE) { for (FlowFile flowFile : toSend) { flowFile = session.penalize(flowFile); logger.error( "Failed to Post {} to {}: response code was {}:{}; will yield processing, since the destination is temporarily unavailable", new Object[] { flowFile, url, responseCode, responseReason }); session.transfer(flowFile, REL_FAILURE); } context.yield(); return; } if (responseCode >= 300) { for (FlowFile flowFile : toSend) { flowFile = session.penalize(flowFile); logger.error("Failed to Post {} to {}: response code was {}:{}", new Object[] { flowFile, url, responseCode, responseReason }); session.transfer(flowFile, REL_FAILURE); } return; } logger.info("Successfully Posted {} to {} in {} at a rate of {}", new Object[] { flowFileDescription, url, FormatUtils.formatMinutesSeconds(uploadMillis, TimeUnit.MILLISECONDS), uploadDataRate }); for (final FlowFile flowFile : toSend) { session.getProvenanceReporter().send(flowFile, url, "Remote DN=" + dnHolder.get(), uploadMillis, true); session.transfer(flowFile, REL_SUCCESS); } return; } // // the response indicated a Hold URI; delete the Hold. // // determine the full URI of the Flow File's Hold; Unfortunately, the responses that are returned have // changed over the past, so we have to take into account a few different possibilities. String fullHoldUri = holdUri; if (holdUri.startsWith("/contentListener")) { // If the Hold URI that we get starts with /contentListener, it may not really be /contentListener, // as this really indicates that it should be whatever we posted to -- if posting directly to the // ListenHTTP component, it will be /contentListener, but if posting to a proxy/load balancer, we may // be posting to some other URL. fullHoldUri = url + holdUri.substring(16); } else if (holdUri.startsWith("/")) { // URL indicates the full path but not hostname or port; use the same hostname & port that we posted // to but use the full path indicated by the response. int firstSlash = url.indexOf("/", 8); if (firstSlash < 0) { firstSlash = url.length(); } final String beforeSlash = url.substring(0, firstSlash); fullHoldUri = beforeSlash + holdUri; } else if (!holdUri.startsWith("http")) { // Absolute URL fullHoldUri = url + (url.endsWith("/") ? "" : "/") + holdUri; } final HttpDelete delete = new HttpDelete(fullHoldUri); delete.setHeader(TRANSACTION_ID_HEADER, transactionId); while (true) { try { final HttpResponse holdResponse = client.execute(delete); EntityUtils.consume(holdResponse.getEntity()); final int holdStatusCode = holdResponse.getStatusLine().getStatusCode(); final String holdReason = holdResponse.getStatusLine().getReasonPhrase(); if (holdStatusCode >= 300) { logger.error( "Failed to delete Hold that destination placed on {}: got response code {}:{}; routing to failure", new Object[] { flowFileDescription, holdStatusCode, holdReason }); for (FlowFile flowFile : toSend) { flowFile = session.penalize(flowFile); session.transfer(flowFile, REL_FAILURE); } return; } logger.info("Successfully Posted {} to {} in {} milliseconds at a rate of {}", new Object[] { flowFileDescription, url, uploadMillis, uploadDataRate }); for (FlowFile flowFile : toSend) { session.getProvenanceReporter().send(flowFile, url); session.transfer(flowFile, REL_SUCCESS); } return; } catch (final IOException e) { logger.warn("Failed to delete Hold that destination placed on {} due to {}", new Object[] { flowFileDescription, e }); } if (!isScheduled()) { context.yield(); logger.warn( "Failed to delete Hold that destination placed on {}; Processor has been stopped so routing FlowFile(s) to failure", new Object[] { flowFileDescription }); for (FlowFile flowFile : toSend) { flowFile = session.penalize(flowFile); session.transfer(flowFile, REL_FAILURE); } return; } } }
From source file:com.cognitivemedicine.nifi.http.PostAdvancedHTTP.java
@Override public void onTrigger(final ProcessContext context, final ProcessSession session) { final boolean sendAsFlowFile = context.getProperty(SEND_AS_FLOWFILE).asBoolean(); final int compressionLevel = context.getProperty(COMPRESSION_LEVEL).asInteger(); final String userAgent = context.getProperty(USER_AGENT).getValue(); final RequestConfig.Builder requestConfigBuilder = RequestConfig.custom(); requestConfigBuilder.setConnectionRequestTimeout( context.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue()); requestConfigBuilder.setConnectTimeout( context.getProperty(CONNECTION_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue()); requestConfigBuilder.setRedirectsEnabled(false); requestConfigBuilder/*from w w w. j a v a2 s . com*/ .setSocketTimeout(context.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue()); final RequestConfig requestConfig = requestConfigBuilder.build(); final StreamThrottler throttler = throttlerRef.get(); final ProcessorLog logger = getLogger(); final Double maxBatchBytes = context.getProperty(MAX_BATCH_SIZE).asDataSize(DataUnit.B); String lastUrl = null; long bytesToSend = 0L; final List<FlowFile> toSend = new ArrayList<>(); DestinationAccepts destinationAccepts = null; CloseableHttpClient client = null; final String transactionId = UUID.randomUUID().toString(); final ObjectHolder<String> dnHolder = new ObjectHolder<>("none"); while (true) { FlowFile flowFile = session.get(); if (flowFile == null) { break; } final String url = context.getProperty(URL).evaluateAttributeExpressions(flowFile).getValue(); try { new java.net.URL(url); } catch (final MalformedURLException e) { logger.error( "After substituting attribute values for {}, URL is {}; this is not a valid URL, so routing to failure", new Object[] { flowFile, url }); flowFile = session.penalize(flowFile); session.transfer(flowFile, REL_FAILURE); continue; } // If this FlowFile doesn't have the same url, throw it back on the queue and stop grabbing FlowFiles if (lastUrl != null && !lastUrl.equals(url)) { session.transfer(flowFile); break; } lastUrl = url; toSend.add(flowFile); if (client == null || destinationAccepts == null) { final Config config = getConfig(url, context); final HttpClientConnectionManager conMan = config.getConnectionManager(); final HttpClientBuilder clientBuilder = HttpClientBuilder.create(); clientBuilder.setConnectionManager(conMan); clientBuilder.setUserAgent(userAgent); clientBuilder.addInterceptorFirst(new HttpResponseInterceptor() { @Override public void process(final HttpResponse response, final HttpContext httpContext) throws HttpException, IOException { HttpCoreContext coreContext = HttpCoreContext.adapt(httpContext); ManagedHttpClientConnection conn = coreContext .getConnection(ManagedHttpClientConnection.class); if (!conn.isOpen()) { return; } SSLSession sslSession = conn.getSSLSession(); if (sslSession != null) { final X509Certificate[] certChain = sslSession.getPeerCertificateChain(); if (certChain == null || certChain.length == 0) { throw new SSLPeerUnverifiedException("No certificates found"); } final X509Certificate cert = certChain[0]; dnHolder.set(cert.getSubjectDN().getName().trim()); } } }); clientBuilder.disableAutomaticRetries(); clientBuilder.disableContentCompression(); final String username = context.getProperty(USERNAME).getValue(); final String password = context.getProperty(PASSWORD).getValue(); // set the credentials if appropriate if (username != null) { final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); if (password == null) { credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username)); } else { credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password)); } ; clientBuilder.setDefaultCredentialsProvider(credentialsProvider); } client = clientBuilder.build(); // determine whether or not destination accepts flowfile/gzip destinationAccepts = config.getDestinationAccepts(); if (destinationAccepts == null) { try { if (sendAsFlowFile) { destinationAccepts = getDestinationAcceptance(client, url, getLogger(), transactionId); } else { destinationAccepts = new DestinationAccepts(false, false, false, false, null); } config.setDestinationAccepts(destinationAccepts); } catch (IOException e) { flowFile = session.penalize(flowFile); session.transfer(flowFile, REL_FAILURE); logger.error( "Unable to communicate with destination {} to determine whether or not it can accept flowfiles/gzip; routing {} to failure due to {}", new Object[] { url, flowFile, e }); context.yield(); return; } } } // if we are not sending as flowfile, or if the destination doesn't accept V3 or V2 (streaming) format, // then only use a single FlowFile if (!sendAsFlowFile || (!destinationAccepts.isFlowFileV3Accepted() && !destinationAccepts.isFlowFileV2Accepted())) { break; } bytesToSend += flowFile.getSize(); if (bytesToSend > maxBatchBytes.longValue()) { break; } } if (toSend.isEmpty()) { return; } final String url = lastUrl; final HttpPost post = new HttpPost(url); final List<FlowFile> flowFileList = toSend; final DestinationAccepts accepts = destinationAccepts; final boolean isDestinationLegacyNiFi = accepts.getProtocolVersion() == null; final EntityTemplate entity = new EntityTemplate(new ContentProducer() { @Override public void writeTo(final OutputStream rawOut) throws IOException { final OutputStream throttled = (throttler == null) ? rawOut : throttler.newThrottledOutputStream(rawOut); OutputStream wrappedOut = new BufferedOutputStream(throttled); if (compressionLevel > 0 && accepts.isGzipAccepted()) { wrappedOut = new GZIPOutputStream(wrappedOut, compressionLevel); } try (final OutputStream out = wrappedOut) { for (final FlowFile flowFile : flowFileList) { session.read(flowFile, new InputStreamCallback() { @Override public void process(final InputStream rawIn) throws IOException { try (final InputStream in = new BufferedInputStream(rawIn)) { FlowFilePackager packager = null; if (!sendAsFlowFile) { packager = null; } else if (accepts.isFlowFileV3Accepted()) { packager = new FlowFilePackagerV3(); } else if (accepts.isFlowFileV2Accepted()) { packager = new FlowFilePackagerV2(); } else if (accepts.isFlowFileV1Accepted()) { packager = new FlowFilePackagerV1(); } // if none of the above conditions is met, we should never get here, because // we will have already verified that at least 1 of the FlowFile packaging // formats is acceptable if sending as FlowFile. if (packager == null) { StreamUtils.copy(in, out); } else { final Map<String, String> flowFileAttributes; if (isDestinationLegacyNiFi) { // Old versions of NiFi expect nf.file.name and nf.file.path to indicate filename & path; // in order to maintain backward compatibility, we copy the filename & path to those attribute keys. flowFileAttributes = new HashMap<>(flowFile.getAttributes()); flowFileAttributes.put("nf.file.name", flowFile.getAttribute(CoreAttributes.FILENAME.key())); flowFileAttributes.put("nf.file.path", flowFile.getAttribute(CoreAttributes.PATH.key())); } else { flowFileAttributes = flowFile.getAttributes(); } packager.packageFlowFile(in, out, flowFileAttributes, flowFile.getSize()); } } } }); } out.flush(); } } }); entity.setChunked(context.getProperty(CHUNKED_ENCODING).asBoolean()); post.setEntity(entity); post.setConfig(requestConfig); final String contentType; if (sendAsFlowFile) { if (accepts.isFlowFileV3Accepted()) { contentType = APPLICATION_FLOW_FILE_V3; } else if (accepts.isFlowFileV2Accepted()) { contentType = APPLICATION_FLOW_FILE_V2; } else if (accepts.isFlowFileV1Accepted()) { contentType = APPLICATION_FLOW_FILE_V1; } else { logger.error( "Cannot send data to {} because the destination does not accept FlowFiles and this processor is configured to deliver FlowFiles; rolling back session", new Object[] { url }); session.rollback(); context.yield(); return; } } else { final String attributeValue = toSend.get(0).getAttribute(CoreAttributes.MIME_TYPE.key()); contentType = (attributeValue == null) ? DEFAULT_CONTENT_TYPE : attributeValue; } final String attributeHeaderRegex = context.getProperty(ATTRIBUTES_AS_HEADERS_REGEX).getValue(); if (attributeHeaderRegex != null && !sendAsFlowFile && flowFileList.size() == 1) { final Pattern pattern = Pattern.compile(attributeHeaderRegex); final Map<String, String> attributes = flowFileList.get(0).getAttributes(); for (final Map.Entry<String, String> entry : attributes.entrySet()) { final String key = entry.getKey(); if (pattern.matcher(key).matches()) { post.setHeader(entry.getKey(), entry.getValue()); } } } post.setHeader(CONTENT_TYPE, contentType); post.setHeader(FLOWFILE_CONFIRMATION_HEADER, "true"); post.setHeader(PROTOCOL_VERSION_HEADER, PROTOCOL_VERSION); post.setHeader(TRANSACTION_ID_HEADER, transactionId); if (compressionLevel > 0 && accepts.isGzipAccepted()) { post.setHeader(GZIPPED_HEADER, "true"); } // Do the actual POST final String flowFileDescription = toSend.size() <= 10 ? toSend.toString() : toSend.size() + " FlowFiles"; final String uploadDataRate; final long uploadMillis; String responseContent; CloseableHttpResponse response = null; try { final StopWatch stopWatch = new StopWatch(true); response = client.execute(post); responseContent = EntityUtils.toString(response.getEntity()); stopWatch.stop(); uploadDataRate = stopWatch.calculateDataRate(bytesToSend); uploadMillis = stopWatch.getDuration(TimeUnit.MILLISECONDS); } catch (final IOException e) { logger.error("Failed to Post {} due to {}; transferring to failure", new Object[] { flowFileDescription, e }); context.yield(); for (FlowFile flowFile : toSend) { flowFile = session.penalize(flowFile); session.transfer(flowFile, REL_FAILURE); } return; } finally { if (response != null) { try { response.close(); } catch (IOException e) { getLogger().warn("Failed to close HTTP Response due to {}", new Object[] { e }); } } } // If we get a 'SEE OTHER' status code and an HTTP header that indicates that the intent // of the Location URI is a flowfile hold, we will store this holdUri. This prevents us // from posting to some other webservice and then attempting to delete some resource to which // we are redirected final int responseCode = response.getStatusLine().getStatusCode(); final String responseReason = response.getStatusLine().getReasonPhrase(); String holdUri = null; if (responseCode == HttpServletResponse.SC_SEE_OTHER) { final Header locationUriHeader = response.getFirstHeader(LOCATION_URI_INTENT_NAME); if (locationUriHeader != null) { if (LOCATION_URI_INTENT_VALUE.equals(locationUriHeader.getValue())) { final Header holdUriHeader = response.getFirstHeader(LOCATION_HEADER_NAME); if (holdUriHeader != null) { holdUri = holdUriHeader.getValue(); } } } if (holdUri == null) { for (FlowFile flowFile : toSend) { flowFile = session.penalize(flowFile); logger.error( "Failed to Post {} to {}: sent content and received status code {}:{} but no Hold URI", new Object[] { flowFile, url, responseCode, responseReason }); session.transfer(flowFile, REL_FAILURE); } return; } } if (holdUri == null) { if (responseCode == HttpServletResponse.SC_SERVICE_UNAVAILABLE) { for (FlowFile flowFile : toSend) { flowFile = session.penalize(flowFile); logger.error( "Failed to Post {} to {}: response code was {}:{}; will yield processing, since the destination is temporarily unavailable", new Object[] { flowFile, url, responseCode, responseReason }); session.transfer(flowFile, REL_FAILURE); } context.yield(); return; } if (responseCode >= 300) { for (FlowFile flowFile : toSend) { flowFile = session.penalize(flowFile); logger.error("Failed to Post {} to {}: response code was {}:{}", new Object[] { flowFile, url, responseCode, responseReason }); session.transfer(flowFile, REL_FAILURE); } return; } logger.info("Successfully Posted {} to {} in {} at a rate of {}", new Object[] { flowFileDescription, url, FormatUtils.formatMinutesSeconds(uploadMillis, TimeUnit.MILLISECONDS), uploadDataRate }); for (FlowFile flowFile : toSend) { flowFile = this.setHttpPostResponse(context, session, responseContent, flowFile); session.getProvenanceReporter().send(flowFile, url, "Remote DN=" + dnHolder.get(), uploadMillis, true); session.transfer(flowFile, REL_SUCCESS); } return; } // // the response indicated a Hold URI; delete the Hold. // // determine the full URI of the Flow File's Hold; Unfortunately, the responses that are returned have // changed over the past, so we have to take into account a few different possibilities. String fullHoldUri = holdUri; if (holdUri.startsWith("/contentListener")) { // If the Hold URI that we get starts with /contentListener, it may not really be /contentListener, // as this really indicates that it should be whatever we posted to -- if posting directly to the // ListenHTTP component, it will be /contentListener, but if posting to a proxy/load balancer, we may // be posting to some other URL. fullHoldUri = url + holdUri.substring(16); } else if (holdUri.startsWith("/")) { // URL indicates the full path but not hostname or port; use the same hostname & port that we posted // to but use the full path indicated by the response. int firstSlash = url.indexOf("/", 8); if (firstSlash < 0) { firstSlash = url.length(); } final String beforeSlash = url.substring(0, firstSlash); fullHoldUri = beforeSlash + holdUri; } else if (!holdUri.startsWith("http")) { // Absolute URL fullHoldUri = url + (url.endsWith("/") ? "" : "/") + holdUri; } final HttpDelete delete = new HttpDelete(fullHoldUri); delete.setHeader(TRANSACTION_ID_HEADER, transactionId); while (true) { try { final HttpResponse holdResponse = client.execute(delete); responseContent = EntityUtils.toString(holdResponse.getEntity()); final int holdStatusCode = holdResponse.getStatusLine().getStatusCode(); final String holdReason = holdResponse.getStatusLine().getReasonPhrase(); if (holdStatusCode >= 300) { logger.error( "Failed to delete Hold that destination placed on {}: got response code {}:{}; routing to failure", new Object[] { flowFileDescription, holdStatusCode, holdReason }); for (FlowFile flowFile : toSend) { flowFile = session.penalize(flowFile); session.transfer(flowFile, REL_FAILURE); } return; } logger.info("Successfully Posted {} to {} in {} milliseconds at a rate of {}", new Object[] { flowFileDescription, url, uploadMillis, uploadDataRate }); for (FlowFile flowFile : toSend) { flowFile = this.setHttpPostResponse(context, session, responseContent, flowFile); session.getProvenanceReporter().send(flowFile, url); session.transfer(flowFile, REL_SUCCESS); } return; } catch (final IOException e) { logger.warn("Failed to delete Hold that destination placed on {} due to {}", new Object[] { flowFileDescription, e }); } if (!isScheduled()) { context.yield(); logger.warn( "Failed to delete Hold that destination placed on {}; Processor has been stopped so routing FlowFile(s) to failure", new Object[] { flowFileDescription }); for (FlowFile flowFile : toSend) { flowFile = session.penalize(flowFile); session.transfer(flowFile, REL_FAILURE); } return; } } }