Example usage for org.apache.http.impl.client HttpClientBuilder addInterceptorFirst

List of usage examples for org.apache.http.impl.client HttpClientBuilder addInterceptorFirst

Introduction

In this page you can find the example usage for org.apache.http.impl.client HttpClientBuilder addInterceptorFirst.

Prototype

public final HttpClientBuilder addInterceptorFirst(final HttpRequestInterceptor itcp) 

Source Link

Document

Adds this protocol interceptor to the head of the protocol processing list.

Usage

From source file:de.undercouch.gradle.tasks.download.internal.DefaultHttpClientFactory.java

@Override
public CloseableHttpClient createHttpClient(HttpHost httpHost, boolean acceptAnyCertificate) {
    HttpClientBuilder builder = HttpClientBuilder.create();

    //configure proxy from system environment
    builder.setRoutePlanner(new SystemDefaultRoutePlanner(null));

    //accept any certificate if necessary
    if ("https".equals(httpHost.getSchemeName()) && acceptAnyCertificate) {
        SSLConnectionSocketFactory icsf = getInsecureSSLSocketFactory();
        builder.setSSLSocketFactory(icsf);
        Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create()
                .register("https", icsf).build();
        HttpClientConnectionManager cm = new BasicHttpClientConnectionManager(registry);
        builder.setConnectionManager(cm);
    }//from w  w w .  ja v  a  2s  .c  om

    //add an interceptor that replaces the invalid Content-Type
    //'none' by 'identity'
    builder.addInterceptorFirst(new ContentEncodingNoneInterceptor());

    CloseableHttpClient client = builder.build();
    return client;
}

From source file:hello.MyPostHTTP.java

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {

    final RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
    requestConfigBuilder.setConnectionRequestTimeout(
            context.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
    requestConfigBuilder.setConnectTimeout(
            context.getProperty(CONNECTION_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
    requestConfigBuilder.setRedirectsEnabled(false);
    requestConfigBuilder/*  w  ww.ja v a  2s.  c o  m*/
            .setSocketTimeout(context.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
    final RequestConfig requestConfig = requestConfigBuilder.build();

    final StreamThrottler throttler = throttlerRef.get();
    final ProcessorLog logger = getLogger();

    String lastUrl = null;
    long bytesToSend = 0L;

    final List<FlowFile> toSend = new ArrayList<>();
    CloseableHttpClient client = null;
    final String transactionId = UUID.randomUUID().toString();

    final ObjectHolder<String> dnHolder = new ObjectHolder<>("none");
    while (true) {
        FlowFile flowFile = session.get();
        if (flowFile == null) {
            break;
        }

        final String url = context.getProperty(URL).evaluateAttributeExpressions(flowFile).getValue();
        try {
            new java.net.URL(url);
        } catch (final MalformedURLException e) {
            logger.error(
                    "After substituting attribute values for {}, URL is {}; this is not a valid URL, so routing to failure",
                    new Object[] { flowFile, url });
            flowFile = session.penalize(flowFile);
            session.transfer(flowFile, REL_FAILURE);
            continue;
        }

        // If this FlowFile doesn't have the same url, throw it back on the queue and stop grabbing FlowFiles
        if (lastUrl != null && !lastUrl.equals(url)) {
            session.transfer(flowFile);
            break;
        }

        lastUrl = url;
        toSend.add(flowFile);

        if (client == null) {
            final Config config = getConfig(url, context);
            final HttpClientConnectionManager conMan = config.getConnectionManager();

            final HttpClientBuilder clientBuilder = HttpClientBuilder.create();
            clientBuilder.setConnectionManager(conMan);
            clientBuilder.addInterceptorFirst(new HttpResponseInterceptor() {
                @Override
                public void process(final HttpResponse response, final HttpContext httpContext)
                        throws HttpException, IOException {
                    final HttpCoreContext coreContext = HttpCoreContext.adapt(httpContext);
                    final ManagedHttpClientConnection conn = coreContext
                            .getConnection(ManagedHttpClientConnection.class);
                    if (!conn.isOpen()) {
                        return;
                    }

                    final SSLSession sslSession = conn.getSSLSession();

                    if (sslSession != null) {
                        final X509Certificate[] certChain = sslSession.getPeerCertificateChain();
                        if (certChain == null || certChain.length == 0) {
                            throw new SSLPeerUnverifiedException("No certificates found");
                        }

                        final X509Certificate cert = certChain[0];
                        dnHolder.set(cert.getSubjectDN().getName().trim());
                    }
                }
            });

            clientBuilder.disableAutomaticRetries();
            clientBuilder.disableContentCompression();

            client = clientBuilder.build();
        }

        bytesToSend += flowFile.getSize();
        break;
    }

    if (toSend.isEmpty()) {
        return;
    }

    final String url = lastUrl;
    final HttpPost post = new HttpPost(url);
    final List<FlowFile> flowFileList = toSend;

    String userName = "Chris";
    String password = "password";
    final MultipartEntityBuilder builder = MultipartEntityBuilder.create();
    builder.addTextBody("userName", userName);
    builder.addTextBody("password", password);
    for (final FlowFile flowFile : flowFileList) {
        session.read(flowFile, new InputStreamCallback() {
            @Override
            public void process(final InputStream rawIn) throws IOException {
                InputStream in = new ByteArrayInputStream(IOUtils.toByteArray(rawIn));
                builder.addBinaryBody("file", in, ContentType.DEFAULT_BINARY, "filename");
            }
        });
    }

    final HttpEntity entity2 = builder.build();

    post.setEntity(entity2);
    post.setConfig(requestConfig);

    final String contentType;

    contentType = DEFAULT_CONTENT_TYPE;
    post.setHeader(CONTENT_TYPE_HEADER, contentType);
    post.setHeader(FLOWFILE_CONFIRMATION_HEADER, "true");
    post.setHeader(PROTOCOL_VERSION_HEADER, PROTOCOL_VERSION);
    post.setHeader(TRANSACTION_ID_HEADER, transactionId);

    // Do the actual POST
    final String flowFileDescription = toSend.size() <= 10 ? toSend.toString() : toSend.size() + " FlowFiles";

    final String uploadDataRate;
    final long uploadMillis;
    CloseableHttpResponse response = null;
    try {
        final StopWatch stopWatch = new StopWatch(true);
        response = client.execute(post);
        // consume input stream entirely, ignoring its contents. If we
        // don't do this, the Connection will not be returned to the pool
        EntityUtils.consume(response.getEntity());
        stopWatch.stop();
        uploadDataRate = stopWatch.calculateDataRate(bytesToSend);
        uploadMillis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
    } catch (final IOException e) {
        logger.error("Failed to Post {} due to {}; transferring to failure",
                new Object[] { flowFileDescription, e });
        context.yield();
        for (FlowFile flowFile : toSend) {
            flowFile = session.penalize(flowFile);
            session.transfer(flowFile, REL_FAILURE);
        }
        return;
    } finally {
        if (response != null) {
            try {
                response.close();
            } catch (final IOException e) {
                getLogger().warn("Failed to close HTTP Response due to {}", new Object[] { e });
            }
        }
    }

    // If we get a 'SEE OTHER' status code and an HTTP header that indicates that the intent
    // of the Location URI is a flowfile hold, we will store this holdUri. This prevents us
    // from posting to some other webservice and then attempting to delete some resource to which
    // we are redirected
    final int responseCode = response.getStatusLine().getStatusCode();
    final String responseReason = response.getStatusLine().getReasonPhrase();
    String holdUri = null;
    if (responseCode == HttpServletResponse.SC_SEE_OTHER) {
        final Header locationUriHeader = response.getFirstHeader(LOCATION_URI_INTENT_NAME);
        if (locationUriHeader != null) {
            if (LOCATION_URI_INTENT_VALUE.equals(locationUriHeader.getValue())) {
                final Header holdUriHeader = response.getFirstHeader(LOCATION_HEADER_NAME);
                if (holdUriHeader != null) {
                    holdUri = holdUriHeader.getValue();
                }
            }
        }

        if (holdUri == null) {
            for (FlowFile flowFile : toSend) {
                flowFile = session.penalize(flowFile);
                logger.error(
                        "Failed to Post {} to {}: sent content and received status code {}:{} but no Hold URI",
                        new Object[] { flowFile, url, responseCode, responseReason });
                session.transfer(flowFile, REL_FAILURE);
            }
            return;
        }
    }

    if (holdUri == null) {
        if (responseCode == HttpServletResponse.SC_SERVICE_UNAVAILABLE) {
            for (FlowFile flowFile : toSend) {
                flowFile = session.penalize(flowFile);
                logger.error(
                        "Failed to Post {} to {}: response code was {}:{}; will yield processing, "
                                + "since the destination is temporarily unavailable",
                        new Object[] { flowFile, url, responseCode, responseReason });
                session.transfer(flowFile, REL_FAILURE);
            }
            context.yield();
            return;
        }

        if (responseCode >= 300) {
            for (FlowFile flowFile : toSend) {
                flowFile = session.penalize(flowFile);
                logger.error("Failed to Post {} to {}: response code was {}:{}",
                        new Object[] { flowFile, url, responseCode, responseReason });
                session.transfer(flowFile, REL_FAILURE);
            }
            return;
        }

        logger.info("Successfully Posted {} to {} in {} at a rate of {}", new Object[] { flowFileDescription,
                url, FormatUtils.formatMinutesSeconds(uploadMillis, TimeUnit.MILLISECONDS), uploadDataRate });

        for (final FlowFile flowFile : toSend) {
            session.getProvenanceReporter().send(flowFile, url, "Remote DN=" + dnHolder.get(), uploadMillis,
                    true);
            session.transfer(flowFile, REL_SUCCESS);
        }
        return;
    }

    //
    // the response indicated a Hold URI; delete the Hold.
    //
    // determine the full URI of the Flow File's Hold; Unfortunately, the responses that are returned have
    // changed over the past, so we have to take into account a few different possibilities.
    String fullHoldUri = holdUri;
    if (holdUri.startsWith("/contentListener")) {
        // If the Hold URI that we get starts with /contentListener, it may not really be /contentListener,
        // as this really indicates that it should be whatever we posted to -- if posting directly to the
        // ListenHTTP component, it will be /contentListener, but if posting to a proxy/load balancer, we may
        // be posting to some other URL.
        fullHoldUri = url + holdUri.substring(16);
    } else if (holdUri.startsWith("/")) {
        // URL indicates the full path but not hostname or port; use the same hostname & port that we posted
        // to but use the full path indicated by the response.
        int firstSlash = url.indexOf("/", 8);
        if (firstSlash < 0) {
            firstSlash = url.length();
        }
        final String beforeSlash = url.substring(0, firstSlash);
        fullHoldUri = beforeSlash + holdUri;
    } else if (!holdUri.startsWith("http")) {
        // Absolute URL
        fullHoldUri = url + (url.endsWith("/") ? "" : "/") + holdUri;
    }

    final HttpDelete delete = new HttpDelete(fullHoldUri);
    delete.setHeader(TRANSACTION_ID_HEADER, transactionId);

    while (true) {
        try {
            final HttpResponse holdResponse = client.execute(delete);
            EntityUtils.consume(holdResponse.getEntity());
            final int holdStatusCode = holdResponse.getStatusLine().getStatusCode();
            final String holdReason = holdResponse.getStatusLine().getReasonPhrase();
            if (holdStatusCode >= 300) {
                logger.error(
                        "Failed to delete Hold that destination placed on {}: got response code {}:{}; routing to failure",
                        new Object[] { flowFileDescription, holdStatusCode, holdReason });

                for (FlowFile flowFile : toSend) {
                    flowFile = session.penalize(flowFile);
                    session.transfer(flowFile, REL_FAILURE);
                }
                return;
            }

            logger.info("Successfully Posted {} to {} in {} milliseconds at a rate of {}",
                    new Object[] { flowFileDescription, url, uploadMillis, uploadDataRate });

            for (final FlowFile flowFile : toSend) {
                session.getProvenanceReporter().send(flowFile, url);
                session.transfer(flowFile, REL_SUCCESS);
            }
            return;
        } catch (final IOException e) {
            logger.warn("Failed to delete Hold that destination placed on {} due to {}",
                    new Object[] { flowFileDescription, e });
        }

        if (!isScheduled()) {
            context.yield();
            logger.warn(
                    "Failed to delete Hold that destination placed on {}; Processor has been stopped so routing FlowFile(s) to failure",
                    new Object[] { flowFileDescription });
            for (FlowFile flowFile : toSend) {
                flowFile = session.penalize(flowFile);
                session.transfer(flowFile, REL_FAILURE);
            }
            return;
        }
    }
}

From source file:com.ngdata.hbaseindexer.indexer.FusionPipelineClient.java

public FusionPipelineClient(String endpointUrl, String fusionUser, String fusionPass, String fusionRealm)
        throws MalformedURLException {

    this.fusionUser = fusionUser;
    this.fusionPass = fusionPass;
    this.fusionRealm = fusionRealm;

    String fusionLoginConf = System.getProperty(FusionKrb5HttpClientConfigurer.LOGIN_CONFIG_PROP);
    if (fusionLoginConf != null && !fusionLoginConf.isEmpty()) {
        httpClient = FusionKrb5HttpClientConfigurer.createClient(fusionUser);
        isKerberos = true;/*www .  ja  va 2s.c o  m*/
    } else {
        globalConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.BEST_MATCH).build();
        cookieStore = new BasicCookieStore();

        // build the HttpClient to be used for all requests
        HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();
        httpClientBuilder.setDefaultRequestConfig(globalConfig).setDefaultCookieStore(cookieStore);
        httpClientBuilder.setMaxConnPerRoute(100);
        httpClientBuilder.setMaxConnTotal(500);

        if (fusionUser != null && fusionRealm == null)
            httpClientBuilder.addInterceptorFirst(new PreEmptiveBasicAuthenticator(fusionUser, fusionPass));

        httpClient = httpClientBuilder.build();
    }

    originalEndpoints = Arrays.asList(endpointUrl.split(","));
    try {
        sessions = establishSessions(originalEndpoints, fusionUser, fusionPass, fusionRealm);
    } catch (Exception exc) {
        if (exc instanceof RuntimeException) {
            throw (RuntimeException) exc;
        } else {
            throw new RuntimeException(exc);
        }
    }

    random = new Random();
    jsonObjectMapper = new ObjectMapper();

    requestCounter = new AtomicInteger(0);
}

From source file:sk.datalan.solr.impl.HttpClientUtil.java

public static HttpClientBuilder configureClient(final HttpClientConfiguration config) {
    HttpClientBuilder clientBuilder = HttpClientBuilder.create();

    // max total connections
    if (config.isSetMaxConnections()) {
        clientBuilder.setMaxConnTotal(config.getMaxConnections());
    }/*from w w w  . ja v  a2 s.co  m*/

    // max connections per route
    if (config.isSetMaxConnectionsPerRoute()) {
        clientBuilder.setMaxConnPerRoute(config.getMaxConnectionsPerRoute());
    }

    RequestConfig.Builder requestConfigBuilder = RequestConfig.custom().setCookieSpec(CookieSpecs.BEST_MATCH)
            .setExpectContinueEnabled(true).setStaleConnectionCheckEnabled(true);

    // connection timeout
    if (config.isSetConnectionTimeout()) {
        requestConfigBuilder.setConnectTimeout(config.getConnectionTimeout());
    }

    // soucket timeout
    if (config.isSetSocketTimeout()) {
        requestConfigBuilder.setSocketTimeout(config.getSocketTimeout());
    }

    // soucket timeout
    if (config.isSetFollowRedirects()) {
        requestConfigBuilder.setRedirectsEnabled(config.getFollowRedirects());
    }
    clientBuilder.setDefaultRequestConfig(requestConfigBuilder.build());

    if (config.isSetUseRetry()) {
        if (config.getUseRetry()) {
            clientBuilder.setRetryHandler(new DefaultHttpRequestRetryHandler());
        } else {
            clientBuilder.setRetryHandler(NO_RETRY);
        }
    }

    // basic authentication
    if (config.isSetBasicAuthUsername() && config.isSetBasicAuthPassword()) {
        CredentialsProvider credsProvider = new BasicCredentialsProvider();
        credsProvider.setCredentials(AuthScope.ANY,
                new UsernamePasswordCredentials(config.getBasicAuthUsername(), config.getBasicAuthPassword()));
    }

    if (config.isSetAllowCompression()) {
        clientBuilder.addInterceptorFirst(new UseCompressionRequestInterceptor());
        clientBuilder.addInterceptorFirst(new UseCompressionResponseInterceptor());
    }

    // SSL context for secure connections can be created either based on
    // system or application specific properties.
    SSLContext sslcontext = SSLContexts.createSystemDefault();
    // Use custom hostname verifier to customize SSL hostname verification.
    X509HostnameVerifier hostnameVerifier = new BrowserCompatHostnameVerifier();

    // Create a registry of custom connection socket factories for supported
    // protocol schemes.
    Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
            .register("http", PlainConnectionSocketFactory.INSTANCE)
            .register("https", new SSLConnectionSocketFactory(sslcontext, hostnameVerifier)).build();

    clientBuilder.setConnectionManager(new PoolingHttpClientConnectionManager(socketFactoryRegistry));

    return clientBuilder;
}

From source file:com.jive.myco.seyren.core.util.graphite.GraphiteHttpClient.java

private HttpClient createHttpClient() {
    HttpClientBuilder clientBuilder = HttpClientBuilder.create().setConnectionManager(createConnectionManager())
            .setDefaultRequestConfig(RequestConfig.custom()
                    .setConnectionRequestTimeout(graphiteConnectionRequestTimeout)
                    .setConnectTimeout(graphiteConnectTimeout).setSocketTimeout(graphiteSocketTimeout).build());

    // Set auth header for graphite if username and password are provided
    if (!StringUtils.isEmpty(graphiteUsername) && !StringUtils.isEmpty(graphitePassword)) {
        CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
        credentialsProvider.setCredentials(new AuthScope(AuthScope.ANY_HOST, AuthScope.ANY_PORT),
                new UsernamePasswordCredentials(graphiteUsername, graphitePassword));
        clientBuilder.setDefaultCredentialsProvider(credentialsProvider);
        context.setAttribute("preemptive-auth", new BasicScheme());
        clientBuilder.addInterceptorFirst(new PreemptiveAuth());
    }/*from w  w  w  .j  a v  a2  s  . com*/

    return clientBuilder.build();
}

From source file:de.tudarmstadt.ukp.shibhttpclient.ShibHttpClient.java

/**
 * Create a new client (with an explicit proxy and possibly transparent authentication)
 * /*from   w  ww .  j  a v  a2  s.c om*/
 * @param aIdpUrl
 *            the URL of the IdP. Should probably be something ending in "/SAML2/SOAP/ECP"
 * @param aUsername
 *            the user name to log into the IdP.
 * @param aPassword
 *            the password to log in to the IdP.
 * @param aProxy
 *            if not {@code null}, use this proxy instead of the default system proxy (if any)
 * @param anyCert
 *            if {@code true}, accept any certificate from any remote host. Otherwise,
 *            certificates need to be installed in the JRE.
 * @param transparentAuth
 *            if {@code true} (default), add a HttpRequestPostProcessor to transparently 
 *            authenticate. Otherwise, you must handle the authentication process yourself.
 */
public ShibHttpClient(String aIdpUrl, String aUsername, String aPassword, HttpHost aProxy, boolean anyCert,
        boolean transparentAuth) {

    setIdpUrl(aIdpUrl);
    setUsername(aUsername);
    setPassword(aPassword);

    // Use a pooling connection manager, because we'll have to do a call out to the IdP
    // while still being in a connection with the SP
    PoolingHttpClientConnectionManager connMgr;
    if (anyCert) {
        try {
            SSLContextBuilder builder = new SSLContextBuilder();
            builder.loadTrustMaterial(null, new TrustSelfSignedStrategy());
            Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder
                    .<ConnectionSocketFactory>create().register("http", new PlainConnectionSocketFactory())
                    .register("https", new SSLConnectionSocketFactory(builder.build(),
                            SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER))
                    .build();
            connMgr = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
        } catch (GeneralSecurityException e) {
            // There shouldn't be any of these exceptions, because we do not use an actual
            // keystore
            throw new IllegalStateException(e);
        }
    } else {
        connMgr = new PoolingHttpClientConnectionManager();
    }
    connMgr.setMaxTotal(10);
    connMgr.setDefaultMaxPerRoute(5);

    // The client needs to remember the auth cookie
    cookieStore = new BasicCookieStore();
    RequestConfig globalRequestConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.BROWSER_COMPATIBILITY)
            .build();

    // Let's throw all common client elements into one builder object
    HttpClientBuilder customClient = HttpClients.custom().setConnectionManager(connMgr)
            // The client needs to remember the auth cookie
            .setDefaultRequestConfig(globalRequestConfig).setDefaultCookieStore(cookieStore)
            // Add the ECP/PAOS headers - needs to be added first so the cookie we get from
            // the authentication can be handled by the RequestAddCookies interceptor later
            .addInterceptorFirst(new HttpRequestPreprocessor());

    // Automatically log into IdP if transparent Shibboleth authentication handling is requested (default)
    if (transparentAuth) {
        customClient = customClient.addInterceptorFirst(new HttpRequestPostprocessor());
    }

    // Build the client with/without proxy settings 
    if (aProxy == null) {
        // use the proxy settings of the JVM, if specified 
        client = customClient.setRoutePlanner(new SystemDefaultRoutePlanner(ProxySelector.getDefault()))
                .build();
    } else {
        // use the explicit proxy
        client = customClient.setProxy(aProxy).build();
    }

    parserPool = new BasicParserPool();
    parserPool.setNamespaceAware(true);
}

From source file:com.seyren.core.util.graphite.GraphiteHttpClient.java

private HttpClient createHttpClient() {
    HttpClientBuilder clientBuilder = HttpClientBuilder.create().useSystemProperties()
            .setConnectionManager(createConnectionManager())
            .setDefaultRequestConfig(RequestConfig.custom()
                    .setConnectionRequestTimeout(graphiteConnectionRequestTimeout)
                    .setConnectTimeout(graphiteConnectTimeout).setSocketTimeout(graphiteSocketTimeout).build());

    // Set auth header for graphite if username and password are provided
    if (!StringUtils.isEmpty(graphiteUsername) && !StringUtils.isEmpty(graphitePassword)) {
        CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
        credentialsProvider.setCredentials(new AuthScope(AuthScope.ANY_HOST, AuthScope.ANY_PORT),
                new UsernamePasswordCredentials(graphiteUsername, graphitePassword));
        clientBuilder.setDefaultCredentialsProvider(credentialsProvider);
        context.setAttribute("preemptive-auth", new BasicScheme());
        clientBuilder.addInterceptorFirst(new PreemptiveAuth());
    }/*from ww w  .  j  a v a  2 s. co m*/

    return clientBuilder.build();
}

From source file:info.bonjean.beluga.connection.BelugaHTTPClient.java

private BelugaHTTPClient() {
    BelugaConfiguration configuration = BelugaConfiguration.getInstance();
    HttpClientBuilder clientBuilder = HttpClients.custom();

    // timeout/*www  .  ja v  a  2 s.c om*/
    RequestConfig config = RequestConfig.custom().setConnectTimeout(TIMEOUT).setSocketTimeout(TIMEOUT)
            .setConnectionRequestTimeout(TIMEOUT).build();
    clientBuilder.setDefaultRequestConfig(config);

    switch (configuration.getConnectionType()) {
    case PROXY_DNS:
        Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create()
                .register("http", PlainConnectionSocketFactory.getSocketFactory())
                .register("https", SSLConnectionSocketFactory.getSocketFactory()).build();
        BelugaDNSResolver dnsOverrider = new BelugaDNSResolver(DNSProxy.PROXY_DNS);
        connectionManager = new PoolingHttpClientConnectionManager(registry, dnsOverrider);
        break;
    case HTTP_PROXY:
        HttpHost proxy = new HttpHost(configuration.getProxyHost(), configuration.getProxyPort(), "http");
        clientBuilder.setProxy(proxy);
        break;
    default:
    }

    // limit the pool size
    connectionManager.setDefaultMaxPerRoute(2);

    // add interceptor, currently for debugging only
    clientBuilder.addInterceptorFirst(new HttpResponseInterceptor() {
        @Override
        public void process(HttpResponse response, HttpContext context) throws HttpException, IOException {
            HttpInetConnection connection = (HttpInetConnection) context
                    .getAttribute(HttpCoreContext.HTTP_CONNECTION);
            log.debug("Remote address: " + connection.getRemoteAddress());
            // TODO: reimplement blacklisting for DNS proxy by maintaining a
            // map [DNS IP,RESOLVED IP] in the DNS resolver for reverse
            // lookup
        }
    });

    // finally create the HTTP client
    clientBuilder.setConnectionManager(connectionManager);
    httpClient = clientBuilder.build();
}

From source file:com.cognitivemedicine.nifi.http.PostHTTP2.java

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    final boolean sendAsFlowFile = context.getProperty(SEND_AS_FLOWFILE).asBoolean();
    final int compressionLevel = context.getProperty(COMPRESSION_LEVEL).asInteger();
    final String userAgent = context.getProperty(USER_AGENT).getValue();

    final RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
    requestConfigBuilder.setConnectionRequestTimeout(
            context.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
    requestConfigBuilder.setConnectTimeout(
            context.getProperty(CONNECTION_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
    requestConfigBuilder.setRedirectsEnabled(false);
    requestConfigBuilder//from   ww  w.j  av a  2s .  c o m
            .setSocketTimeout(context.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
    final RequestConfig requestConfig = requestConfigBuilder.build();

    final StreamThrottler throttler = throttlerRef.get();
    final ProcessorLog logger = getLogger();

    final Double maxBatchBytes = context.getProperty(MAX_BATCH_SIZE).asDataSize(DataUnit.B);
    String lastUrl = null;
    long bytesToSend = 0L;

    final List<FlowFile> toSend = new ArrayList<>();
    DestinationAccepts destinationAccepts = null;
    CloseableHttpClient client = null;
    final String transactionId = UUID.randomUUID().toString();

    final ObjectHolder<String> dnHolder = new ObjectHolder<>("none");
    while (true) {
        FlowFile flowFile = session.get();
        if (flowFile == null) {
            break;
        }

        final String url = context.getProperty(URL).evaluateAttributeExpressions(flowFile).getValue();
        try {
            new java.net.URL(url);
        } catch (final MalformedURLException e) {
            logger.error(
                    "After substituting attribute values for {}, URL is {}; this is not a valid URL, so routing to failure",
                    new Object[] { flowFile, url });
            flowFile = session.penalize(flowFile);
            session.transfer(flowFile, REL_FAILURE);
            continue;
        }

        // If this FlowFile doesn't have the same url, throw it back on the queue and stop grabbing FlowFiles
        if (lastUrl != null && !lastUrl.equals(url)) {
            session.transfer(flowFile);
            break;
        }

        lastUrl = url;
        toSend.add(flowFile);

        if (client == null || destinationAccepts == null) {
            final Config config = getConfig(url, context);
            final HttpClientConnectionManager conMan = config.getConnectionManager();

            final HttpClientBuilder clientBuilder = HttpClientBuilder.create();
            clientBuilder.setConnectionManager(conMan);
            clientBuilder.setUserAgent(userAgent);
            clientBuilder.addInterceptorFirst(new HttpResponseInterceptor() {
                @Override
                public void process(final HttpResponse response, final HttpContext httpContext)
                        throws HttpException, IOException {
                    HttpCoreContext coreContext = HttpCoreContext.adapt(httpContext);
                    ManagedHttpClientConnection conn = coreContext
                            .getConnection(ManagedHttpClientConnection.class);
                    if (!conn.isOpen()) {
                        return;
                    }

                    SSLSession sslSession = conn.getSSLSession();

                    if (sslSession != null) {
                        final X509Certificate[] certChain = sslSession.getPeerCertificateChain();
                        if (certChain == null || certChain.length == 0) {
                            throw new SSLPeerUnverifiedException("No certificates found");
                        }

                        final X509Certificate cert = certChain[0];
                        dnHolder.set(cert.getSubjectDN().getName().trim());
                    }
                }
            });

            clientBuilder.disableAutomaticRetries();
            clientBuilder.disableContentCompression();

            final String username = context.getProperty(USERNAME).getValue();
            final String password = context.getProperty(PASSWORD).getValue();
            // set the credentials if appropriate
            if (username != null) {
                final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
                if (password == null) {
                    credentialsProvider.setCredentials(AuthScope.ANY,
                            new UsernamePasswordCredentials(username));
                } else {
                    credentialsProvider.setCredentials(AuthScope.ANY,
                            new UsernamePasswordCredentials(username, password));
                }
                ;
                clientBuilder.setDefaultCredentialsProvider(credentialsProvider);
            }
            client = clientBuilder.build();

            // determine whether or not destination accepts flowfile/gzip
            destinationAccepts = config.getDestinationAccepts();
            if (destinationAccepts == null) {
                try {
                    if (sendAsFlowFile) {
                        destinationAccepts = getDestinationAcceptance(client, url, getLogger(), transactionId);
                    } else {
                        destinationAccepts = new DestinationAccepts(false, false, false, false, null);
                    }

                    config.setDestinationAccepts(destinationAccepts);
                } catch (IOException e) {
                    flowFile = session.penalize(flowFile);
                    session.transfer(flowFile, REL_FAILURE);
                    logger.error(
                            "Unable to communicate with destination {} to determine whether or not it can accept flowfiles/gzip; routing {} to failure due to {}",
                            new Object[] { url, flowFile, e });
                    context.yield();
                    return;
                }
            }
        }

        // if we are not sending as flowfile, or if the destination doesn't accept V3 or V2 (streaming) format,
        // then only use a single FlowFile
        if (!sendAsFlowFile
                || (!destinationAccepts.isFlowFileV3Accepted() && !destinationAccepts.isFlowFileV2Accepted())) {
            break;
        }

        bytesToSend += flowFile.getSize();
        if (bytesToSend > maxBatchBytes.longValue()) {
            break;
        }
    }

    if (toSend.isEmpty()) {
        return;
    }

    final String url = lastUrl;
    final HttpPost post = new HttpPost(url);
    final List<FlowFile> flowFileList = toSend;
    final DestinationAccepts accepts = destinationAccepts;
    final boolean isDestinationLegacyNiFi = accepts.getProtocolVersion() == null;

    final EntityTemplate entity = new EntityTemplate(new ContentProducer() {
        @Override
        public void writeTo(final OutputStream rawOut) throws IOException {
            final OutputStream throttled = (throttler == null) ? rawOut
                    : throttler.newThrottledOutputStream(rawOut);
            OutputStream wrappedOut = new BufferedOutputStream(throttled);
            if (compressionLevel > 0 && accepts.isGzipAccepted()) {
                wrappedOut = new GZIPOutputStream(wrappedOut, compressionLevel);
            }

            try (final OutputStream out = wrappedOut) {
                for (final FlowFile flowFile : flowFileList) {
                    session.read(flowFile, new InputStreamCallback() {
                        @Override
                        public void process(final InputStream rawIn) throws IOException {
                            try (final InputStream in = new BufferedInputStream(rawIn)) {

                                FlowFilePackager packager = null;
                                if (!sendAsFlowFile) {
                                    packager = null;
                                } else if (accepts.isFlowFileV3Accepted()) {
                                    packager = new FlowFilePackagerV3();
                                } else if (accepts.isFlowFileV2Accepted()) {
                                    packager = new FlowFilePackagerV2();
                                } else if (accepts.isFlowFileV1Accepted()) {
                                    packager = new FlowFilePackagerV1();
                                }

                                // if none of the above conditions is met, we should never get here, because
                                // we will have already verified that at least 1 of the FlowFile packaging
                                // formats is acceptable if sending as FlowFile.
                                if (packager == null) {
                                    StreamUtils.copy(in, out);
                                } else {
                                    final Map<String, String> flowFileAttributes;
                                    if (isDestinationLegacyNiFi) {
                                        // Old versions of NiFi expect nf.file.name and nf.file.path to indicate filename & path;
                                        // in order to maintain backward compatibility, we copy the filename & path to those attribute keys.
                                        flowFileAttributes = new HashMap<>(flowFile.getAttributes());
                                        flowFileAttributes.put("nf.file.name",
                                                flowFile.getAttribute(CoreAttributes.FILENAME.key()));
                                        flowFileAttributes.put("nf.file.path",
                                                flowFile.getAttribute(CoreAttributes.PATH.key()));
                                    } else {
                                        flowFileAttributes = flowFile.getAttributes();
                                    }

                                    packager.packageFlowFile(in, out, flowFileAttributes, flowFile.getSize());
                                }
                            }
                        }
                    });
                }

                out.flush();
            }
        }
    });

    entity.setChunked(context.getProperty(CHUNKED_ENCODING).asBoolean());
    post.setEntity(entity);
    post.setConfig(requestConfig);

    final String contentType;
    if (sendAsFlowFile) {
        if (accepts.isFlowFileV3Accepted()) {
            contentType = APPLICATION_FLOW_FILE_V3;
        } else if (accepts.isFlowFileV2Accepted()) {
            contentType = APPLICATION_FLOW_FILE_V2;
        } else if (accepts.isFlowFileV1Accepted()) {
            contentType = APPLICATION_FLOW_FILE_V1;
        } else {
            logger.error(
                    "Cannot send data to {} because the destination does not accept FlowFiles and this processor is configured to deliver FlowFiles; rolling back session",
                    new Object[] { url });
            session.rollback();
            context.yield();
            return;
        }
    } else {
        final String attributeValue = toSend.get(0).getAttribute(CoreAttributes.MIME_TYPE.key());
        contentType = (attributeValue == null) ? DEFAULT_CONTENT_TYPE : attributeValue;
    }

    final String attributeHeaderRegex = context.getProperty(ATTRIBUTES_AS_HEADERS_REGEX).getValue();
    if (attributeHeaderRegex != null && !sendAsFlowFile && flowFileList.size() == 1) {
        final Pattern pattern = Pattern.compile(attributeHeaderRegex);

        final Map<String, String> attributes = flowFileList.get(0).getAttributes();
        for (final Map.Entry<String, String> entry : attributes.entrySet()) {
            final String key = entry.getKey();
            if (pattern.matcher(key).matches()) {
                post.setHeader(entry.getKey(), entry.getValue());
            }
        }
    }

    post.setHeader(CONTENT_TYPE, contentType);
    post.setHeader(FLOWFILE_CONFIRMATION_HEADER, "true");
    post.setHeader(PROTOCOL_VERSION_HEADER, PROTOCOL_VERSION);
    post.setHeader(TRANSACTION_ID_HEADER, transactionId);
    if (compressionLevel > 0 && accepts.isGzipAccepted()) {
        post.setHeader(GZIPPED_HEADER, "true");
    }

    // Do the actual POST
    final String flowFileDescription = toSend.size() <= 10 ? toSend.toString() : toSend.size() + " FlowFiles";

    final String uploadDataRate;
    final long uploadMillis;
    CloseableHttpResponse response = null;
    try {
        final StopWatch stopWatch = new StopWatch(true);
        response = client.execute(post);

        // consume input stream entirely, ignoring its contents. If we
        // don't do this, the Connection will not be returned to the pool
        EntityUtils.consume(response.getEntity());
        stopWatch.stop();
        uploadDataRate = stopWatch.calculateDataRate(bytesToSend);
        uploadMillis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
    } catch (final IOException e) {
        logger.error("Failed to Post {} due to {}; transferring to failure",
                new Object[] { flowFileDescription, e });
        context.yield();
        for (FlowFile flowFile : toSend) {
            flowFile = session.penalize(flowFile);
            session.transfer(flowFile, REL_FAILURE);
        }
        return;
    } finally {
        if (response != null) {
            try {
                response.close();
            } catch (IOException e) {
                getLogger().warn("Failed to close HTTP Response due to {}", new Object[] { e });
            }
        }
    }

    // If we get a 'SEE OTHER' status code and an HTTP header that indicates that the intent
    // of the Location URI is a flowfile hold, we will store this holdUri. This prevents us
    // from posting to some other webservice and then attempting to delete some resource to which
    // we are redirected
    final int responseCode = response.getStatusLine().getStatusCode();
    final String responseReason = response.getStatusLine().getReasonPhrase();
    String holdUri = null;
    if (responseCode == HttpServletResponse.SC_SEE_OTHER) {
        final Header locationUriHeader = response.getFirstHeader(LOCATION_URI_INTENT_NAME);
        if (locationUriHeader != null) {
            if (LOCATION_URI_INTENT_VALUE.equals(locationUriHeader.getValue())) {
                final Header holdUriHeader = response.getFirstHeader(LOCATION_HEADER_NAME);
                if (holdUriHeader != null) {
                    holdUri = holdUriHeader.getValue();
                }
            }
        }

        if (holdUri == null) {
            for (FlowFile flowFile : toSend) {
                flowFile = session.penalize(flowFile);
                logger.error(
                        "Failed to Post {} to {}: sent content and received status code {}:{} but no Hold URI",
                        new Object[] { flowFile, url, responseCode, responseReason });
                session.transfer(flowFile, REL_FAILURE);
            }
            return;
        }
    }

    if (holdUri == null) {
        if (responseCode == HttpServletResponse.SC_SERVICE_UNAVAILABLE) {
            for (FlowFile flowFile : toSend) {
                flowFile = session.penalize(flowFile);
                logger.error(
                        "Failed to Post {} to {}: response code was {}:{}; will yield processing, since the destination is temporarily unavailable",
                        new Object[] { flowFile, url, responseCode, responseReason });
                session.transfer(flowFile, REL_FAILURE);
            }
            context.yield();
            return;
        }

        if (responseCode >= 300) {
            for (FlowFile flowFile : toSend) {
                flowFile = session.penalize(flowFile);
                logger.error("Failed to Post {} to {}: response code was {}:{}",
                        new Object[] { flowFile, url, responseCode, responseReason });
                session.transfer(flowFile, REL_FAILURE);
            }
            return;
        }

        logger.info("Successfully Posted {} to {} in {} at a rate of {}", new Object[] { flowFileDescription,
                url, FormatUtils.formatMinutesSeconds(uploadMillis, TimeUnit.MILLISECONDS), uploadDataRate });

        for (final FlowFile flowFile : toSend) {
            session.getProvenanceReporter().send(flowFile, url, "Remote DN=" + dnHolder.get(), uploadMillis,
                    true);
            session.transfer(flowFile, REL_SUCCESS);
        }
        return;
    }

    //
    // the response indicated a Hold URI; delete the Hold.
    //
    // determine the full URI of the Flow File's Hold; Unfortunately, the responses that are returned have
    // changed over the past, so we have to take into account a few different possibilities.
    String fullHoldUri = holdUri;
    if (holdUri.startsWith("/contentListener")) {
        // If the Hold URI that we get starts with /contentListener, it may not really be /contentListener,
        // as this really indicates that it should be whatever we posted to -- if posting directly to the
        // ListenHTTP component, it will be /contentListener, but if posting to a proxy/load balancer, we may
        // be posting to some other URL.
        fullHoldUri = url + holdUri.substring(16);
    } else if (holdUri.startsWith("/")) {
        // URL indicates the full path but not hostname or port; use the same hostname & port that we posted
        // to but use the full path indicated by the response.
        int firstSlash = url.indexOf("/", 8);
        if (firstSlash < 0) {
            firstSlash = url.length();
        }
        final String beforeSlash = url.substring(0, firstSlash);
        fullHoldUri = beforeSlash + holdUri;
    } else if (!holdUri.startsWith("http")) {
        // Absolute URL
        fullHoldUri = url + (url.endsWith("/") ? "" : "/") + holdUri;
    }

    final HttpDelete delete = new HttpDelete(fullHoldUri);
    delete.setHeader(TRANSACTION_ID_HEADER, transactionId);

    while (true) {
        try {
            final HttpResponse holdResponse = client.execute(delete);
            EntityUtils.consume(holdResponse.getEntity());
            final int holdStatusCode = holdResponse.getStatusLine().getStatusCode();
            final String holdReason = holdResponse.getStatusLine().getReasonPhrase();
            if (holdStatusCode >= 300) {
                logger.error(
                        "Failed to delete Hold that destination placed on {}: got response code {}:{}; routing to failure",
                        new Object[] { flowFileDescription, holdStatusCode, holdReason });

                for (FlowFile flowFile : toSend) {
                    flowFile = session.penalize(flowFile);
                    session.transfer(flowFile, REL_FAILURE);
                }
                return;
            }

            logger.info("Successfully Posted {} to {} in {} milliseconds at a rate of {}",
                    new Object[] { flowFileDescription, url, uploadMillis, uploadDataRate });

            for (FlowFile flowFile : toSend) {
                session.getProvenanceReporter().send(flowFile, url);
                session.transfer(flowFile, REL_SUCCESS);
            }
            return;
        } catch (final IOException e) {
            logger.warn("Failed to delete Hold that destination placed on {} due to {}",
                    new Object[] { flowFileDescription, e });
        }

        if (!isScheduled()) {
            context.yield();
            logger.warn(
                    "Failed to delete Hold that destination placed on {}; Processor has been stopped so routing FlowFile(s) to failure",
                    new Object[] { flowFileDescription });
            for (FlowFile flowFile : toSend) {
                flowFile = session.penalize(flowFile);
                session.transfer(flowFile, REL_FAILURE);
            }
            return;
        }
    }
}