Example usage for org.apache.http.impl.client HttpClientBuilder setUserAgent

List of usage examples for org.apache.http.impl.client HttpClientBuilder setUserAgent

Introduction

In this page you can find the example usage for org.apache.http.impl.client HttpClientBuilder setUserAgent.

Prototype

public final HttpClientBuilder setUserAgent(final String userAgent) 

Source Link

Document

Assigns User-Agent value.

Usage

From source file:com.cognitivemedicine.nifi.http.PostHTTP2.java

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    final boolean sendAsFlowFile = context.getProperty(SEND_AS_FLOWFILE).asBoolean();
    final int compressionLevel = context.getProperty(COMPRESSION_LEVEL).asInteger();
    final String userAgent = context.getProperty(USER_AGENT).getValue();

    final RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
    requestConfigBuilder.setConnectionRequestTimeout(
            context.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
    requestConfigBuilder.setConnectTimeout(
            context.getProperty(CONNECTION_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
    requestConfigBuilder.setRedirectsEnabled(false);
    requestConfigBuilder/*from www  .  ja  v  a  2  s  .c om*/
            .setSocketTimeout(context.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
    final RequestConfig requestConfig = requestConfigBuilder.build();

    final StreamThrottler throttler = throttlerRef.get();
    final ProcessorLog logger = getLogger();

    final Double maxBatchBytes = context.getProperty(MAX_BATCH_SIZE).asDataSize(DataUnit.B);
    String lastUrl = null;
    long bytesToSend = 0L;

    final List<FlowFile> toSend = new ArrayList<>();
    DestinationAccepts destinationAccepts = null;
    CloseableHttpClient client = null;
    final String transactionId = UUID.randomUUID().toString();

    final ObjectHolder<String> dnHolder = new ObjectHolder<>("none");
    while (true) {
        FlowFile flowFile = session.get();
        if (flowFile == null) {
            break;
        }

        final String url = context.getProperty(URL).evaluateAttributeExpressions(flowFile).getValue();
        try {
            new java.net.URL(url);
        } catch (final MalformedURLException e) {
            logger.error(
                    "After substituting attribute values for {}, URL is {}; this is not a valid URL, so routing to failure",
                    new Object[] { flowFile, url });
            flowFile = session.penalize(flowFile);
            session.transfer(flowFile, REL_FAILURE);
            continue;
        }

        // If this FlowFile doesn't have the same url, throw it back on the queue and stop grabbing FlowFiles
        if (lastUrl != null && !lastUrl.equals(url)) {
            session.transfer(flowFile);
            break;
        }

        lastUrl = url;
        toSend.add(flowFile);

        if (client == null || destinationAccepts == null) {
            final Config config = getConfig(url, context);
            final HttpClientConnectionManager conMan = config.getConnectionManager();

            final HttpClientBuilder clientBuilder = HttpClientBuilder.create();
            clientBuilder.setConnectionManager(conMan);
            clientBuilder.setUserAgent(userAgent);
            clientBuilder.addInterceptorFirst(new HttpResponseInterceptor() {
                @Override
                public void process(final HttpResponse response, final HttpContext httpContext)
                        throws HttpException, IOException {
                    HttpCoreContext coreContext = HttpCoreContext.adapt(httpContext);
                    ManagedHttpClientConnection conn = coreContext
                            .getConnection(ManagedHttpClientConnection.class);
                    if (!conn.isOpen()) {
                        return;
                    }

                    SSLSession sslSession = conn.getSSLSession();

                    if (sslSession != null) {
                        final X509Certificate[] certChain = sslSession.getPeerCertificateChain();
                        if (certChain == null || certChain.length == 0) {
                            throw new SSLPeerUnverifiedException("No certificates found");
                        }

                        final X509Certificate cert = certChain[0];
                        dnHolder.set(cert.getSubjectDN().getName().trim());
                    }
                }
            });

            clientBuilder.disableAutomaticRetries();
            clientBuilder.disableContentCompression();

            final String username = context.getProperty(USERNAME).getValue();
            final String password = context.getProperty(PASSWORD).getValue();
            // set the credentials if appropriate
            if (username != null) {
                final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
                if (password == null) {
                    credentialsProvider.setCredentials(AuthScope.ANY,
                            new UsernamePasswordCredentials(username));
                } else {
                    credentialsProvider.setCredentials(AuthScope.ANY,
                            new UsernamePasswordCredentials(username, password));
                }
                ;
                clientBuilder.setDefaultCredentialsProvider(credentialsProvider);
            }
            client = clientBuilder.build();

            // determine whether or not destination accepts flowfile/gzip
            destinationAccepts = config.getDestinationAccepts();
            if (destinationAccepts == null) {
                try {
                    if (sendAsFlowFile) {
                        destinationAccepts = getDestinationAcceptance(client, url, getLogger(), transactionId);
                    } else {
                        destinationAccepts = new DestinationAccepts(false, false, false, false, null);
                    }

                    config.setDestinationAccepts(destinationAccepts);
                } catch (IOException e) {
                    flowFile = session.penalize(flowFile);
                    session.transfer(flowFile, REL_FAILURE);
                    logger.error(
                            "Unable to communicate with destination {} to determine whether or not it can accept flowfiles/gzip; routing {} to failure due to {}",
                            new Object[] { url, flowFile, e });
                    context.yield();
                    return;
                }
            }
        }

        // if we are not sending as flowfile, or if the destination doesn't accept V3 or V2 (streaming) format,
        // then only use a single FlowFile
        if (!sendAsFlowFile
                || (!destinationAccepts.isFlowFileV3Accepted() && !destinationAccepts.isFlowFileV2Accepted())) {
            break;
        }

        bytesToSend += flowFile.getSize();
        if (bytesToSend > maxBatchBytes.longValue()) {
            break;
        }
    }

    if (toSend.isEmpty()) {
        return;
    }

    final String url = lastUrl;
    final HttpPost post = new HttpPost(url);
    final List<FlowFile> flowFileList = toSend;
    final DestinationAccepts accepts = destinationAccepts;
    final boolean isDestinationLegacyNiFi = accepts.getProtocolVersion() == null;

    final EntityTemplate entity = new EntityTemplate(new ContentProducer() {
        @Override
        public void writeTo(final OutputStream rawOut) throws IOException {
            final OutputStream throttled = (throttler == null) ? rawOut
                    : throttler.newThrottledOutputStream(rawOut);
            OutputStream wrappedOut = new BufferedOutputStream(throttled);
            if (compressionLevel > 0 && accepts.isGzipAccepted()) {
                wrappedOut = new GZIPOutputStream(wrappedOut, compressionLevel);
            }

            try (final OutputStream out = wrappedOut) {
                for (final FlowFile flowFile : flowFileList) {
                    session.read(flowFile, new InputStreamCallback() {
                        @Override
                        public void process(final InputStream rawIn) throws IOException {
                            try (final InputStream in = new BufferedInputStream(rawIn)) {

                                FlowFilePackager packager = null;
                                if (!sendAsFlowFile) {
                                    packager = null;
                                } else if (accepts.isFlowFileV3Accepted()) {
                                    packager = new FlowFilePackagerV3();
                                } else if (accepts.isFlowFileV2Accepted()) {
                                    packager = new FlowFilePackagerV2();
                                } else if (accepts.isFlowFileV1Accepted()) {
                                    packager = new FlowFilePackagerV1();
                                }

                                // if none of the above conditions is met, we should never get here, because
                                // we will have already verified that at least 1 of the FlowFile packaging
                                // formats is acceptable if sending as FlowFile.
                                if (packager == null) {
                                    StreamUtils.copy(in, out);
                                } else {
                                    final Map<String, String> flowFileAttributes;
                                    if (isDestinationLegacyNiFi) {
                                        // Old versions of NiFi expect nf.file.name and nf.file.path to indicate filename & path;
                                        // in order to maintain backward compatibility, we copy the filename & path to those attribute keys.
                                        flowFileAttributes = new HashMap<>(flowFile.getAttributes());
                                        flowFileAttributes.put("nf.file.name",
                                                flowFile.getAttribute(CoreAttributes.FILENAME.key()));
                                        flowFileAttributes.put("nf.file.path",
                                                flowFile.getAttribute(CoreAttributes.PATH.key()));
                                    } else {
                                        flowFileAttributes = flowFile.getAttributes();
                                    }

                                    packager.packageFlowFile(in, out, flowFileAttributes, flowFile.getSize());
                                }
                            }
                        }
                    });
                }

                out.flush();
            }
        }
    });

    entity.setChunked(context.getProperty(CHUNKED_ENCODING).asBoolean());
    post.setEntity(entity);
    post.setConfig(requestConfig);

    final String contentType;
    if (sendAsFlowFile) {
        if (accepts.isFlowFileV3Accepted()) {
            contentType = APPLICATION_FLOW_FILE_V3;
        } else if (accepts.isFlowFileV2Accepted()) {
            contentType = APPLICATION_FLOW_FILE_V2;
        } else if (accepts.isFlowFileV1Accepted()) {
            contentType = APPLICATION_FLOW_FILE_V1;
        } else {
            logger.error(
                    "Cannot send data to {} because the destination does not accept FlowFiles and this processor is configured to deliver FlowFiles; rolling back session",
                    new Object[] { url });
            session.rollback();
            context.yield();
            return;
        }
    } else {
        final String attributeValue = toSend.get(0).getAttribute(CoreAttributes.MIME_TYPE.key());
        contentType = (attributeValue == null) ? DEFAULT_CONTENT_TYPE : attributeValue;
    }

    final String attributeHeaderRegex = context.getProperty(ATTRIBUTES_AS_HEADERS_REGEX).getValue();
    if (attributeHeaderRegex != null && !sendAsFlowFile && flowFileList.size() == 1) {
        final Pattern pattern = Pattern.compile(attributeHeaderRegex);

        final Map<String, String> attributes = flowFileList.get(0).getAttributes();
        for (final Map.Entry<String, String> entry : attributes.entrySet()) {
            final String key = entry.getKey();
            if (pattern.matcher(key).matches()) {
                post.setHeader(entry.getKey(), entry.getValue());
            }
        }
    }

    post.setHeader(CONTENT_TYPE, contentType);
    post.setHeader(FLOWFILE_CONFIRMATION_HEADER, "true");
    post.setHeader(PROTOCOL_VERSION_HEADER, PROTOCOL_VERSION);
    post.setHeader(TRANSACTION_ID_HEADER, transactionId);
    if (compressionLevel > 0 && accepts.isGzipAccepted()) {
        post.setHeader(GZIPPED_HEADER, "true");
    }

    // Do the actual POST
    final String flowFileDescription = toSend.size() <= 10 ? toSend.toString() : toSend.size() + " FlowFiles";

    final String uploadDataRate;
    final long uploadMillis;
    CloseableHttpResponse response = null;
    try {
        final StopWatch stopWatch = new StopWatch(true);
        response = client.execute(post);

        // consume input stream entirely, ignoring its contents. If we
        // don't do this, the Connection will not be returned to the pool
        EntityUtils.consume(response.getEntity());
        stopWatch.stop();
        uploadDataRate = stopWatch.calculateDataRate(bytesToSend);
        uploadMillis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
    } catch (final IOException e) {
        logger.error("Failed to Post {} due to {}; transferring to failure",
                new Object[] { flowFileDescription, e });
        context.yield();
        for (FlowFile flowFile : toSend) {
            flowFile = session.penalize(flowFile);
            session.transfer(flowFile, REL_FAILURE);
        }
        return;
    } finally {
        if (response != null) {
            try {
                response.close();
            } catch (IOException e) {
                getLogger().warn("Failed to close HTTP Response due to {}", new Object[] { e });
            }
        }
    }

    // If we get a 'SEE OTHER' status code and an HTTP header that indicates that the intent
    // of the Location URI is a flowfile hold, we will store this holdUri. This prevents us
    // from posting to some other webservice and then attempting to delete some resource to which
    // we are redirected
    final int responseCode = response.getStatusLine().getStatusCode();
    final String responseReason = response.getStatusLine().getReasonPhrase();
    String holdUri = null;
    if (responseCode == HttpServletResponse.SC_SEE_OTHER) {
        final Header locationUriHeader = response.getFirstHeader(LOCATION_URI_INTENT_NAME);
        if (locationUriHeader != null) {
            if (LOCATION_URI_INTENT_VALUE.equals(locationUriHeader.getValue())) {
                final Header holdUriHeader = response.getFirstHeader(LOCATION_HEADER_NAME);
                if (holdUriHeader != null) {
                    holdUri = holdUriHeader.getValue();
                }
            }
        }

        if (holdUri == null) {
            for (FlowFile flowFile : toSend) {
                flowFile = session.penalize(flowFile);
                logger.error(
                        "Failed to Post {} to {}: sent content and received status code {}:{} but no Hold URI",
                        new Object[] { flowFile, url, responseCode, responseReason });
                session.transfer(flowFile, REL_FAILURE);
            }
            return;
        }
    }

    if (holdUri == null) {
        if (responseCode == HttpServletResponse.SC_SERVICE_UNAVAILABLE) {
            for (FlowFile flowFile : toSend) {
                flowFile = session.penalize(flowFile);
                logger.error(
                        "Failed to Post {} to {}: response code was {}:{}; will yield processing, since the destination is temporarily unavailable",
                        new Object[] { flowFile, url, responseCode, responseReason });
                session.transfer(flowFile, REL_FAILURE);
            }
            context.yield();
            return;
        }

        if (responseCode >= 300) {
            for (FlowFile flowFile : toSend) {
                flowFile = session.penalize(flowFile);
                logger.error("Failed to Post {} to {}: response code was {}:{}",
                        new Object[] { flowFile, url, responseCode, responseReason });
                session.transfer(flowFile, REL_FAILURE);
            }
            return;
        }

        logger.info("Successfully Posted {} to {} in {} at a rate of {}", new Object[] { flowFileDescription,
                url, FormatUtils.formatMinutesSeconds(uploadMillis, TimeUnit.MILLISECONDS), uploadDataRate });

        for (final FlowFile flowFile : toSend) {
            session.getProvenanceReporter().send(flowFile, url, "Remote DN=" + dnHolder.get(), uploadMillis,
                    true);
            session.transfer(flowFile, REL_SUCCESS);
        }
        return;
    }

    //
    // the response indicated a Hold URI; delete the Hold.
    //
    // determine the full URI of the Flow File's Hold; Unfortunately, the responses that are returned have
    // changed over the past, so we have to take into account a few different possibilities.
    String fullHoldUri = holdUri;
    if (holdUri.startsWith("/contentListener")) {
        // If the Hold URI that we get starts with /contentListener, it may not really be /contentListener,
        // as this really indicates that it should be whatever we posted to -- if posting directly to the
        // ListenHTTP component, it will be /contentListener, but if posting to a proxy/load balancer, we may
        // be posting to some other URL.
        fullHoldUri = url + holdUri.substring(16);
    } else if (holdUri.startsWith("/")) {
        // URL indicates the full path but not hostname or port; use the same hostname & port that we posted
        // to but use the full path indicated by the response.
        int firstSlash = url.indexOf("/", 8);
        if (firstSlash < 0) {
            firstSlash = url.length();
        }
        final String beforeSlash = url.substring(0, firstSlash);
        fullHoldUri = beforeSlash + holdUri;
    } else if (!holdUri.startsWith("http")) {
        // Absolute URL
        fullHoldUri = url + (url.endsWith("/") ? "" : "/") + holdUri;
    }

    final HttpDelete delete = new HttpDelete(fullHoldUri);
    delete.setHeader(TRANSACTION_ID_HEADER, transactionId);

    while (true) {
        try {
            final HttpResponse holdResponse = client.execute(delete);
            EntityUtils.consume(holdResponse.getEntity());
            final int holdStatusCode = holdResponse.getStatusLine().getStatusCode();
            final String holdReason = holdResponse.getStatusLine().getReasonPhrase();
            if (holdStatusCode >= 300) {
                logger.error(
                        "Failed to delete Hold that destination placed on {}: got response code {}:{}; routing to failure",
                        new Object[] { flowFileDescription, holdStatusCode, holdReason });

                for (FlowFile flowFile : toSend) {
                    flowFile = session.penalize(flowFile);
                    session.transfer(flowFile, REL_FAILURE);
                }
                return;
            }

            logger.info("Successfully Posted {} to {} in {} milliseconds at a rate of {}",
                    new Object[] { flowFileDescription, url, uploadMillis, uploadDataRate });

            for (FlowFile flowFile : toSend) {
                session.getProvenanceReporter().send(flowFile, url);
                session.transfer(flowFile, REL_SUCCESS);
            }
            return;
        } catch (final IOException e) {
            logger.warn("Failed to delete Hold that destination placed on {} due to {}",
                    new Object[] { flowFileDescription, e });
        }

        if (!isScheduled()) {
            context.yield();
            logger.warn(
                    "Failed to delete Hold that destination placed on {}; Processor has been stopped so routing FlowFile(s) to failure",
                    new Object[] { flowFileDescription });
            for (FlowFile flowFile : toSend) {
                flowFile = session.penalize(flowFile);
                session.transfer(flowFile, REL_FAILURE);
            }
            return;
        }
    }
}

From source file:com.cognitivemedicine.nifi.http.PostAdvancedHTTP.java

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    final boolean sendAsFlowFile = context.getProperty(SEND_AS_FLOWFILE).asBoolean();
    final int compressionLevel = context.getProperty(COMPRESSION_LEVEL).asInteger();
    final String userAgent = context.getProperty(USER_AGENT).getValue();

    final RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
    requestConfigBuilder.setConnectionRequestTimeout(
            context.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
    requestConfigBuilder.setConnectTimeout(
            context.getProperty(CONNECTION_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
    requestConfigBuilder.setRedirectsEnabled(false);
    requestConfigBuilder/*from   w  w w.j av a  2 s . co m*/
            .setSocketTimeout(context.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
    final RequestConfig requestConfig = requestConfigBuilder.build();

    final StreamThrottler throttler = throttlerRef.get();
    final ProcessorLog logger = getLogger();

    final Double maxBatchBytes = context.getProperty(MAX_BATCH_SIZE).asDataSize(DataUnit.B);
    String lastUrl = null;
    long bytesToSend = 0L;

    final List<FlowFile> toSend = new ArrayList<>();
    DestinationAccepts destinationAccepts = null;
    CloseableHttpClient client = null;
    final String transactionId = UUID.randomUUID().toString();

    final ObjectHolder<String> dnHolder = new ObjectHolder<>("none");
    while (true) {
        FlowFile flowFile = session.get();
        if (flowFile == null) {
            break;
        }

        final String url = context.getProperty(URL).evaluateAttributeExpressions(flowFile).getValue();
        try {
            new java.net.URL(url);
        } catch (final MalformedURLException e) {
            logger.error(
                    "After substituting attribute values for {}, URL is {}; this is not a valid URL, so routing to failure",
                    new Object[] { flowFile, url });
            flowFile = session.penalize(flowFile);
            session.transfer(flowFile, REL_FAILURE);
            continue;
        }

        // If this FlowFile doesn't have the same url, throw it back on the queue and stop grabbing FlowFiles
        if (lastUrl != null && !lastUrl.equals(url)) {
            session.transfer(flowFile);
            break;
        }

        lastUrl = url;
        toSend.add(flowFile);

        if (client == null || destinationAccepts == null) {
            final Config config = getConfig(url, context);
            final HttpClientConnectionManager conMan = config.getConnectionManager();

            final HttpClientBuilder clientBuilder = HttpClientBuilder.create();
            clientBuilder.setConnectionManager(conMan);
            clientBuilder.setUserAgent(userAgent);
            clientBuilder.addInterceptorFirst(new HttpResponseInterceptor() {
                @Override
                public void process(final HttpResponse response, final HttpContext httpContext)
                        throws HttpException, IOException {
                    HttpCoreContext coreContext = HttpCoreContext.adapt(httpContext);
                    ManagedHttpClientConnection conn = coreContext
                            .getConnection(ManagedHttpClientConnection.class);
                    if (!conn.isOpen()) {
                        return;
                    }

                    SSLSession sslSession = conn.getSSLSession();

                    if (sslSession != null) {
                        final X509Certificate[] certChain = sslSession.getPeerCertificateChain();
                        if (certChain == null || certChain.length == 0) {
                            throw new SSLPeerUnverifiedException("No certificates found");
                        }

                        final X509Certificate cert = certChain[0];
                        dnHolder.set(cert.getSubjectDN().getName().trim());
                    }
                }
            });

            clientBuilder.disableAutomaticRetries();
            clientBuilder.disableContentCompression();

            final String username = context.getProperty(USERNAME).getValue();
            final String password = context.getProperty(PASSWORD).getValue();
            // set the credentials if appropriate
            if (username != null) {
                final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
                if (password == null) {
                    credentialsProvider.setCredentials(AuthScope.ANY,
                            new UsernamePasswordCredentials(username));
                } else {
                    credentialsProvider.setCredentials(AuthScope.ANY,
                            new UsernamePasswordCredentials(username, password));
                }
                ;
                clientBuilder.setDefaultCredentialsProvider(credentialsProvider);
            }
            client = clientBuilder.build();

            // determine whether or not destination accepts flowfile/gzip
            destinationAccepts = config.getDestinationAccepts();
            if (destinationAccepts == null) {
                try {
                    if (sendAsFlowFile) {
                        destinationAccepts = getDestinationAcceptance(client, url, getLogger(), transactionId);
                    } else {
                        destinationAccepts = new DestinationAccepts(false, false, false, false, null);
                    }

                    config.setDestinationAccepts(destinationAccepts);
                } catch (IOException e) {
                    flowFile = session.penalize(flowFile);
                    session.transfer(flowFile, REL_FAILURE);
                    logger.error(
                            "Unable to communicate with destination {} to determine whether or not it can accept flowfiles/gzip; routing {} to failure due to {}",
                            new Object[] { url, flowFile, e });
                    context.yield();
                    return;
                }
            }
        }

        // if we are not sending as flowfile, or if the destination doesn't accept V3 or V2 (streaming) format,
        // then only use a single FlowFile
        if (!sendAsFlowFile
                || (!destinationAccepts.isFlowFileV3Accepted() && !destinationAccepts.isFlowFileV2Accepted())) {
            break;
        }

        bytesToSend += flowFile.getSize();
        if (bytesToSend > maxBatchBytes.longValue()) {
            break;
        }
    }

    if (toSend.isEmpty()) {
        return;
    }

    final String url = lastUrl;
    final HttpPost post = new HttpPost(url);
    final List<FlowFile> flowFileList = toSend;
    final DestinationAccepts accepts = destinationAccepts;
    final boolean isDestinationLegacyNiFi = accepts.getProtocolVersion() == null;

    final EntityTemplate entity = new EntityTemplate(new ContentProducer() {
        @Override
        public void writeTo(final OutputStream rawOut) throws IOException {
            final OutputStream throttled = (throttler == null) ? rawOut
                    : throttler.newThrottledOutputStream(rawOut);
            OutputStream wrappedOut = new BufferedOutputStream(throttled);
            if (compressionLevel > 0 && accepts.isGzipAccepted()) {
                wrappedOut = new GZIPOutputStream(wrappedOut, compressionLevel);
            }

            try (final OutputStream out = wrappedOut) {
                for (final FlowFile flowFile : flowFileList) {
                    session.read(flowFile, new InputStreamCallback() {
                        @Override
                        public void process(final InputStream rawIn) throws IOException {
                            try (final InputStream in = new BufferedInputStream(rawIn)) {

                                FlowFilePackager packager = null;
                                if (!sendAsFlowFile) {
                                    packager = null;
                                } else if (accepts.isFlowFileV3Accepted()) {
                                    packager = new FlowFilePackagerV3();
                                } else if (accepts.isFlowFileV2Accepted()) {
                                    packager = new FlowFilePackagerV2();
                                } else if (accepts.isFlowFileV1Accepted()) {
                                    packager = new FlowFilePackagerV1();
                                }

                                // if none of the above conditions is met, we should never get here, because
                                // we will have already verified that at least 1 of the FlowFile packaging
                                // formats is acceptable if sending as FlowFile.
                                if (packager == null) {
                                    StreamUtils.copy(in, out);
                                } else {
                                    final Map<String, String> flowFileAttributes;
                                    if (isDestinationLegacyNiFi) {
                                        // Old versions of NiFi expect nf.file.name and nf.file.path to indicate filename & path;
                                        // in order to maintain backward compatibility, we copy the filename & path to those attribute keys.
                                        flowFileAttributes = new HashMap<>(flowFile.getAttributes());
                                        flowFileAttributes.put("nf.file.name",
                                                flowFile.getAttribute(CoreAttributes.FILENAME.key()));
                                        flowFileAttributes.put("nf.file.path",
                                                flowFile.getAttribute(CoreAttributes.PATH.key()));
                                    } else {
                                        flowFileAttributes = flowFile.getAttributes();
                                    }

                                    packager.packageFlowFile(in, out, flowFileAttributes, flowFile.getSize());
                                }
                            }
                        }
                    });
                }

                out.flush();
            }
        }
    });

    entity.setChunked(context.getProperty(CHUNKED_ENCODING).asBoolean());
    post.setEntity(entity);
    post.setConfig(requestConfig);

    final String contentType;
    if (sendAsFlowFile) {
        if (accepts.isFlowFileV3Accepted()) {
            contentType = APPLICATION_FLOW_FILE_V3;
        } else if (accepts.isFlowFileV2Accepted()) {
            contentType = APPLICATION_FLOW_FILE_V2;
        } else if (accepts.isFlowFileV1Accepted()) {
            contentType = APPLICATION_FLOW_FILE_V1;
        } else {
            logger.error(
                    "Cannot send data to {} because the destination does not accept FlowFiles and this processor is configured to deliver FlowFiles; rolling back session",
                    new Object[] { url });
            session.rollback();
            context.yield();
            return;
        }
    } else {
        final String attributeValue = toSend.get(0).getAttribute(CoreAttributes.MIME_TYPE.key());
        contentType = (attributeValue == null) ? DEFAULT_CONTENT_TYPE : attributeValue;
    }

    final String attributeHeaderRegex = context.getProperty(ATTRIBUTES_AS_HEADERS_REGEX).getValue();
    if (attributeHeaderRegex != null && !sendAsFlowFile && flowFileList.size() == 1) {
        final Pattern pattern = Pattern.compile(attributeHeaderRegex);

        final Map<String, String> attributes = flowFileList.get(0).getAttributes();
        for (final Map.Entry<String, String> entry : attributes.entrySet()) {
            final String key = entry.getKey();
            if (pattern.matcher(key).matches()) {
                post.setHeader(entry.getKey(), entry.getValue());
            }
        }
    }

    post.setHeader(CONTENT_TYPE, contentType);
    post.setHeader(FLOWFILE_CONFIRMATION_HEADER, "true");
    post.setHeader(PROTOCOL_VERSION_HEADER, PROTOCOL_VERSION);
    post.setHeader(TRANSACTION_ID_HEADER, transactionId);
    if (compressionLevel > 0 && accepts.isGzipAccepted()) {
        post.setHeader(GZIPPED_HEADER, "true");
    }

    // Do the actual POST
    final String flowFileDescription = toSend.size() <= 10 ? toSend.toString() : toSend.size() + " FlowFiles";

    final String uploadDataRate;
    final long uploadMillis;
    String responseContent;

    CloseableHttpResponse response = null;
    try {
        final StopWatch stopWatch = new StopWatch(true);
        response = client.execute(post);
        responseContent = EntityUtils.toString(response.getEntity());
        stopWatch.stop();
        uploadDataRate = stopWatch.calculateDataRate(bytesToSend);
        uploadMillis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
    } catch (final IOException e) {
        logger.error("Failed to Post {} due to {}; transferring to failure",
                new Object[] { flowFileDescription, e });
        context.yield();
        for (FlowFile flowFile : toSend) {
            flowFile = session.penalize(flowFile);
            session.transfer(flowFile, REL_FAILURE);
        }
        return;
    } finally {
        if (response != null) {
            try {
                response.close();
            } catch (IOException e) {
                getLogger().warn("Failed to close HTTP Response due to {}", new Object[] { e });
            }
        }
    }

    // If we get a 'SEE OTHER' status code and an HTTP header that indicates that the intent
    // of the Location URI is a flowfile hold, we will store this holdUri. This prevents us
    // from posting to some other webservice and then attempting to delete some resource to which
    // we are redirected
    final int responseCode = response.getStatusLine().getStatusCode();
    final String responseReason = response.getStatusLine().getReasonPhrase();
    String holdUri = null;
    if (responseCode == HttpServletResponse.SC_SEE_OTHER) {
        final Header locationUriHeader = response.getFirstHeader(LOCATION_URI_INTENT_NAME);
        if (locationUriHeader != null) {
            if (LOCATION_URI_INTENT_VALUE.equals(locationUriHeader.getValue())) {
                final Header holdUriHeader = response.getFirstHeader(LOCATION_HEADER_NAME);
                if (holdUriHeader != null) {
                    holdUri = holdUriHeader.getValue();
                }
            }
        }

        if (holdUri == null) {
            for (FlowFile flowFile : toSend) {
                flowFile = session.penalize(flowFile);
                logger.error(
                        "Failed to Post {} to {}: sent content and received status code {}:{} but no Hold URI",
                        new Object[] { flowFile, url, responseCode, responseReason });
                session.transfer(flowFile, REL_FAILURE);
            }
            return;
        }
    }

    if (holdUri == null) {
        if (responseCode == HttpServletResponse.SC_SERVICE_UNAVAILABLE) {
            for (FlowFile flowFile : toSend) {
                flowFile = session.penalize(flowFile);
                logger.error(
                        "Failed to Post {} to {}: response code was {}:{}; will yield processing, since the destination is temporarily unavailable",
                        new Object[] { flowFile, url, responseCode, responseReason });
                session.transfer(flowFile, REL_FAILURE);
            }
            context.yield();
            return;
        }

        if (responseCode >= 300) {
            for (FlowFile flowFile : toSend) {
                flowFile = session.penalize(flowFile);
                logger.error("Failed to Post {} to {}: response code was {}:{}",
                        new Object[] { flowFile, url, responseCode, responseReason });
                session.transfer(flowFile, REL_FAILURE);
            }
            return;
        }

        logger.info("Successfully Posted {} to {} in {} at a rate of {}", new Object[] { flowFileDescription,
                url, FormatUtils.formatMinutesSeconds(uploadMillis, TimeUnit.MILLISECONDS), uploadDataRate });

        for (FlowFile flowFile : toSend) {

            flowFile = this.setHttpPostResponse(context, session, responseContent, flowFile);

            session.getProvenanceReporter().send(flowFile, url, "Remote DN=" + dnHolder.get(), uploadMillis,
                    true);
            session.transfer(flowFile, REL_SUCCESS);
        }

        return;
    }

    //
    // the response indicated a Hold URI; delete the Hold.
    //
    // determine the full URI of the Flow File's Hold; Unfortunately, the responses that are returned have
    // changed over the past, so we have to take into account a few different possibilities.
    String fullHoldUri = holdUri;
    if (holdUri.startsWith("/contentListener")) {
        // If the Hold URI that we get starts with /contentListener, it may not really be /contentListener,
        // as this really indicates that it should be whatever we posted to -- if posting directly to the
        // ListenHTTP component, it will be /contentListener, but if posting to a proxy/load balancer, we may
        // be posting to some other URL.
        fullHoldUri = url + holdUri.substring(16);
    } else if (holdUri.startsWith("/")) {
        // URL indicates the full path but not hostname or port; use the same hostname & port that we posted
        // to but use the full path indicated by the response.
        int firstSlash = url.indexOf("/", 8);
        if (firstSlash < 0) {
            firstSlash = url.length();
        }
        final String beforeSlash = url.substring(0, firstSlash);
        fullHoldUri = beforeSlash + holdUri;
    } else if (!holdUri.startsWith("http")) {
        // Absolute URL
        fullHoldUri = url + (url.endsWith("/") ? "" : "/") + holdUri;
    }

    final HttpDelete delete = new HttpDelete(fullHoldUri);
    delete.setHeader(TRANSACTION_ID_HEADER, transactionId);

    while (true) {
        try {
            final HttpResponse holdResponse = client.execute(delete);
            responseContent = EntityUtils.toString(holdResponse.getEntity());
            final int holdStatusCode = holdResponse.getStatusLine().getStatusCode();
            final String holdReason = holdResponse.getStatusLine().getReasonPhrase();
            if (holdStatusCode >= 300) {
                logger.error(
                        "Failed to delete Hold that destination placed on {}: got response code {}:{}; routing to failure",
                        new Object[] { flowFileDescription, holdStatusCode, holdReason });

                for (FlowFile flowFile : toSend) {
                    flowFile = session.penalize(flowFile);
                    session.transfer(flowFile, REL_FAILURE);
                }
                return;
            }

            logger.info("Successfully Posted {} to {} in {} milliseconds at a rate of {}",
                    new Object[] { flowFileDescription, url, uploadMillis, uploadDataRate });

            for (FlowFile flowFile : toSend) {
                flowFile = this.setHttpPostResponse(context, session, responseContent, flowFile);
                session.getProvenanceReporter().send(flowFile, url);
                session.transfer(flowFile, REL_SUCCESS);
            }
            return;
        } catch (final IOException e) {
            logger.warn("Failed to delete Hold that destination placed on {} due to {}",
                    new Object[] { flowFileDescription, e });
        }

        if (!isScheduled()) {
            context.yield();
            logger.warn(
                    "Failed to delete Hold that destination placed on {}; Processor has been stopped so routing FlowFile(s) to failure",
                    new Object[] { flowFileDescription });
            for (FlowFile flowFile : toSend) {
                flowFile = session.penalize(flowFile);
                session.transfer(flowFile, REL_FAILURE);
            }
            return;
        }
    }
}

From source file:com.dnanexus.DXHTTPRequest.java

/**
 * Construct the DXHTTPRequest using the given DXEnvironment.
 */// w  w w  .j ava 2s  . c  om
public DXHTTPRequest(DXEnvironment env) {
    this.securityContext = env.getSecurityContextJson();
    this.apiserver = env.getApiserverPath();
    this.disableRetry = env.isRetryDisabled();

    SSLContextBuilder builder = new SSLContextBuilder();
    try {
        builder.loadTrustMaterial(null, new TrustStrategy() {
            @Override
            public boolean isTrusted(X509Certificate[] chain, String authType) throws CertificateException {
                return true;
            }
        });
    } catch (NoSuchAlgorithmException e) {
        e.printStackTrace();
    } catch (KeyStoreException e) {
        e.printStackTrace();
    }

    SSLConnectionSocketFactory sslSF = null;
    try {
        sslSF = new SSLConnectionSocketFactory(builder.build(),
                SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
    } catch (NoSuchAlgorithmException e) {
        e.printStackTrace();
    } catch (KeyManagementException e) {
        e.printStackTrace();
    }
    HttpClientBuilder httpClientBuilder = HttpClients.custom().useSystemProperties();
    String proxyHost = System.getProperty("http.proxyHost");
    String proxyPort = System.getProperty("http.proxyPort");
    String proxyHostS = System.getProperty("https.proxyHost");
    String proxyPortS = System.getProperty("https.proxyPort");
    if ((proxyHost == null || proxyPort == null) && (proxyHostS == null || proxyPortS == null)) {
        this.httpclient = HttpClientBuilder.create().setUserAgent(USER_AGENT).build();
    } else {
        HttpHost proxy = null;
        if (proxyHostS != null && proxyPortS != null) {
            proxy = new HttpHost(proxyHostS, Integer.parseInt(proxyPortS));
        } else {
            proxy = new HttpHost(proxyHost, Integer.parseInt(proxyPort));
        }
        httpClientBuilder.setProxy(proxy);
        HttpRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy);
        httpClientBuilder.setRoutePlanner(routePlanner).setSSLSocketFactory(sslSF);
        httpclient = httpClientBuilder.setUserAgent(USER_AGENT).build();
    }
}

From source file:com.aliyun.api.gateway.demo.Client.java

/**
 * <br>/*from  w  w w  .  java  2s.c  o m*/
 * Client?httpsURL?keystore?storePasswordkeystore??? 
 * <a href="http://docs.oracle.com/javase/6/docs/technotes/tools/windows/keytool.html">keytool</a>
 * 
 * @param appKey
 *            APP Key?APIAPP?
 * @param appSecret
 *            APP?APIAPP?
 * @param testEnv
 *            ?truefalse
 */
public Client(String appKey, String appSecret, boolean testEnv) {
    HttpClientBuilder builder = HttpClients.custom();
    try {
        SSLContext sslContext = null;
        if (testEnv) {
            sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() {
                @Override
                public boolean isTrusted(X509Certificate[] chain, String authType) throws CertificateException {
                    //truetrue
                    return true;
                }
            }).build();
        } else {
            //keytool?keystorekeystore
            KeyStore ks = KeyStore.getInstance(KeyStore.getDefaultType());
            ks.load(null, null);
            sslContext = SSLContexts.custom().loadTrustMaterial(ks, new TrustSelfSignedStrategy()).build();
        }
        SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext, new String[] { "TLSv1" },
                null, SSLConnectionSocketFactory.getDefaultHostnameVerifier());
        builder.setSSLSocketFactory(sslsf);
    } catch (KeyStoreException | KeyManagementException | NoSuchAlgorithmException | CertificateException
            | IOException e) {
        log.error(e.getMessage(), e);
    }
    httpClient = builder.setUserAgent(Constants.USER_AGENT).build();
    this.appKey = appKey;
    this.appSecret = appSecret;
    this.testEnv = testEnv;
}

From source file:org.codelibs.fess.crawler.client.http.HcHttpClient.java

public synchronized void init() {
    if (httpClient != null) {
        return;// w ww.j a  v  a2  s . c om
    }

    if (logger.isDebugEnabled()) {
        logger.debug("Initializing " + HcHttpClient.class.getName());
    }

    super.init();

    // robots.txt parser
    final Boolean robotsTxtEnabled = getInitParameter(ROBOTS_TXT_ENABLED_PROPERTY, Boolean.TRUE, Boolean.class);
    if (robotsTxtHelper != null) {
        robotsTxtHelper.setEnabled(robotsTxtEnabled.booleanValue());
    }

    // httpclient
    final org.apache.http.client.config.RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
    final HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();

    final Integer connectionTimeoutParam = getInitParameter(CONNECTION_TIMEOUT_PROPERTY, connectionTimeout,
            Integer.class);
    if (connectionTimeoutParam != null) {
        requestConfigBuilder.setConnectTimeout(connectionTimeoutParam);

    }
    final Integer soTimeoutParam = getInitParameter(SO_TIMEOUT_PROPERTY, soTimeout, Integer.class);
    if (soTimeoutParam != null) {
        requestConfigBuilder.setSocketTimeout(soTimeoutParam);
    }

    // AuthSchemeFactory
    final RegistryBuilder<AuthSchemeProvider> authSchemeProviderBuilder = RegistryBuilder.create();
    @SuppressWarnings("unchecked")
    final Map<String, AuthSchemeProvider> factoryMap = getInitParameter(AUTH_SCHEME_PROVIDERS_PROPERTY,
            authSchemeProviderMap, Map.class);
    if (factoryMap != null) {
        for (final Map.Entry<String, AuthSchemeProvider> entry : factoryMap.entrySet()) {
            authSchemeProviderBuilder.register(entry.getKey(), entry.getValue());
        }
    }

    // user agent
    userAgent = getInitParameter(USER_AGENT_PROPERTY, userAgent, String.class);
    if (StringUtil.isNotBlank(userAgent)) {
        httpClientBuilder.setUserAgent(userAgent);
    }

    final HttpRoutePlanner planner = buildRoutePlanner();
    if (planner != null) {
        httpClientBuilder.setRoutePlanner(planner);
    }

    // Authentication
    final Authentication[] siteCredentialList = getInitParameter(BASIC_AUTHENTICATIONS_PROPERTY,
            new Authentication[0], Authentication[].class);
    for (final Authentication authentication : siteCredentialList) {
        final AuthScope authScope = authentication.getAuthScope();
        credentialsProvider.setCredentials(authScope, authentication.getCredentials());
        final AuthScheme authScheme = authentication.getAuthScheme();
        if (authScope.getHost() != null && authScheme != null) {
            final HttpHost targetHost = new HttpHost(authScope.getHost(), authScope.getPort());
            authCache.put(targetHost, authScheme);
        }
    }

    httpClientContext.setAuthCache(authCache);
    httpClientContext.setCredentialsProvider(credentialsProvider);

    // Request Header
    final RequestHeader[] requestHeaders = getInitParameter(REQUERT_HEADERS_PROPERTY, new RequestHeader[0],
            RequestHeader[].class);
    for (final RequestHeader requestHeader : requestHeaders) {
        if (requestHeader.isValid()) {
            requestHeaderList.add(new BasicHeader(requestHeader.getName(), requestHeader.getValue()));
        }
    }

    // do not redirect
    requestConfigBuilder
            .setRedirectsEnabled(getInitParameter(REDIRECTS_ENABLED, redirectsEnabled, Boolean.class));

    // cookie
    if (cookieSpec != null) {
        requestConfigBuilder.setCookieSpec(cookieSpec);
    }

    // cookie store
    httpClientBuilder.setDefaultCookieStore(cookieStore);
    if (cookieStore != null) {
        final Cookie[] cookies = getInitParameter(COOKIES_PROPERTY, new Cookie[0], Cookie[].class);
        for (final Cookie cookie : cookies) {
            cookieStore.addCookie(cookie);
        }
    }

    connectionMonitorTask = TimeoutManager.getInstance().addTimeoutTarget(
            new HcConnectionMonitorTarget(clientConnectionManager, idleConnectionTimeout),
            connectionCheckInterval, true);

    final CloseableHttpClient closeableHttpClient = httpClientBuilder
            .setConnectionManager(clientConnectionManager).setDefaultRequestConfig(requestConfigBuilder.build())
            .build();
    if (!httpClientPropertyMap.isEmpty()) {
        final BeanDesc beanDesc = BeanDescFactory.getBeanDesc(closeableHttpClient.getClass());
        for (final Map.Entry<String, Object> entry : httpClientPropertyMap.entrySet()) {
            final String propertyName = entry.getKey();
            if (beanDesc.hasPropertyDesc(propertyName)) {
                final PropertyDesc propertyDesc = beanDesc.getPropertyDesc(propertyName);
                propertyDesc.setValue(closeableHttpClient, entry.getValue());
            } else {
                logger.warn("DefaultHttpClient does not have " + propertyName + ".");
            }
        }
    }

    httpClient = closeableHttpClient;
}

From source file:org.seasar.robot.client.http.HcHttpClient.java

public synchronized void init() {
    if (httpClient != null) {
        return;/*  www .jav  a2  s. com*/
    }

    if (logger.isDebugEnabled()) {
        logger.debug("Initializing " + HcHttpClient.class.getName());
    }

    // access timeout
    final Integer accessTimeoutParam = getInitParameter(ACCESS_TIMEOUT_PROPERTY, accessTimeout);
    if (accessTimeoutParam != null) {
        accessTimeout = accessTimeoutParam;
    }

    // robots.txt parser
    final Boolean robotsTxtEnabled = getInitParameter(ROBOTS_TXT_ENABLED_PROPERTY, Boolean.TRUE);
    if (robotsTxtHelper != null) {
        robotsTxtHelper.setEnabled(robotsTxtEnabled.booleanValue());
    }

    // httpclient
    final org.apache.http.client.config.RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
    final HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();

    final Integer connectionTimeoutParam = getInitParameter(CONNECTION_TIMEOUT_PROPERTY, connectionTimeout);
    if (connectionTimeoutParam != null) {
        requestConfigBuilder.setConnectTimeout(connectionTimeoutParam);

    }
    final Boolean staleCheckingEnabledParam = getInitParameter(STALE_CHECKING_ENABLED_PROPERTY,
            staleCheckingEnabled);
    if (staleCheckingEnabledParam != null) {
        requestConfigBuilder.setStaleConnectionCheckEnabled(staleCheckingEnabledParam);
    }
    final Integer soTimeoutParam = getInitParameter(SO_TIMEOUT_PROPERTY, soTimeout);
    if (soTimeoutParam != null) {
        requestConfigBuilder.setSocketTimeout(soTimeoutParam);
    }

    // AuthSchemeFactory
    final RegistryBuilder<AuthSchemeProvider> authSchemeProviderBuilder = RegistryBuilder.create();
    final Map<String, AuthSchemeProvider> factoryMap = getInitParameter(AUTH_SCHEME_PROVIDERS_PROPERTY,
            authSchemeProviderMap);
    if (factoryMap != null) {
        for (final Map.Entry<String, AuthSchemeProvider> entry : factoryMap.entrySet()) {
            authSchemeProviderBuilder.register(entry.getKey(), entry.getValue());
        }
    }

    // user agent
    userAgent = getInitParameter(USER_AGENT_PROPERTY, userAgent);
    if (StringUtil.isNotBlank(userAgent)) {
        httpClientBuilder.setUserAgent(userAgent);
    }

    CredentialsProvider credsProvider = null;
    AuthCache authCache = null;

    // proxy
    final String proxyHost = getInitParameter(PROXY_HOST_PROPERTY, this.proxyHost);
    final Integer proxyPort = getInitParameter(PROXY_PORT_PROPERTY, this.proxyPort);
    if (proxyHost != null && proxyPort != null) {
        final HttpHost proxy = new HttpHost(proxyHost, proxyPort);
        final DefaultProxyRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy);
        httpClientBuilder.setRoutePlanner(routePlanner);

        final Credentials credentials = getInitParameter(PROXY_CREDENTIALS_PROPERTY, proxyCredentials);
        if (credentials != null) {
            authCache = new BasicAuthCache();
            credsProvider = new BasicCredentialsProvider();

            credsProvider.setCredentials(new AuthScope(proxyHost, proxyPort), credentials);
            final AuthScheme authScheme = getInitParameter(PROXY_AUTH_SCHEME_PROPERTY, proxyAuthScheme);
            if (authScheme != null) {
                authCache.put(proxy, authScheme);
            }
        }
    }

    // Authentication
    final Authentication[] siteCredentialList = getInitParameter(BASIC_AUTHENTICATIONS_PROPERTY,
            new Authentication[0]);
    if (siteCredentialList != null && siteCredentialList.length > 0 && authCache == null) {
        authCache = new BasicAuthCache();
        credsProvider = new BasicCredentialsProvider();
    }
    for (final Authentication authentication : siteCredentialList) {
        final AuthScope authScope = authentication.getAuthScope();
        credsProvider.setCredentials(authScope, authentication.getCredentials());
        final AuthScheme authScheme = authentication.getAuthScheme();
        if (authScope.getHost() != null && authScheme != null) {
            final HttpHost targetHost = new HttpHost(authScope.getHost(), authScope.getPort());
            authCache.put(targetHost, authScheme);
        }
    }
    if (authCache != null) {
        httpClientContext.setAuthCache(authCache);
        httpClientContext.setCredentialsProvider(credsProvider);
    }

    // Request Header
    final RequestHeader[] requestHeaders = getInitParameter(REQUERT_HEADERS_PROPERTY, new RequestHeader[0]);
    for (final RequestHeader requestHeader : requestHeaders) {
        if (requestHeader.isValid()) {
            requestHeaderList.add(new BasicHeader(requestHeader.getName(), requestHeader.getValue()));
        }
    }

    // do not redirect
    requestConfigBuilder.setRedirectsEnabled(false);

    // cookie
    if (cookieSpec != null) {
        requestConfigBuilder.setCookieSpec(cookieSpec);
    }

    // cookie store
    httpClientBuilder.setDefaultCookieStore(cookieStore);
    if (cookieStore != null) {
        final Cookie[] cookies = getInitParameter(COOKIES_PROPERTY, new Cookie[0]);
        for (final Cookie cookie : cookies) {
            cookieStore.addCookie(cookie);
        }
    }

    connectionMonitorTask = TimeoutManager.getInstance().addTimeoutTarget(
            new HcConnectionMonitorTarget(clientConnectionManager, idleConnectionTimeout),
            connectionCheckInterval, true);

    final CloseableHttpClient closeableHttpClient = httpClientBuilder
            .setConnectionManager(clientConnectionManager).setDefaultRequestConfig(requestConfigBuilder.build())
            .build();
    if (!httpClientPropertyMap.isEmpty()) {
        final BeanDesc beanDesc = BeanDescFactory.getBeanDesc(closeableHttpClient.getClass());
        for (final Map.Entry<String, Object> entry : httpClientPropertyMap.entrySet()) {
            final String propertyName = entry.getKey();
            if (beanDesc.hasPropertyDesc(propertyName)) {
                final PropertyDesc propertyDesc = beanDesc.getPropertyDesc(propertyName);
                propertyDesc.setValue(closeableHttpClient, entry.getValue());
            } else {
                logger.warn("DefaultHttpClient does not have " + propertyName + ".");
            }
        }
    }

    httpClient = closeableHttpClient;
}

From source file:crawlercommons.fetcher.http.SimpleHttpFetcher.java

private void init() {
    if (_httpClient == null) {
        synchronized (SimpleHttpFetcher.class) {
            if (_httpClient != null)
                return;

            final HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();
            final RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();

            // Set the socket and connection timeout to be something
            // reasonable.
            requestConfigBuilder.setSocketTimeout(_socketTimeout);
            requestConfigBuilder.setConnectTimeout(_connectionTimeout);
            requestConfigBuilder.setConnectionRequestTimeout(_connectionRequestTimeout);

            /*//  www  .  j a  va2 s . c o  m
             * CoreConnectionPNames.TCP_NODELAY='http.tcp.nodelay':
             * determines whether Nagle's algorithm is to be used. Nagle's
             * algorithm tries to conserve bandwidth by minimizing the
             * number of segments that are sent. When applications wish to
             * decrease network latency and increase performance, they can
             * disable Nagle's algorithm (that is enable TCP_NODELAY. Data
             * will be sent earlier, at the cost of an increase in bandwidth
             * consumption. This parameter expects a value of type
             * java.lang.Boolean. If this parameter is not set, TCP_NODELAY
             * will be enabled (no delay).
             */
            // FIXME Could not find this parameter in http-client version
            // 4.5
            // HttpConnectionParams.setTcpNoDelay(params, true);
            // HttpProtocolParams.setVersion(params, _httpVersion);

            httpClientBuilder.setUserAgent(_userAgent.getUserAgentString());

            // HttpProtocolParams.setContentCharset(params, "UTF-8");
            // HttpProtocolParams.setHttpElementCharset(params, "UTF-8");

            /*
             * CoreProtocolPNames.USE_EXPECT_CONTINUE=
             * 'http.protocol.expect-continue': activates the Expect:
             * 100-Continue handshake for the entity enclosing methods. The
             * purpose of the Expect: 100-Continue handshake is to allow the
             * client that is sending a request message with a request body
             * to determine if the origin server is willing to accept the
             * request (based on the request headers) before the client
             * sends the request body. The use of the Expect: 100-continue
             * handshake can result in a noticeable performance improvement
             * for entity enclosing requests (such as POST and PUT) that
             * require the target server's authentication. The Expect:
             * 100-continue handshake should be used with caution, as it may
             * cause problems with HTTP servers and proxies that do not
             * support HTTP/1.1 protocol. This parameter expects a value of
             * type java.lang.Boolean. If this parameter is not set,
             * HttpClient will not attempt to use the handshake.
             */
            requestConfigBuilder.setExpectContinueEnabled(true);

            /*
             * CoreProtocolPNames.WAIT_FOR_CONTINUE=
             * 'http.protocol.wait-for-continue': defines the maximum period
             * of time in milliseconds the client should spend waiting for a
             * 100-continue response. This parameter expects a value of type
             * java.lang.Integer. If this parameter is not set HttpClient
             * will wait 3 seconds for a confirmation before resuming the
             * transmission of the request body.
             */
            // FIXME Could not find this parameter in http-client version
            // 4.5
            // params.setIntParameter(CoreProtocolPNames.WAIT_FOR_CONTINUE,
            // 5000);

            // FIXME Could not find this parameter in http-client version
            // 4.5
            // CookieSpecParamBean cookieParams = new
            // CookieSpecParamBean(params);
            // cookieParams.setSingleHeader(false);

            // Create and initialize connection socket factory registry
            RegistryBuilder<ConnectionSocketFactory> registry = RegistryBuilder
                    .<ConnectionSocketFactory>create();
            registry.register("http", PlainConnectionSocketFactory.getSocketFactory());
            SSLConnectionSocketFactory sf = createSSLConnectionSocketFactory();
            if (sf != null) {
                registry.register("https", sf);
            } else {
                LOGGER.warn("No valid SSLContext found for https");
            }

            _connectionManager = new PoolingHttpClientConnectionManager(registry.build());
            _connectionManager.setMaxTotal(_maxThreads);
            _connectionManager.setDefaultMaxPerRoute(getMaxConnectionsPerHost());

            /*
             * CoreConnectionPNames.STALE_CONNECTION_CHECK=
             * 'http.connection.stalecheck': determines whether stale
             * connection check is to be used. Disabling stale connection
             * check may result in a noticeable performance improvement (the
             * check can cause up to 30 millisecond overhead per request) at
             * the risk of getting an I/O error when executing a request
             * over a connection that has been closed at the server side.
             * This parameter expects a value of type java.lang.Boolean. For
             * performance critical operations the check should be disabled.
             * If this parameter is not set, the stale connection check will
             * be performed before each request execution.
             * 
             * We don't need I/O exceptions in case if Server doesn't
             * support Kee-Alive option; our client by default always tries
             * keep-alive.
             */
            // Even with stale checking enabled, a connection can "go stale"
            // between the check and the next request. So we still need to
            // handle the case of a closed socket (from the server side),
            // and disabling this check improves performance.
            // Stale connections will be checked in a separate monitor
            // thread
            _connectionManager.setValidateAfterInactivity(-1);

            httpClientBuilder.setConnectionManager(_connectionManager);
            httpClientBuilder.setRetryHandler(new MyRequestRetryHandler(_maxRetryCount));
            httpClientBuilder.setRedirectStrategy(new MyRedirectStrategy(getRedirectMode()));
            httpClientBuilder.setRequestExecutor(new MyHttpRequestExecutor());

            // FUTURE KKr - support authentication
            // FIXME Could not find this parameter in http-client version
            // 4.5
            // HttpClientParams.setAuthenticating(params, false);

            requestConfigBuilder.setCookieSpec(CookieSpecs.DEFAULT);

            if (getMaxRedirects() == 0) {
                requestConfigBuilder.setRedirectsEnabled(false);
            } else {
                requestConfigBuilder.setRedirectsEnabled(true);
                requestConfigBuilder.setMaxRedirects(getMaxRedirects());
            }

            // Set up default headers. This helps us get back from servers
            // what we want.
            HashSet<Header> defaultHeaders = new HashSet<Header>();
            defaultHeaders.add(new BasicHeader(HttpHeaders.ACCEPT_LANGUAGE, getAcceptLanguage()));
            defaultHeaders.add(new BasicHeader(HttpHeaders.ACCEPT_CHARSET, DEFAULT_ACCEPT_CHARSET));
            defaultHeaders.add(new BasicHeader(HttpHeaders.ACCEPT_ENCODING, DEFAULT_ACCEPT_ENCODING));
            defaultHeaders.add(new BasicHeader(HttpHeaders.ACCEPT, DEFAULT_ACCEPT));

            httpClientBuilder.setDefaultHeaders(defaultHeaders);

            httpClientBuilder.setKeepAliveStrategy(new MyConnectionKeepAliveStrategy());

            monitor = new IdleConnectionMonitorThread(_connectionManager);
            monitor.start();

            httpClientBuilder.setDefaultRequestConfig(requestConfigBuilder.build());
            _httpClient = httpClientBuilder.build();
        }
    }

}

From source file:com.igormaznitsa.mvngolang.AbstractGolangMojo.java

@Nonnull
private synchronized HttpClient getHttpClient(@Nullable final ProxySettings proxy) {
    if (this.httpClient == null) {
        final HttpClientBuilder builder = HttpClients.custom();

        if (proxy != null) {
            if (proxy.hasCredentials()) {
                final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
                credentialsProvider.setCredentials(new AuthScope(proxy.host, proxy.port),
                        new UsernamePasswordCredentials(proxy.username, proxy.password));
                builder.setDefaultCredentialsProvider(credentialsProvider);
                getLog().debug(//ww  w  . j a v  a 2  s . c o  m
                        String.format("Credentials provider has been created for proxy (username : %s): %s",
                                proxy.username, proxy.toString()));
            }

            final String[] ignoreForAddresses = proxy.nonProxyHosts == null ? new String[0]
                    : proxy.nonProxyHosts.split("\\|");
            if (ignoreForAddresses.length > 0) {
                final WildCardMatcher[] matchers = new WildCardMatcher[ignoreForAddresses.length];
                for (int i = 0; i < ignoreForAddresses.length; i++) {
                    matchers[i] = new WildCardMatcher(ignoreForAddresses[i]);
                }

                final HttpRoutePlanner routePlanner = new DefaultProxyRoutePlanner(
                        new HttpHost(proxy.host, proxy.port, proxy.protocol)) {
                    @Override
                    @Nonnull
                    public HttpRoute determineRoute(@Nonnull final HttpHost host,
                            @Nonnull final HttpRequest request, @Nonnull final HttpContext context)
                            throws HttpException {
                        final String hostName = host.getHostName();
                        for (final WildCardMatcher m : matchers) {
                            if (m.match(hostName)) {
                                getLog().debug("Ignoring proxy for host : " + hostName);
                                return new HttpRoute(host);
                            }
                        }
                        return super.determineRoute(host, request, context);
                    }
                };
                builder.setRoutePlanner(routePlanner);
                getLog().debug(
                        "Route planner tuned to ignore proxy for addresses : " + Arrays.toString(matchers));
            }
        }

        builder.setUserAgent("mvn-golang-wrapper-agent/1.0");
        this.httpClient = builder.build();

    }
    return this.httpClient;
}