Example usage for org.apache.http.impl.client HttpClientBuilder build

List of usage examples for org.apache.http.impl.client HttpClientBuilder build

  1. HOME
  2. Java
  3. org.apache.http
  4. org.apache.http.impl.client.*
  5. HttpClientBuilder
  6. build

Introduction

In this page you can find the example usage for org.apache.http.impl.client HttpClientBuilder build.

Prototype

public CloseableHttpClient build()

Source Link

Usage

From source file:org.apache.marmotta.client.util.HTTPUtil.java

public static HttpClient createClient(ClientConfiguration config, String context) {

    final HttpClientBuilder httpClientBuilder = HttpClients.custom();

    httpClientBuilder.setUserAgent("Marmotta Client Library/" + MetaUtil.getVersion());
    httpClientBuilder.setRedirectStrategy(new MarmottaRedirectStrategy());
    httpClientBuilder.setRetryHandler(new MarmottaHttpRequestRetryHandler());

    final RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
    requestConfigBuilder.setSocketTimeout(config.getSoTimeout());
    requestConfigBuilder.setConnectTimeout(config.getConnectionTimeout());
    requestConfigBuilder.setRedirectsEnabled(true);
    requestConfigBuilder.setMaxRedirects(3);
    httpClientBuilder.setDefaultRequestConfig(requestConfigBuilder.build());

    if (config.getConectionManager() != null) {
        httpClientBuilder.setConnectionManager(config.getConectionManager());
    } else {/*ww w .j a  va2 s  .c om*/
        final Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create()
                .register("http", PlainConnectionSocketFactory.getSocketFactory())
                //.register("https", )
                .build();

        final PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(registry);
        cm.setMaxTotal(100);
        httpClientBuilder.setConnectionManager(cm);
    }

    return httpClientBuilder.build();
}

From source file:org.apache.nifi.processors.standard.GetHTTP.java

@Override
public void onTrigger(final ProcessContext context, final ProcessSessionFactory sessionFactory)
        throws ProcessException {
    final ComponentLog logger = getLogger();

    final ProcessSession session = sessionFactory.createSession();
    final FlowFile incomingFlowFile = session.get();
    if (incomingFlowFile != null) {
        session.transfer(incomingFlowFile, REL_SUCCESS);
        logger.warn("found FlowFile {} in input queue; transferring to success",
                new Object[] { incomingFlowFile });
    }// w w  w .j  av a2  s.c om

    // get the URL
    final String url = context.getProperty(URL).evaluateAttributeExpressions().getValue();
    final URI uri;
    String source = url;
    try {
        uri = new URI(url);
        source = uri.getHost();
    } catch (final URISyntaxException swallow) {
        // this won't happen as the url has already been validated
    }

    // get the ssl context service
    final SSLContextService sslContextService = context.getProperty(SSL_CONTEXT_SERVICE)
            .asControllerService(SSLContextService.class);

    // create the connection manager
    final HttpClientConnectionManager conMan;
    if (sslContextService == null) {
        conMan = new BasicHttpClientConnectionManager();
    } else {
        final SSLContext sslContext;
        try {
            sslContext = createSSLContext(sslContextService);
        } catch (final Exception e) {
            throw new ProcessException(e);
        }

        final SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext);

        // Also include a plain socket factory for regular http connections (especially proxies)
        final Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder
                .<ConnectionSocketFactory>create().register("https", sslsf)
                .register("http", PlainConnectionSocketFactory.getSocketFactory()).build();

        conMan = new BasicHttpClientConnectionManager(socketFactoryRegistry);
    }

    try {
        // build the request configuration
        final RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
        requestConfigBuilder.setConnectionRequestTimeout(
                context.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
        requestConfigBuilder.setConnectTimeout(
                context.getProperty(CONNECTION_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
        requestConfigBuilder.setSocketTimeout(
                context.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
        requestConfigBuilder.setRedirectsEnabled(context.getProperty(FOLLOW_REDIRECTS).asBoolean());
        switch (context.getProperty(REDIRECT_COOKIE_POLICY).getValue()) {
        case STANDARD_COOKIE_POLICY_STR:
            requestConfigBuilder.setCookieSpec(CookieSpecs.STANDARD);
            break;
        case STRICT_COOKIE_POLICY_STR:
            requestConfigBuilder.setCookieSpec(CookieSpecs.STANDARD_STRICT);
            break;
        case NETSCAPE_COOKIE_POLICY_STR:
            requestConfigBuilder.setCookieSpec(CookieSpecs.NETSCAPE);
            break;
        case IGNORE_COOKIE_POLICY_STR:
            requestConfigBuilder.setCookieSpec(CookieSpecs.IGNORE_COOKIES);
            break;
        case DEFAULT_COOKIE_POLICY_STR:
        default:
            requestConfigBuilder.setCookieSpec(CookieSpecs.DEFAULT);
        }

        // build the http client
        final HttpClientBuilder clientBuilder = HttpClientBuilder.create();
        clientBuilder.setConnectionManager(conMan);

        // include the user agent
        final String userAgent = context.getProperty(USER_AGENT).getValue();
        if (userAgent != null) {
            clientBuilder.setUserAgent(userAgent);
        }

        // set the ssl context if necessary
        if (sslContextService != null) {
            clientBuilder.setSslcontext(sslContextService.createSSLContext(ClientAuth.REQUIRED));
        }

        final String username = context.getProperty(USERNAME).getValue();
        final String password = context.getProperty(PASSWORD).getValue();

        // set the credentials if appropriate
        if (username != null) {
            final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
            if (password == null) {
                credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username));
            } else {
                credentialsProvider.setCredentials(AuthScope.ANY,
                        new UsernamePasswordCredentials(username, password));
            }
            clientBuilder.setDefaultCredentialsProvider(credentialsProvider);
        }

        // Set the proxy if specified
        if (context.getProperty(PROXY_HOST).isSet() && context.getProperty(PROXY_PORT).isSet()) {
            final String host = context.getProperty(PROXY_HOST).getValue();
            final int port = context.getProperty(PROXY_PORT).asInteger();
            clientBuilder.setProxy(new HttpHost(host, port));
        }

        // create request
        final HttpGet get = new HttpGet(url);
        get.setConfig(requestConfigBuilder.build());

        final StateMap beforeStateMap;

        try {
            beforeStateMap = context.getStateManager().getState(Scope.LOCAL);
            final String lastModified = beforeStateMap.get(LAST_MODIFIED + ":" + url);
            if (lastModified != null) {
                get.addHeader(HEADER_IF_MODIFIED_SINCE, parseStateValue(lastModified).getValue());
            }

            final String etag = beforeStateMap.get(ETAG + ":" + url);
            if (etag != null) {
                get.addHeader(HEADER_IF_NONE_MATCH, parseStateValue(etag).getValue());
            }
        } catch (final IOException ioe) {
            throw new ProcessException(ioe);
        }

        final String accept = context.getProperty(ACCEPT_CONTENT_TYPE).getValue();
        if (accept != null) {
            get.addHeader(HEADER_ACCEPT, accept);
        }

        // Add dynamic headers

        PropertyValue customHeaderValue;
        for (PropertyDescriptor customProperty : customHeaders) {
            customHeaderValue = context.getProperty(customProperty).evaluateAttributeExpressions();
            if (StringUtils.isNotBlank(customHeaderValue.getValue())) {
                get.addHeader(customProperty.getName(), customHeaderValue.getValue());
            }
        }

        // create the http client
        try (final CloseableHttpClient client = clientBuilder.build()) {
            // NOTE: including this inner try in order to swallow exceptions on close
            try {
                final StopWatch stopWatch = new StopWatch(true);
                final HttpResponse response = client.execute(get);
                final int statusCode = response.getStatusLine().getStatusCode();
                if (statusCode == NOT_MODIFIED) {
                    logger.info(
                            "content not retrieved because server returned HTTP Status Code {}: Not Modified",
                            new Object[] { NOT_MODIFIED });
                    context.yield();
                    // doing a commit in case there were flow files in the input queue
                    session.commit();
                    return;
                }
                final String statusExplanation = response.getStatusLine().getReasonPhrase();

                if ((statusCode >= 300) || (statusCode == 204)) {
                    logger.error("received status code {}:{} from {}",
                            new Object[] { statusCode, statusExplanation, url });
                    // doing a commit in case there were flow files in the input queue
                    session.commit();
                    return;
                }

                FlowFile flowFile = session.create();
                flowFile = session.putAttribute(flowFile, CoreAttributes.FILENAME.key(),
                        context.getProperty(FILENAME).evaluateAttributeExpressions().getValue());
                flowFile = session.putAttribute(flowFile,
                        this.getClass().getSimpleName().toLowerCase() + ".remote.source", source);
                flowFile = session.importFrom(response.getEntity().getContent(), flowFile);

                final Header contentTypeHeader = response.getFirstHeader("Content-Type");
                if (contentTypeHeader != null) {
                    final String contentType = contentTypeHeader.getValue();
                    if (!contentType.trim().isEmpty()) {
                        flowFile = session.putAttribute(flowFile, CoreAttributes.MIME_TYPE.key(),
                                contentType.trim());
                    }
                }

                final long flowFileSize = flowFile.getSize();
                stopWatch.stop();
                final String dataRate = stopWatch.calculateDataRate(flowFileSize);
                session.getProvenanceReporter().receive(flowFile, url,
                        stopWatch.getDuration(TimeUnit.MILLISECONDS));
                session.transfer(flowFile, REL_SUCCESS);
                logger.info("Successfully received {} from {} at a rate of {}; transferred to success",
                        new Object[] { flowFile, url, dataRate });
                session.commit();

                updateStateMap(context, response, beforeStateMap, url);

            } catch (final IOException e) {
                context.yield();
                session.rollback();
                logger.error("Failed to retrieve file from {} due to {}; rolling back session",
                        new Object[] { url, e.getMessage() }, e);
                throw new ProcessException(e);
            } catch (final Throwable t) {
                context.yield();
                session.rollback();
                logger.error("Failed to process due to {}; rolling back session",
                        new Object[] { t.getMessage() }, t);
                throw t;
            }
        } catch (final IOException e) {
            logger.debug("Error closing client due to {}, continuing.", new Object[] { e.getMessage() });
        }
    } finally {
        conMan.shutdown();
    }
}

From source file:org.apache.nifi.processors.standard.PostHTTP.java

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    final boolean sendAsFlowFile = context.getProperty(SEND_AS_FLOWFILE).asBoolean();
    final int compressionLevel = context.getProperty(COMPRESSION_LEVEL).asInteger();
    final String userAgent = context.getProperty(USER_AGENT).getValue();

    final RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
    requestConfigBuilder.setConnectionRequestTimeout(
            context.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
    requestConfigBuilder.setConnectTimeout(
            context.getProperty(CONNECTION_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
    requestConfigBuilder.setRedirectsEnabled(false);
    requestConfigBuilder//ww w.  j av a2s  . c  o m
            .setSocketTimeout(context.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
    final RequestConfig requestConfig = requestConfigBuilder.build();

    final StreamThrottler throttler = throttlerRef.get();
    final ComponentLog logger = getLogger();

    final Double maxBatchBytes = context.getProperty(MAX_BATCH_SIZE).asDataSize(DataUnit.B);
    String lastUrl = null;
    long bytesToSend = 0L;

    final List<FlowFile> toSend = new ArrayList<>();
    DestinationAccepts destinationAccepts = null;
    CloseableHttpClient client = null;
    final String transactionId = UUID.randomUUID().toString();

    final AtomicReference<String> dnHolder = new AtomicReference<>("none");
    while (true) {
        FlowFile flowFile = session.get();
        if (flowFile == null) {
            break;
        }

        final String url = context.getProperty(URL).evaluateAttributeExpressions(flowFile).getValue();
        try {
            new java.net.URL(url);
        } catch (final MalformedURLException e) {
            logger.error(
                    "After substituting attribute values for {}, URL is {}; this is not a valid URL, so routing to failure",
                    new Object[] { flowFile, url });
            flowFile = session.penalize(flowFile);
            session.transfer(flowFile, REL_FAILURE);
            continue;
        }

        // If this FlowFile doesn't have the same url, throw it back on the queue and stop grabbing FlowFiles
        if (lastUrl != null && !lastUrl.equals(url)) {
            session.transfer(flowFile);
            break;
        }

        lastUrl = url;
        toSend.add(flowFile);

        if (client == null || destinationAccepts == null) {
            final Config config = getConfig(url, context);
            final HttpClientConnectionManager conMan = config.getConnectionManager();

            final HttpClientBuilder clientBuilder = HttpClientBuilder.create();
            clientBuilder.setConnectionManager(conMan);
            clientBuilder.setUserAgent(userAgent);
            clientBuilder.addInterceptorFirst(new HttpResponseInterceptor() {
                @Override
                public void process(final HttpResponse response, final HttpContext httpContext)
                        throws HttpException, IOException {
                    final HttpCoreContext coreContext = HttpCoreContext.adapt(httpContext);
                    final ManagedHttpClientConnection conn = coreContext
                            .getConnection(ManagedHttpClientConnection.class);
                    if (!conn.isOpen()) {
                        return;
                    }

                    final SSLSession sslSession = conn.getSSLSession();

                    if (sslSession != null) {
                        final Certificate[] certChain = sslSession.getPeerCertificates();
                        if (certChain == null || certChain.length == 0) {
                            throw new SSLPeerUnverifiedException("No certificates found");
                        }

                        try {
                            final X509Certificate cert = CertificateUtils
                                    .convertAbstractX509Certificate(certChain[0]);
                            dnHolder.set(cert.getSubjectDN().getName().trim());
                        } catch (CertificateException e) {
                            final String msg = "Could not extract subject DN from SSL session peer certificate";
                            logger.warn(msg);
                            throw new SSLPeerUnverifiedException(msg);
                        }
                    }
                }
            });

            clientBuilder.disableAutomaticRetries();
            clientBuilder.disableContentCompression();

            final String username = context.getProperty(USERNAME).getValue();
            final String password = context.getProperty(PASSWORD).getValue();
            // set the credentials if appropriate
            if (username != null) {
                final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
                if (password == null) {
                    credentialsProvider.setCredentials(AuthScope.ANY,
                            new UsernamePasswordCredentials(username));
                } else {
                    credentialsProvider.setCredentials(AuthScope.ANY,
                            new UsernamePasswordCredentials(username, password));
                }
                clientBuilder.setDefaultCredentialsProvider(credentialsProvider);
            }

            // Set the proxy if specified
            if (context.getProperty(PROXY_HOST).isSet() && context.getProperty(PROXY_PORT).isSet()) {
                final String host = context.getProperty(PROXY_HOST).getValue();
                final int port = context.getProperty(PROXY_PORT).asInteger();
                clientBuilder.setProxy(new HttpHost(host, port));
            }

            client = clientBuilder.build();

            // determine whether or not destination accepts flowfile/gzip
            destinationAccepts = config.getDestinationAccepts();
            if (destinationAccepts == null) {
                try {
                    destinationAccepts = getDestinationAcceptance(sendAsFlowFile, client, url, getLogger(),
                            transactionId);
                    config.setDestinationAccepts(destinationAccepts);
                } catch (final IOException e) {
                    flowFile = session.penalize(flowFile);
                    session.transfer(flowFile, REL_FAILURE);
                    logger.error(
                            "Unable to communicate with destination {} to determine whether or not it can accept "
                                    + "flowfiles/gzip; routing {} to failure due to {}",
                            new Object[] { url, flowFile, e });
                    context.yield();
                    return;
                }
            }
        }

        bytesToSend += flowFile.getSize();
        if (bytesToSend > maxBatchBytes.longValue()) {
            break;
        }

        // if we are not sending as flowfile, or if the destination doesn't accept V3 or V2 (streaming) format,
        // then only use a single FlowFile
        if (!sendAsFlowFile
                || !destinationAccepts.isFlowFileV3Accepted() && !destinationAccepts.isFlowFileV2Accepted()) {
            break;
        }
    }

    if (toSend.isEmpty()) {
        return;
    }

    final String url = lastUrl;
    final HttpPost post = new HttpPost(url);
    final List<FlowFile> flowFileList = toSend;
    final DestinationAccepts accepts = destinationAccepts;
    final boolean isDestinationLegacyNiFi = accepts.getProtocolVersion() == null;

    final EntityTemplate entity = new EntityTemplate(new ContentProducer() {
        @Override
        public void writeTo(final OutputStream rawOut) throws IOException {
            final OutputStream throttled = throttler == null ? rawOut
                    : throttler.newThrottledOutputStream(rawOut);
            OutputStream wrappedOut = new BufferedOutputStream(throttled);
            if (compressionLevel > 0 && accepts.isGzipAccepted()) {
                wrappedOut = new GZIPOutputStream(wrappedOut, compressionLevel);
            }

            try (final OutputStream out = wrappedOut) {
                for (final FlowFile flowFile : flowFileList) {
                    session.read(flowFile, new InputStreamCallback() {
                        @Override
                        public void process(final InputStream rawIn) throws IOException {
                            try (final InputStream in = new BufferedInputStream(rawIn)) {

                                FlowFilePackager packager = null;
                                if (!sendAsFlowFile) {
                                    packager = null;
                                } else if (accepts.isFlowFileV3Accepted()) {
                                    packager = new FlowFilePackagerV3();
                                } else if (accepts.isFlowFileV2Accepted()) {
                                    packager = new FlowFilePackagerV2();
                                } else if (accepts.isFlowFileV1Accepted()) {
                                    packager = new FlowFilePackagerV1();
                                }

                                // if none of the above conditions is met, we should never get here, because
                                // we will have already verified that at least 1 of the FlowFile packaging
                                // formats is acceptable if sending as FlowFile.
                                if (packager == null) {
                                    StreamUtils.copy(in, out);
                                } else {
                                    final Map<String, String> flowFileAttributes;
                                    if (isDestinationLegacyNiFi) {
                                        // Old versions of NiFi expect nf.file.name and nf.file.path to indicate filename & path;
                                        // in order to maintain backward compatibility, we copy the filename & path to those attribute keys.
                                        flowFileAttributes = new HashMap<>(flowFile.getAttributes());
                                        flowFileAttributes.put("nf.file.name",
                                                flowFile.getAttribute(CoreAttributes.FILENAME.key()));
                                        flowFileAttributes.put("nf.file.path",
                                                flowFile.getAttribute(CoreAttributes.PATH.key()));
                                    } else {
                                        flowFileAttributes = flowFile.getAttributes();
                                    }

                                    packager.packageFlowFile(in, out, flowFileAttributes, flowFile.getSize());
                                }
                            }
                        }
                    });
                }

                out.flush();
            }
        }
    }) {

        @Override
        public long getContentLength() {
            if (compressionLevel == 0 && !sendAsFlowFile
                    && !context.getProperty(CHUNKED_ENCODING).asBoolean()) {
                return toSend.get(0).getSize();
            } else {
                return -1;
            }
        }
    };

    if (context.getProperty(CHUNKED_ENCODING).isSet()) {
        entity.setChunked(context.getProperty(CHUNKED_ENCODING).asBoolean());
    }
    post.setEntity(entity);
    post.setConfig(requestConfig);

    final String contentType;
    if (sendAsFlowFile) {
        if (accepts.isFlowFileV3Accepted()) {
            contentType = APPLICATION_FLOW_FILE_V3;
        } else if (accepts.isFlowFileV2Accepted()) {
            contentType = APPLICATION_FLOW_FILE_V2;
        } else if (accepts.isFlowFileV1Accepted()) {
            contentType = APPLICATION_FLOW_FILE_V1;
        } else {
            logger.error(
                    "Cannot send data to {} because the destination does not accept FlowFiles and this processor is "
                            + "configured to deliver FlowFiles; rolling back session",
                    new Object[] { url });
            session.rollback();
            context.yield();
            IOUtils.closeQuietly(client);
            return;
        }
    } else {
        final String contentTypeValue = context.getProperty(CONTENT_TYPE)
                .evaluateAttributeExpressions(toSend.get(0)).getValue();
        contentType = StringUtils.isBlank(contentTypeValue) ? DEFAULT_CONTENT_TYPE : contentTypeValue;
    }

    final String attributeHeaderRegex = context.getProperty(ATTRIBUTES_AS_HEADERS_REGEX).getValue();
    if (attributeHeaderRegex != null && !sendAsFlowFile && flowFileList.size() == 1) {
        final Pattern pattern = Pattern.compile(attributeHeaderRegex);

        final Map<String, String> attributes = flowFileList.get(0).getAttributes();
        for (final Map.Entry<String, String> entry : attributes.entrySet()) {
            final String key = entry.getKey();
            if (pattern.matcher(key).matches()) {
                post.setHeader(entry.getKey(), entry.getValue());
            }
        }
    }

    post.setHeader(CONTENT_TYPE_HEADER, contentType);
    post.setHeader(FLOWFILE_CONFIRMATION_HEADER, "true");
    post.setHeader(PROTOCOL_VERSION_HEADER, PROTOCOL_VERSION);
    post.setHeader(TRANSACTION_ID_HEADER, transactionId);
    if (compressionLevel > 0 && accepts.isGzipAccepted()) {
        if (sendAsFlowFile) {
            post.setHeader(GZIPPED_HEADER, "true");
        } else {
            post.setHeader(CONTENT_ENCODING_HEADER, CONTENT_ENCODING_GZIP_VALUE);
        }
    }

    // Do the actual POST
    final String flowFileDescription = toSend.size() <= 10 ? toSend.toString() : toSend.size() + " FlowFiles";

    final String uploadDataRate;
    final long uploadMillis;
    CloseableHttpResponse response = null;
    try {
        final StopWatch stopWatch = new StopWatch(true);
        response = client.execute(post);

        // consume input stream entirely, ignoring its contents. If we
        // don't do this, the Connection will not be returned to the pool
        EntityUtils.consume(response.getEntity());
        stopWatch.stop();
        uploadDataRate = stopWatch.calculateDataRate(bytesToSend);
        uploadMillis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
    } catch (final IOException e) {
        logger.error("Failed to Post {} due to {}; transferring to failure",
                new Object[] { flowFileDescription, e });
        context.yield();
        for (FlowFile flowFile : toSend) {
            flowFile = session.penalize(flowFile);
            session.transfer(flowFile, REL_FAILURE);
        }
        return;
    } finally {
        if (response != null) {
            try {
                response.close();
            } catch (final IOException e) {
                getLogger().warn("Failed to close HTTP Response due to {}", new Object[] { e });
            }
        }
    }

    // If we get a 'SEE OTHER' status code and an HTTP header that indicates that the intent
    // of the Location URI is a flowfile hold, we will store this holdUri. This prevents us
    // from posting to some other webservice and then attempting to delete some resource to which
    // we are redirected
    final int responseCode = response.getStatusLine().getStatusCode();
    final String responseReason = response.getStatusLine().getReasonPhrase();
    String holdUri = null;
    if (responseCode == HttpServletResponse.SC_SEE_OTHER) {
        final Header locationUriHeader = response.getFirstHeader(LOCATION_URI_INTENT_NAME);
        if (locationUriHeader != null) {
            if (LOCATION_URI_INTENT_VALUE.equals(locationUriHeader.getValue())) {
                final Header holdUriHeader = response.getFirstHeader(LOCATION_HEADER_NAME);
                if (holdUriHeader != null) {
                    holdUri = holdUriHeader.getValue();
                }
            }
        }

        if (holdUri == null) {
            for (FlowFile flowFile : toSend) {
                flowFile = session.penalize(flowFile);
                logger.error(
                        "Failed to Post {} to {}: sent content and received status code {}:{} but no Hold URI",
                        new Object[] { flowFile, url, responseCode, responseReason });
                session.transfer(flowFile, REL_FAILURE);
            }
            return;
        }
    }

    if (holdUri == null) {
        if (responseCode == HttpServletResponse.SC_SERVICE_UNAVAILABLE) {
            for (FlowFile flowFile : toSend) {
                flowFile = session.penalize(flowFile);
                logger.error(
                        "Failed to Post {} to {}: response code was {}:{}; will yield processing, "
                                + "since the destination is temporarily unavailable",
                        new Object[] { flowFile, url, responseCode, responseReason });
                session.transfer(flowFile, REL_FAILURE);
            }
            context.yield();
            return;
        }

        if (responseCode >= 300) {
            for (FlowFile flowFile : toSend) {
                flowFile = session.penalize(flowFile);
                logger.error("Failed to Post {} to {}: response code was {}:{}",
                        new Object[] { flowFile, url, responseCode, responseReason });
                session.transfer(flowFile, REL_FAILURE);
            }
            return;
        }

        logger.info("Successfully Posted {} to {} in {} at a rate of {}", new Object[] { flowFileDescription,
                url, FormatUtils.formatMinutesSeconds(uploadMillis, TimeUnit.MILLISECONDS), uploadDataRate });

        for (final FlowFile flowFile : toSend) {
            session.getProvenanceReporter().send(flowFile, url, "Remote DN=" + dnHolder.get(), uploadMillis,
                    true);
            session.transfer(flowFile, REL_SUCCESS);
        }
        return;
    }

    //
    // the response indicated a Hold URI; delete the Hold.
    //
    // determine the full URI of the Flow File's Hold; Unfortunately, the responses that are returned have
    // changed over the past, so we have to take into account a few different possibilities.
    String fullHoldUri = holdUri;
    if (holdUri.startsWith("/contentListener")) {
        // If the Hold URI that we get starts with /contentListener, it may not really be /contentListener,
        // as this really indicates that it should be whatever we posted to -- if posting directly to the
        // ListenHTTP component, it will be /contentListener, but if posting to a proxy/load balancer, we may
        // be posting to some other URL.
        fullHoldUri = url + holdUri.substring(16);
    } else if (holdUri.startsWith("/")) {
        // URL indicates the full path but not hostname or port; use the same hostname & port that we posted
        // to but use the full path indicated by the response.
        int firstSlash = url.indexOf("/", 8);
        if (firstSlash < 0) {
            firstSlash = url.length();
        }
        final String beforeSlash = url.substring(0, firstSlash);
        fullHoldUri = beforeSlash + holdUri;
    } else if (!holdUri.startsWith("http")) {
        // Absolute URL
        fullHoldUri = url + (url.endsWith("/") ? "" : "/") + holdUri;
    }

    final HttpDelete delete = new HttpDelete(fullHoldUri);
    delete.setHeader(TRANSACTION_ID_HEADER, transactionId);

    while (true) {
        try {
            final HttpResponse holdResponse = client.execute(delete);
            EntityUtils.consume(holdResponse.getEntity());
            final int holdStatusCode = holdResponse.getStatusLine().getStatusCode();
            final String holdReason = holdResponse.getStatusLine().getReasonPhrase();
            if (holdStatusCode >= 300) {
                logger.error(
                        "Failed to delete Hold that destination placed on {}: got response code {}:{}; routing to failure",
                        new Object[] { flowFileDescription, holdStatusCode, holdReason });

                for (FlowFile flowFile : toSend) {
                    flowFile = session.penalize(flowFile);
                    session.transfer(flowFile, REL_FAILURE);
                }
                return;
            }

            logger.info("Successfully Posted {} to {} in {} milliseconds at a rate of {}",
                    new Object[] { flowFileDescription, url, uploadMillis, uploadDataRate });

            for (final FlowFile flowFile : toSend) {
                session.getProvenanceReporter().send(flowFile, url);
                session.transfer(flowFile, REL_SUCCESS);
            }
            return;
        } catch (final IOException e) {
            logger.warn("Failed to delete Hold that destination placed on {} due to {}",
                    new Object[] { flowFileDescription, e });
        }

        if (!isScheduled()) {
            context.yield();
            logger.warn(
                    "Failed to delete Hold that destination placed on {}; Processor has been stopped so routing FlowFile(s) to failure",
                    new Object[] { flowFileDescription });
            for (FlowFile flowFile : toSend) {
                flowFile = session.penalize(flowFile);
                session.transfer(flowFile, REL_FAILURE);
            }
            return;
        }
    }
}

From source file:org.apache.zeppelin.submarine.hadoop.YarnClient.java

private static HttpClient buildSpengoHttpClient() {
    HttpClientBuilder builder = HttpClientBuilder.create();
    Lookup<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
            .register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory(true)).build();
    builder.setDefaultAuthSchemeRegistry(authSchemeRegistry);
    BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider();
    credentialsProvider.setCredentials(new AuthScope(null, -1, null), new Credentials() {
        @Override/*from  w ww. j a  va2  s . c  o  m*/
        public Principal getUserPrincipal() {
            return null;
        }

        @Override
        public String getPassword() {
            return null;
        }
    });
    builder.setDefaultCredentialsProvider(credentialsProvider);

    // Avoid output WARN: Cookie rejected
    RequestConfig globalConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.IGNORE_COOKIES).build();
    builder.setDefaultRequestConfig(globalConfig);

    CloseableHttpClient httpClient = builder.build();

    return httpClient;
}

From source file:org.codice.solr.factory.impl.HttpSolrClientFactory.java

@VisibleForTesting
SolrClient createSolrHttpClient(String url, String coreName, String coreUrl)
        throws IOException, SolrServerException {

    final HttpClientBuilder builder = httpClientBuilder.get();
    createSolrCore(url, coreName, null, builder.build());
    try (final Closer closer = new Closer()) {
        final HttpSolrClient noRetryClient = closer
                .with(new HttpSolrClient.Builder(coreUrl).withHttpClient(builder.build()).build());
        final HttpSolrClient retryClient = closer.with(new HttpSolrClient.Builder(coreUrl)
                .withHttpClient(builder.setRetryHandler(new SolrHttpRequestRetryHandler(coreName)).build())
                .build());//from ww w  .ja  v a  2  s.c  o  m

        return closer.returning(new PingAwareSolrClientProxy(retryClient, noRetryClient));
    }
}

From source file:org.codice.solr.factory.SolrClientFactory.java

private static CloseableHttpClient getHttpClient(boolean retryRequestsOnError) {
    SSLConnectionSocketFactory sslConnectionSocketFactory = new SSLConnectionSocketFactory(getSslContext(),
            getProtocols(), getCipherSuites(), SSLConnectionSocketFactory.BROWSER_COMPATIBLE_HOSTNAME_VERIFIER);
    HttpRequestRetryHandler solrRetryHandler = new SolrHttpRequestRetryHandler();

    HttpClientBuilder builder = HttpClients.custom().setSSLSocketFactory(sslConnectionSocketFactory)
            .setDefaultCookieStore(new BasicCookieStore()).setMaxConnTotal(128).setMaxConnPerRoute(32);

    if (retryRequestsOnError) {
        builder.setRetryHandler(solrRetryHandler);
    }/*  w  w  w  .ja v a 2  s .c o m*/

    return builder.build();
}

From source file:org.dasein.cloud.azure.tests.compute.vm.AzureVmTest.java

@Test
public void testCanDeleteDeploymentReturnsFalseWhenDeploymentExists(
        @Mocked final HttpClientBuilder httpClientBuilderMock) throws CloudException {
    final DeploymentModel responseDeploymentModel = new DeploymentModel();
    ArrayList<DeploymentModel.RoleModel> roles = new ArrayList<DeploymentModel.RoleModel>();
    for (Integer i = 1; i < 3; i++) {
        DeploymentModel.RoleModel role = new DeploymentModel.RoleModel();
        role.setRoleName(String.format("SOME_NAME_%s", i));
        roles.add(role);/*  w w w.ja  va2s  .com*/
    }
    responseDeploymentModel.setRoles(roles);

    final CloseableHttpResponse mockedHttpResponse = getHttpResponseMock(
            getStatusLineMock(HttpServletResponse.SC_OK),
            new DaseinObjectToXmlEntity<DeploymentModel>(responseDeploymentModel), new Header[] {});
    final String expectedUrl = String.format(DEPLOYMENT_URL, ENDPOINT, ACCOUNT_NO, SERVICE_NAME,
            DEPLOYMENT_NAME);
    final CloseableHttpClient closeableHttpClient = new MockUp<CloseableHttpClient>() {
        @Mock(invocations = 1)
        CloseableHttpResponse doExecute(HttpHost target, HttpRequest request, HttpContext context)
                throws IOException, ClientProtocolException {
            assertGet((HttpUriRequest) request, expectedUrl,
                    new Header[] { new BasicHeader("x-ms-version", "2014-05-01") });
            return mockedHttpResponse;
        }

    }.getMockInstance();
    new NonStrictExpectations() {
        {
            azureMock.getAzureClientBuilder();
            result = httpClientBuilderMock;
        }
        {
            httpClientBuilderMock.build();
            result = closeableHttpClient;
        }
    };

    AzureVM azureVM = new AzureVM(azureMock);
    boolean actualReturnValue = invoke(azureVM, "canDeleteDeployment", SERVICE_NAME, DEPLOYMENT_NAME,
            ROLE_NAME);
    Assert.assertFalse(actualReturnValue);
}

From source file:org.dasein.cloud.azure.tests.compute.vm.AzureVmTest.java

@Test
public void testCanDeleteDeploymentReturnsTrueWhenNoRoles(@Mocked final HttpClientBuilder httpClientBuilderMock)
        throws CloudException {
    final DeploymentModel responseDeploymentModel = new DeploymentModel();

    final CloseableHttpResponse mockedHttpResponse = getHttpResponseMock(
            getStatusLineMock(HttpServletResponse.SC_OK),
            new DaseinObjectToXmlEntity<DeploymentModel>(responseDeploymentModel), new Header[] {});
    final String expectedUrl = String.format(DEPLOYMENT_URL, ENDPOINT, ACCOUNT_NO, SERVICE_NAME,
            DEPLOYMENT_NAME);/*from  w  ww  . j  av  a2s.co m*/
    final CloseableHttpClient closeableHttpClient = new MockUp<CloseableHttpClient>() {
        @Mock(invocations = 1)
        CloseableHttpResponse doExecute(HttpHost target, HttpRequest request, HttpContext context)
                throws IOException, ClientProtocolException {
            assertGet((HttpUriRequest) request, expectedUrl,
                    new Header[] { new BasicHeader("x-ms-version", "2014-05-01") });
            return mockedHttpResponse;
        }

    }.getMockInstance();
    new NonStrictExpectations() {
        {
            azureMock.getAzureClientBuilder();
            result = httpClientBuilderMock;
        }
        {
            httpClientBuilderMock.build();
            result = closeableHttpClient;
        }
    };

    AzureVM azureVM = new AzureVM(azureMock);
    boolean actualReturnValue = invoke(azureVM, "canDeleteDeployment", SERVICE_NAME, DEPLOYMENT_NAME,
            ROLE_NAME);
    Assert.assertTrue(actualReturnValue);
}

From source file:org.dasein.cloud.azure.tests.compute.vm.AzureVmTest.java

@Test
public void testCanDeleteDeploymentReturnsTrueWhenTheCorrectRoleExists(
        @Mocked final HttpClientBuilder httpClientBuilderMock) throws CloudException {
    final DeploymentModel responseDeploymentModel = new DeploymentModel();
    final DeploymentModel.RoleModel role = new DeploymentModel.RoleModel();
    role.setRoleName(ROLE_NAME);/*w w w .  ja  va  2s .c o m*/
    responseDeploymentModel.setRoles(new ArrayList<DeploymentModel.RoleModel>() {
        {
            add(role);
        }
    });

    final CloseableHttpResponse mockedHttpResponse = getHttpResponseMock(
            getStatusLineMock(HttpServletResponse.SC_OK),
            new DaseinObjectToXmlEntity<DeploymentModel>(responseDeploymentModel), new Header[] {});
    final String expectedUrl = String.format(DEPLOYMENT_URL, ENDPOINT, ACCOUNT_NO, SERVICE_NAME,
            DEPLOYMENT_NAME);
    final CloseableHttpClient closeableHttpClient = new MockUp<CloseableHttpClient>() {
        @Mock(invocations = 1)
        CloseableHttpResponse doExecute(HttpHost target, HttpRequest request, HttpContext context)
                throws IOException, ClientProtocolException {
            assertGet((HttpUriRequest) request, expectedUrl,
                    new Header[] { new BasicHeader("x-ms-version", "2014-05-01") });
            return mockedHttpResponse;
        }

    }.getMockInstance();
    new NonStrictExpectations() {
        {
            azureMock.getAzureClientBuilder();
            result = httpClientBuilderMock;
        }
        {
            httpClientBuilderMock.build();
            result = closeableHttpClient;
        }
    };

    AzureVM azureVM = new AzureVM(azureMock);
    boolean actualReturnValue = invoke(azureVM, "canDeleteDeployment", SERVICE_NAME, DEPLOYMENT_NAME,
            ROLE_NAME);
    Assert.assertTrue(actualReturnValue);
}

From source file:org.dasein.cloud.azure.tests.compute.vm.AzureVmTest.java

@Test
public void testCanDeleteDeploymentReturnsFalseWhenIncorrectRoleExists(
        @Mocked final HttpClientBuilder httpClientBuilderMock) throws CloudException {
    final DeploymentModel responseDeploymentModel = new DeploymentModel();
    final DeploymentModel.RoleModel role = new DeploymentModel.RoleModel();
    role.setRoleName("SOME_ROLE");
    responseDeploymentModel.setRoles(new ArrayList<DeploymentModel.RoleModel>() {
        {/*from w  w  w. java 2  s.  c  o  m*/
            add(role);
        }
    });

    final CloseableHttpResponse mockedHttpResponse = getHttpResponseMock(
            getStatusLineMock(HttpServletResponse.SC_OK),
            new DaseinObjectToXmlEntity<DeploymentModel>(responseDeploymentModel), new Header[] {});
    final String expectedUrl = String.format(DEPLOYMENT_URL, ENDPOINT, ACCOUNT_NO, SERVICE_NAME,
            DEPLOYMENT_NAME);
    final CloseableHttpClient closeableHttpClient = new MockUp<CloseableHttpClient>() {
        @Mock(invocations = 1)
        CloseableHttpResponse doExecute(HttpHost target, HttpRequest request, HttpContext context)
                throws IOException, ClientProtocolException {
            assertGet((HttpUriRequest) request, expectedUrl,
                    new Header[] { new BasicHeader("x-ms-version", "2014-05-01") });
            return mockedHttpResponse;
        }

    }.getMockInstance();
    new NonStrictExpectations() {
        {
            azureMock.getAzureClientBuilder();
            result = httpClientBuilderMock;
        }
        {
            httpClientBuilderMock.build();
            result = closeableHttpClient;
        }
    };

    AzureVM azureVM = new AzureVM(azureMock);
    boolean actualReturnValue = invoke(azureVM, "canDeleteDeployment", SERVICE_NAME, DEPLOYMENT_NAME,
            ROLE_NAME);
    Assert.assertFalse(actualReturnValue);
}