com.subgraph.vega.internal.http.requests.RequestTask.java Source code

Java tutorial

Introduction

Here is the source code for com.subgraph.vega.internal.http.requests.RequestTask.java

Source

/*******************************************************************************
 * Copyright (c) 2011 Subgraph.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 * 
 * Contributors:
 *     Subgraph - initial API and implementation
 ******************************************************************************/
package com.subgraph.vega.internal.http.requests;

import java.io.IOException;
import java.io.InputStream;
import java.util.concurrent.Callable;
import java.util.logging.Logger;
import java.util.zip.GZIPInputStream;

import org.apache.http.Header;
import org.apache.http.HeaderElement;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpRequest;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.entity.ByteArrayEntity;
import org.apache.http.protocol.ExecutionContext;
import org.apache.http.protocol.HTTP;
import org.apache.http.protocol.HttpContext;

import com.subgraph.vega.api.html.IHTMLParser;
import com.subgraph.vega.api.http.requests.IHttpRequestEngineConfig;
import com.subgraph.vega.api.http.requests.IHttpResponse;
import com.subgraph.vega.api.http.requests.IHttpResponseProcessor;

class RequestTask implements Callable<IHttpResponse> {
    private final static Logger logger = Logger.getLogger("request-engine");

    private final HttpClient client;
    private final RateLimiter rateLimit;
    private final HttpUriRequest request;
    private final HttpContext context;
    private final IHttpRequestEngineConfig config;
    private final IHTMLParser htmlParser;

    RequestTask(HttpClient client, RateLimiter rateLimit, HttpUriRequest request, HttpContext context,
            IHttpRequestEngineConfig config, IHTMLParser htmlParser) {
        this.client = client;
        this.rateLimit = rateLimit;
        this.request = request;
        this.context = context;
        this.config = config;
        this.htmlParser = htmlParser;
    }

    @Override
    public IHttpResponse call() throws Exception {
        if (config.getForceIdentityEncoding())
            request.setHeader(HTTP.CONTENT_ENCODING, HTTP.IDENTITY_CODING);

        if (rateLimit != null)
            rateLimit.maybeDelayRequest();

        final long start = System.currentTimeMillis();
        final HttpResponse httpResponse = client.execute(request, context);
        final long elapsed = System.currentTimeMillis() - start;

        final HttpEntity entity = httpResponse.getEntity();

        if (entity != null) {
            if (config.getMaximumResponseKilobytes() > 0
                    && entity.getContentLength() > (config.getMaximumResponseKilobytes() * 1024)) {
                logger.warning("Aborting request " + request.getURI().toString() + " because response length "
                        + entity.getContentLength() + " exceeds maximum length of "
                        + config.getMaximumResponseKilobytes() + " kb.");
                request.abort();
                httpResponse.setEntity(createEmptyEntity());
            }

            final HttpEntity newEntity = processEntity(httpResponse, entity);
            httpResponse.setEntity(newEntity);
        }
        final HttpHost host = (HttpHost) context.getAttribute(ExecutionContext.HTTP_TARGET_HOST);
        final HttpRequest sentRequest = (HttpRequest) context.getAttribute(HttpRequestEngine.VEGA_SENT_REQUEST);

        final IHttpResponse response = new EngineHttpResponse(request.getURI(), host,
                (sentRequest == null) ? (request) : (sentRequest), httpResponse, elapsed, htmlParser);

        for (IHttpResponseProcessor p : config.getResponseProcessors()) {
            p.processResponse(response.getOriginalRequest(), response, context);
        }

        return response;
    }

    private HttpEntity processEntity(HttpResponse response, HttpEntity entity) throws IOException {
        if (entity == null)
            return null;

        if (isGzipEncoded(entity) && config.getDecompressGzipEncoding())
            return processGzipEncodedEntity(response, entity);

        final InputStream input = entity.getContent();

        if (input == null) {
            return createEmptyEntity();
        }

        String contentType = (entity.getContentType() == null) ? (null) : (entity.getContentType().getValue());
        String contentEncoding = (entity.getContentEncoding() == null) ? (null)
                : (entity.getContentEncoding().getValue());
        RepeatableStreamingEntity e = new RepeatableStreamingEntity(input, entity.getContentLength(), false,
                entity.isChunked(), contentType, contentEncoding);
        if (config.getMaximumResponseKilobytes() > 0) {
            e.setMaximumInputKilobytes(config.getMaximumResponseKilobytes());
        }
        return e;
    }

    private HttpEntity createEmptyEntity() {
        return new ByteArrayEntity(new byte[0]);
    }

    private HttpEntity processGzipEncodedEntity(HttpResponse response, HttpEntity entity) throws IOException {
        final InputStream input = entity.getContent();
        if (input == null) {
            response.setHeader(HTTP.CONTENT_LEN, "0");
            return new ByteArrayEntity(new byte[0]);
        }
        final InputStream gzipInput = new GZIPInputStream(input);
        response.removeHeaders(HTTP.CONTENT_ENCODING);
        String contentType = (entity.getContentType() == null) ? (null) : (entity.getContentType().getValue());
        RepeatableStreamingEntity newEntity = new RepeatableStreamingEntity(gzipInput, -1, true, entity.isChunked(),
                contentType, null);
        response.setHeader(HTTP.CONTENT_LEN, Long.toString(newEntity.getContentLength()));
        if (config.getMaximumResponseKilobytes() > 0) {
            newEntity.setMaximumInputKilobytes(config.getMaximumResponseKilobytes());
        }
        return newEntity;
    }

    private boolean isGzipEncoded(HttpEntity entity) {
        if (entity == null)
            return false;
        final Header ceh = entity.getContentEncoding();
        if (ceh == null)
            return false;
        for (HeaderElement element : ceh.getElements()) {
            if ("gzip".equalsIgnoreCase(element.getName()))
                return true;
        }
        return false;
    }
}