com.subgraph.vega.internal.crawler.RequestConsumer.java Source code

Introduction

Here is the source code for com.subgraph.vega.internal.crawler.RequestConsumer.java

Source

/*******************************************************************************
 * Copyright (c) 2011 Subgraph.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 * 
 * Contributors:
 *     Subgraph - initial API and implementation
 ******************************************************************************/
package com.subgraph.vega.internal.crawler;

import java.io.InterruptedIOException;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.CountDownLatch;
import java.util.logging.Level;
import java.util.logging.Logger;

import org.apache.http.client.methods.HttpUriRequest;

import com.subgraph.vega.api.http.requests.IHttpRequestEngine;
import com.subgraph.vega.api.http.requests.IHttpResponse;
import com.subgraph.vega.api.http.requests.RequestEngineException;

public class RequestConsumer implements Runnable {
    private final Logger logger = Logger.getLogger("crawler");
    private final IHttpRequestEngine requestEngine;
    private final BlockingQueue<CrawlerTask> crawlerRequestQueue;
    private final BlockingQueue<CrawlerTask> crawlerResponseQueue;
    private final CountDownLatch latch;
    private volatile boolean stop;
    private final Object requestLock = new Object();
    private volatile HttpUriRequest activeRequest = null;

    RequestConsumer(IHttpRequestEngine requestEngine, BlockingQueue<CrawlerTask> requestQueue,
            BlockingQueue<CrawlerTask> responseQueue, CountDownLatch latch) {
        this.requestEngine = requestEngine;
        this.crawlerRequestQueue = requestQueue;
        this.crawlerResponseQueue = responseQueue;
        this.latch = latch;
    }

    @Override
    public void run() {
        try {
            runLoop();
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
        } finally {
            latch.countDown();
        }
    }

    void stop() {
        stop = true;
        synchronized (requestLock) {
            if (activeRequest != null)
                activeRequest.abort();
        }
    }

    private void runLoop() throws InterruptedException {
        while (!stop) {
            CrawlerTask task = (CrawlerTask) crawlerRequestQueue.take();

            if (task.isExitTask()) {
                // Put poison pill back in queue so every other RequestConsumer task will see it.
                crawlerRequestQueue.add(task);
                return;
            }

            logger.info("Retrieving: " + task.getRequest().getRequestLine().getUri());

            if (!sendRequest(task)) {
                if (!stop && !task.causedException()) {
                    logger.log(Level.WARNING,
                            "No response was receiven for request to " + task.getRequest().getURI());
                }
            }
            crawlerResponseQueue.put(task);
        }
    }

    private boolean sendRequest(CrawlerTask task) {
        try {
            activeRequest = task.getRequest();
            final IHttpResponse response = requestEngine.sendRequest(task.getRequest());
            task.setResponse(response);
            return response != null;
        } catch (RequestEngineException e) {
            if (e.getCause() instanceof InterruptedIOException) {
                stop = true;
                return false;
            }
            if (e.getMessage().contains("abort")) {
                return false;
            }
            task.setException(e);
            logger.log(Level.WARNING, e.getMessage());
            return false;
        } finally {
            synchronized (requestLock) {
                activeRequest = null;
            }
        }
    }
}