Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.iflytek.spider.protocol.http; // JDK imports import java.io.IOException; import java.net.InetAddress; import java.net.URL; import java.net.UnknownHostException; import java.util.HashMap; import java.util.LinkedList; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import com.iflytek.spider.crawl.CrawlDatum; import com.iflytek.spider.protocol.Content; import com.iflytek.spider.protocol.Protocol; import com.iflytek.spider.protocol.ProtocolException; import com.iflytek.spider.protocol.ProtocolOutput; import com.iflytek.spider.protocol.ProtocolStatus; import com.iflytek.spider.protocol.Response; import com.iflytek.spider.util.DeflateUtils; import com.iflytek.spider.util.GZIPUtils; import com.iflytek.spider.util.LogUtil; /** * @author Jérôme Charron */ public abstract class HttpBase implements Protocol { public static final int BUFFER_SIZE = 8 * 1024; private static final byte[] EMPTY_CONTENT = new byte[0]; /** The proxy hostname. */ protected String proxyHost = null; /** The proxy port. */ protected int proxyPort = 8080; /** Indicates if a proxy is used */ protected boolean useProxy = false; /** The network timeout in millisecond */ protected int timeout = 10000; /** The length limit for downloaded content, in bytes. */ protected int maxContent = 64 * 1024; /** The number of times a thread will delay when trying to fetch a page. */ protected int maxDelays = 3; /** * The maximum number of threads that should be allowed to access a host at * one time. */ protected int maxThreadsPerHost = 1; /** * The number of seconds the fetcher will delay between successive requests * to the same server. */ protected long serverDelay = 1000; /** The Nutch 'User-Agent' request header */ protected String userAgent = getAgentString("NutchCVS", null, "Nutch", "http://lucene.apache.org/nutch/bot.html", "nutch-agent@lucene.apache.org"); /** The "Accept-Language" request header value. */ protected String acceptLanguage = "en-us,en-gb,en;q=0.7,*;q=0.3"; /** * Maps from host to a Long naming the time it should be unblocked. The Long * is zero while the host is in use, then set to now+wait when a request * finishes. This way only one thread at a time accesses a host. */ private static HashMap BLOCKED_ADDR_TO_TIME = new HashMap(); /** * Maps a host to the number of threads accessing that host. */ private static HashMap THREADS_PER_HOST_COUNT = new HashMap(); /** * Queue of blocked hosts. This contains all of the non-zero entries from * BLOCKED_ADDR_TO_TIME, ordered by increasing time. */ private static LinkedList BLOCKED_ADDR_QUEUE = new LinkedList(); /** The default logger */ private final static Log LOGGER = LogFactory.getLog(HttpBase.class); /** The specified logger */ private Log logger = LOGGER; /** The nutch configuration */ private Configuration conf = null; /** Do we block by IP addresses or by hostnames? */ private boolean byIP = true; /** Do we use HTTP/1.1? */ protected boolean useHttp11 = false; /** Skip page if Crawl-Delay longer than this value. */ protected long maxCrawlDelay = -1L; /** Plugin should handle host blocking internally. */ protected boolean checkBlocking = true; /** Creates a new instance of HttpBase */ public HttpBase() { this(null); } /** Creates a new instance of HttpBase */ public HttpBase(Log logger) { if (logger != null) { this.logger = logger; } } // Inherited Javadoc public void setConf(Configuration conf) { this.conf = conf; this.proxyHost = conf.get("http.proxy.host"); this.proxyPort = conf.getInt("http.proxy.port", 8080); this.useProxy = (proxyHost != null && proxyHost.length() > 0); this.timeout = conf.getInt("http.timeout", 10000); this.maxContent = conf.getInt("http.content.limit", 64 * 1024); this.maxDelays = conf.getInt("http.max.delays", 3); this.maxThreadsPerHost = conf.getInt("fetcher.threads.per.host", 1); this.userAgent = getAgentString(conf.get("http.agent.name"), conf.get("http.agent.version"), conf.get("http.agent.description"), conf.get("http.agent.url"), conf.get("http.agent.email")); this.acceptLanguage = conf.get("http.accept.language", acceptLanguage); this.serverDelay = (long) (conf.getFloat("fetcher.server.delay", 1.0f) * 1000); this.maxCrawlDelay = (long) (conf.getInt("fetcher.max.crawl.delay", -1) * 1000); // backward-compatible default setting this.byIP = conf.getBoolean("fetcher.threads.per.host.by.ip", true); this.useHttp11 = conf.getBoolean("http.useHttp11", false); //logConf(); } // Inherited Javadoc public Configuration getConf() { return this.conf; } public ProtocolOutput getProtocolOutput(String url, CrawlDatum datum) { String urlString = url; try { URL u = new URL(urlString); long delay = serverDelay; if (checkBlocking && maxCrawlDelay >= 0 && delay > maxCrawlDelay) { // skip this page, otherwise the thread would block for too // long. LOGGER.info("Skipping: " + u + " exceeds fetcher.max.crawl.delay, max=" + (maxCrawlDelay / 1000) + ", Crawl-Delay=" + (delay / 1000)); return new ProtocolOutput(null, ProtocolStatus.STATUS_WOULDBLOCK); } String host = null; if (checkBlocking) { try { host = blockAddr(u, delay); } catch (BlockedException be) { return new ProtocolOutput(null, ProtocolStatus.STATUS_BLOCKED); } } Response response; try { response = getResponse(u, datum, false); // make a request } finally { if (checkBlocking) unblockAddr(host, delay); } int code = response.getCode(); byte[] content = response.getContent(); Content c = new Content(u.toString(), (content == null ? EMPTY_CONTENT : content), response.getHeaders()); if (code == 200) { // got a good response return new ProtocolOutput(c); // return it } else if (code == 410) { // page is gone return new ProtocolOutput(c, new ProtocolStatus(ProtocolStatus.GONE, "Http: " + code + " url=" + url)); } else if (code >= 300 && code < 400) { // handle redirect String location = response.getHeader("Location"); // some broken servers, such as MS IIS, use lowercase header // name... if (location == null) location = response.getHeader("location"); if (location == null) location = ""; u = new URL(u, location); int protocolStatusCode; switch (code) { case 300: // multiple choices, preferred value in Location protocolStatusCode = ProtocolStatus.MOVED; break; case 301: // moved permanently case 305: // use proxy (Location is URL of proxy) protocolStatusCode = ProtocolStatus.MOVED; break; case 302: // found (temporarily moved) case 303: // see other (redirect after POST) case 307: // temporary redirect protocolStatusCode = ProtocolStatus.TEMP_MOVED; break; case 304: // not modified protocolStatusCode = ProtocolStatus.NOTMODIFIED; break; default: protocolStatusCode = ProtocolStatus.MOVED; } // handle this in the higher layer. return new ProtocolOutput(c, new ProtocolStatus(protocolStatusCode, u)); } else if (code == 400) { // bad request, mark as GONE if (logger.isTraceEnabled()) { logger.trace("400 Bad request: " + u); } return new ProtocolOutput(c, new ProtocolStatus(ProtocolStatus.GONE, u)); } else if (code == 401) { // requires authorization, but no valid // auth provided. if (logger.isTraceEnabled()) { logger.trace("401 Authentication Required"); } return new ProtocolOutput(c, new ProtocolStatus(ProtocolStatus.ACCESS_DENIED, "Authentication required: " + urlString)); } else if (code == 404) { return new ProtocolOutput(c, new ProtocolStatus(ProtocolStatus.NOTFOUND, u)); } else if (code == 410) { // permanently GONE return new ProtocolOutput(c, new ProtocolStatus(ProtocolStatus.GONE, u)); } else { return new ProtocolOutput(c, new ProtocolStatus(ProtocolStatus.EXCEPTION, "Http code=" + code + ", url=" + u)); } } catch (Throwable e) { e.printStackTrace(LogUtil.getErrorStream(logger)); return new ProtocolOutput(null, new ProtocolStatus(e)); } } /* * -------------------------- * </implementation:Protocol> * * -------------------------- */ public String getProxyHost() { return proxyHost; } public int getProxyPort() { return proxyPort; } public boolean useProxy() { return useProxy; } public int getTimeout() { return timeout; } public int getMaxContent() { return maxContent; } public int getMaxDelays() { return maxDelays; } public int getMaxThreadsPerHost() { return maxThreadsPerHost; } public long getServerDelay() { return serverDelay; } public String getUserAgent() { return userAgent; } /** * Value of "Accept-Language" request header sent by Nutch. * * @return The value of the header "Accept-Language" header. */ public String getAcceptLanguage() { return acceptLanguage; } public boolean getUseHttp11() { return useHttp11; } private String blockAddr(URL url, long crawlDelay) throws ProtocolException { String host; if (byIP) { try { InetAddress addr = InetAddress.getByName(url.getHost()); host = addr.getHostAddress(); } catch (UnknownHostException e) { // unable to resolve it, so don't fall back to host name throw new HttpException(e); } } else { host = url.getHost(); if (host == null) throw new HttpException("Unknown host for url: " + url); host = host.toLowerCase(); } int delays = 0; while (true) { cleanExpiredServerBlocks(); // free held addresses Long time; synchronized (BLOCKED_ADDR_TO_TIME) { time = (Long) BLOCKED_ADDR_TO_TIME.get(host); if (time == null) { // address is free // get # of threads already accessing this addr Integer counter = (Integer) THREADS_PER_HOST_COUNT.get(host); int count = (counter == null) ? 0 : counter.intValue(); count++; // increment & store THREADS_PER_HOST_COUNT.put(host, new Integer(count)); if (count >= maxThreadsPerHost) { BLOCKED_ADDR_TO_TIME.put(host, new Long(0)); // block it } return host; } } if (delays == maxDelays) throw new BlockedException("Exceeded http.max.delays: retry later."); long done = time.longValue(); long now = System.currentTimeMillis(); long sleep = 0; if (done == 0) { // address is still in use sleep = crawlDelay; // wait at least delay } else if (now < done) { // address is on hold sleep = done - now; // wait until its free } try { Thread.sleep(sleep); } catch (InterruptedException e) { } delays++; } } private void unblockAddr(String host, long crawlDelay) { synchronized (BLOCKED_ADDR_TO_TIME) { int addrCount = ((Integer) THREADS_PER_HOST_COUNT.get(host)).intValue(); if (addrCount == 1) { THREADS_PER_HOST_COUNT.remove(host); BLOCKED_ADDR_QUEUE.addFirst(host); BLOCKED_ADDR_TO_TIME.put(host, new Long(System.currentTimeMillis() + crawlDelay)); } else { THREADS_PER_HOST_COUNT.put(host, new Integer(addrCount - 1)); } } } private static void cleanExpiredServerBlocks() { synchronized (BLOCKED_ADDR_TO_TIME) { for (int i = BLOCKED_ADDR_QUEUE.size() - 1; i >= 0; i--) { String host = (String) BLOCKED_ADDR_QUEUE.get(i); long time = ((Long) BLOCKED_ADDR_TO_TIME.get(host)).longValue(); if (time <= System.currentTimeMillis()) { BLOCKED_ADDR_TO_TIME.remove(host); BLOCKED_ADDR_QUEUE.remove(i); } } } } private static String getAgentString(String agentName, String agentVersion, String agentDesc, String agentURL, String agentEmail) { if ((agentName == null) || (agentName.trim().length() == 0)) { // TODO : NUTCH-258 if (LOGGER.isFatalEnabled()) { LOGGER.fatal("No User-Agent string set (http.agent.name)!"); } } StringBuffer buf = new StringBuffer(); buf.append(agentName); if (agentVersion != null) { buf.append("/"); buf.append(agentVersion); } if (((agentDesc != null) && (agentDesc.length() != 0)) || ((agentEmail != null) && (agentEmail.length() != 0)) || ((agentURL != null) && (agentURL.length() != 0))) { buf.append(" ("); if ((agentDesc != null) && (agentDesc.length() != 0)) { buf.append(agentDesc); if ((agentURL != null) || (agentEmail != null)) buf.append("; "); } if ((agentURL != null) && (agentURL.length() != 0)) { buf.append(agentURL); if (agentEmail != null) buf.append("; "); } if ((agentEmail != null) && (agentEmail.length() != 0)) buf.append(agentEmail); buf.append(")"); } return buf.toString(); } protected void logConf() { if (logger.isInfoEnabled()) { logger.info("http.proxy.host = " + proxyHost); logger.info("http.proxy.port = " + proxyPort); logger.info("http.timeout = " + timeout); logger.info("http.content.limit = " + maxContent); logger.info("http.agent = " + userAgent); logger.info("http.accept.language = " + acceptLanguage); if (checkBlocking) { logger.info("fetcher.server.delay = " + serverDelay); logger.info("http.max.delays = " + maxDelays); } } } public byte[] processGzipEncoded(byte[] compressed, URL url) throws IOException { if (LOGGER.isTraceEnabled()) { LOGGER.trace("uncompressing...."); } byte[] content; if (getMaxContent() >= 0) { content = GZIPUtils.unzipBestEffort(compressed, getMaxContent()); } else { content = GZIPUtils.unzipBestEffort(compressed); } if (content == null) throw new IOException("unzipBestEffort returned null"); if (LOGGER.isTraceEnabled()) { LOGGER.trace("fetched " + compressed.length + " bytes of compressed content (expanded to " + content.length + " bytes) from " + url); } return content; } public byte[] processDeflateEncoded(byte[] compressed, URL url) throws IOException { if (LOGGER.isTraceEnabled()) { LOGGER.trace("inflating...."); } byte[] content = DeflateUtils.inflateBestEffort(compressed, getMaxContent()); if (content == null) throw new IOException("inflateBestEffort returned null"); if (LOGGER.isTraceEnabled()) { LOGGER.trace("fetched " + compressed.length + " bytes of compressed content (expanded to " + content.length + " bytes) from " + url); } return content; } protected static void main(HttpBase http, String[] args) throws Exception { boolean verbose = false; String url = null; String usage = "Usage: Http [-verbose] [-timeout N] url"; if (args.length == 0) { System.err.println(usage); System.exit(-1); } for (int i = 0; i < args.length; i++) { // parse command line if (args[i].equals("-timeout")) { // found -timeout option http.timeout = Integer.parseInt(args[++i]) * 1000; } else if (args[i].equals("-verbose")) { // found -verbose option verbose = true; } else if (i != args.length - 1) { System.err.println(usage); System.exit(-1); } else // root is required parameter url = args[i]; } // if (verbose) { // LOGGER.setLevel(Level.FINE); // } ProtocolOutput out = http.getProtocolOutput(url, new CrawlDatum()); Content content = out.getContent(); System.out.println("Status: " + out.getStatus()); if (content != null) { System.out.println("Content Length: " + content.getMetadata().get(Response.CONTENT_LENGTH)); System.out.println("Content:"); String text = new String(content.getContent()); System.out.println(text); } } public ProtocolOutput getProtocolOutput(String url) { return getProtocolOutput(url, null); } protected abstract Response getResponse(URL url, CrawlDatum datum, boolean followRedirects) throws ProtocolException, IOException; }