Java tutorial
/* $Id: ExperimentalWARCWriterProcessor.java 4935 2007-02-23 00:27:24Z gojomo $ * * Created on August 1st, 2006. * * Copyright (C) 2006 Internet Archive. * * This file is part of the Heritrix web crawler (crawler.archive.org). * * Heritrix is free software; you can redistribute it and/or modify * it under the terms of the GNU Lesser Public License as published by * the Free Software Foundation; either version 2.1 of the License, or * any later version. * * Heritrix is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser Public License for more details. * * You should have received a copy of the GNU Lesser Public License * along with Heritrix; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ package com.cyberway.issue.crawler.writer; import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; import java.net.InetAddress; import java.net.URI; import java.net.URISyntaxException; import java.net.UnknownHostException; import java.text.ParseException; import java.util.Collection; import java.util.Date; import java.util.HashMap; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; import java.util.logging.Logger; import org.apache.commons.httpclient.Header; import org.apache.commons.httpclient.HttpMethodBase; import org.apache.commons.httpclient.HttpStatus; import org.apache.commons.lang.StringUtils; import com.cyberway.issue.crawler.Heritrix; import com.cyberway.issue.crawler.datamodel.CoreAttributeConstants; import com.cyberway.issue.crawler.datamodel.CrawlURI; import com.cyberway.issue.crawler.datamodel.FetchStatusCodes; import com.cyberway.issue.crawler.deciderules.recrawl.IdenticalDigestDecideRule; import com.cyberway.issue.crawler.event.CrawlStatusListener; import com.cyberway.issue.crawler.extractor.Link; import com.cyberway.issue.crawler.framework.WriterPoolProcessor; import com.cyberway.issue.crawler.settings.SimpleType; import com.cyberway.issue.crawler.settings.Type; import com.cyberway.issue.io.ReplayInputStream; import com.cyberway.issue.io.WriterPoolMember; import com.cyberway.issue.io.WriterPoolSettings; import com.cyberway.issue.io.warc.WARCWriter; import com.cyberway.issue.io.warc.WARCConstants; import com.cyberway.issue.io.warc.WARCWriterPool; import com.cyberway.issue.uid.GeneratorFactory; import com.cyberway.issue.util.ArchiveUtils; import com.cyberway.issue.util.XmlUtils; import com.cyberway.issue.util.anvl.ANVLRecord; import org.w3c.dom.Document; /** * WARCWriterProcessor. * Goes against the 0.18 version of the WARC specification (which * is functionally identical to 0.17 except in the protocol * identifier string). * See http://archive-access.sourceforge.net/warc/ * * <p>TODO: Remove ANVLRecord. Rename NameValue or use RFC822 * (commons-httpclient?) or find something else. * * @author stack */ public class WARCWriterProcessor extends WriterPoolProcessor implements CoreAttributeConstants, CrawlStatusListener, WriterPoolSettings, FetchStatusCodes, WARCConstants { private static final long serialVersionUID = 6182850087635847443L; private final Logger logger = Logger.getLogger(this.getClass().getName()); /** * Key for whether to write 'request' type records where possible */ public static final String ATTR_WRITE_REQUESTS = "write-requests"; /** * Key for whether to write 'metadata' type records where possible */ public static final String ATTR_WRITE_METADATA = "write-metadata"; /** * Key for whether to write 'revisit' type records when * consecutive identical digest */ public static final String ATTR_WRITE_REVISIT_FOR_IDENTICAL_DIGESTS = "write-revisit-for-identical-digests"; /** * Key for whether to write 'revisit' type records for server * "304 not modified" responses */ public static final String ATTR_WRITE_REVISIT_FOR_NOT_MODIFIED = "write-revisit-for-not-modified"; /** * Default path list. */ private static final String[] DEFAULT_PATH = { "warcs" }; protected String[] getDefaultPath() { return DEFAULT_PATH; } /** * @param name Name of this writer. */ public WARCWriterProcessor(final String name) { super(name, "Experimental WARCWriter processor (Version 0.17)"); Type e = addElementToDefinition(new SimpleType(ATTR_WRITE_REQUESTS, "Whether to write 'request' type records. " + "Default is true.", new Boolean(true))); e.setOverrideable(true); e.setExpertSetting(true); e = addElementToDefinition(new SimpleType(ATTR_WRITE_METADATA, "Whether to write 'metadata' type records. " + "Default is true.", new Boolean(true))); e.setOverrideable(true); e.setExpertSetting(true); e = addElementToDefinition(new SimpleType(ATTR_WRITE_REVISIT_FOR_IDENTICAL_DIGESTS, "Whether to write 'revisit' type records when a URI's " + "history indicates the previous fetch had an identical " + "content digest. " + "Default is true.", new Boolean(true))); e.setOverrideable(true); e.setExpertSetting(true); e = addElementToDefinition( new SimpleType(ATTR_WRITE_REVISIT_FOR_NOT_MODIFIED, "Whether to write 'revisit' type records when a " + "304-Not Modified response is received. " + "Default is true.", new Boolean(true))); e.setOverrideable(true); e.setExpertSetting(true); } protected void setupPool(final AtomicInteger serialNo) { setPool(new WARCWriterPool(serialNo, this, getPoolMaximumActive(), getPoolMaximumWait())); } /** * Writes a CrawlURI and its associated data to store file. * * Currently this method understands the following uri types: dns, http, and * https. * * @param curi CrawlURI to process. * */ protected void innerProcess(CrawlURI curi) { // If failure, or we haven't fetched the resource yet, return if (curi.getFetchStatus() <= 0) { return; } // If no recorded content at all, don't write record. long recordLength = curi.getContentSize(); if (recordLength <= 0) { // getContentSize() should be > 0 if any material (even just // HTTP headers with zero-length body) is available. return; } String scheme = curi.getUURI().getScheme().toLowerCase(); try { if (shouldWrite(curi)) { write(scheme, curi); } else { logger.info("This writer does not write out scheme " + scheme + " content"); } } catch (IOException e) { curi.addLocalizedError(this.getName(), e, "WriteRecord: " + curi.toString()); logger.log(Level.SEVERE, "Failed write of Record: " + curi.toString(), e); } } protected void write(final String lowerCaseScheme, final CrawlURI curi) throws IOException { WriterPoolMember writer = getPool().borrowFile(); long position = writer.getPosition(); // See if we need to open a new file because we've exceeed maxBytes. // Call to checkFileSize will open new file if we're at maximum for // current file. writer.checkSize(); if (writer.getPosition() != position) { // We just closed the file because it was larger than maxBytes. // Add to the totalBytesWritten the size of the first record // in the file, if any. setTotalBytesWritten(getTotalBytesWritten() + (writer.getPosition() - position)); position = writer.getPosition(); } WARCWriter w = (WARCWriter) writer; try { // Write a request, response, and metadata all in the one // 'transaction'. final URI baseid = getRecordID(); final String timestamp = ArchiveUtils.getLog14Date(curi.getLong(A_FETCH_BEGAN_TIME)); if (lowerCaseScheme.startsWith("http")) { // Add named fields for ip, checksum, and relate the metadata // and request to the resource field. // TODO: Use other than ANVL (or rename ANVL as NameValue or // use RFC822 (commons-httpclient?). ANVLRecord headers = new ANVLRecord(5); if (curi.getContentDigest() != null) { headers.addLabelValue(HEADER_KEY_PAYLOAD_DIGEST, curi.getContentDigestSchemeString()); } headers.addLabelValue(HEADER_KEY_IP, getHostAddress(curi)); URI rid; if (IdenticalDigestDecideRule.hasIdenticalDigest(curi) && ((Boolean) getUncheckedAttribute(curi, ATTR_WRITE_REVISIT_FOR_IDENTICAL_DIGESTS))) { rid = writeRevisitDigest(w, timestamp, HTTP_RESPONSE_MIMETYPE, baseid, curi, headers); } else if (curi.getFetchStatus() == HttpStatus.SC_NOT_MODIFIED && ((Boolean) getUncheckedAttribute(curi, ATTR_WRITE_REVISIT_FOR_NOT_MODIFIED))) { rid = writeRevisitNotModified(w, timestamp, baseid, curi, headers); } else { if (curi.isTruncatedFetch()) { String value = curi.isTimeTruncatedFetch() ? NAMED_FIELD_TRUNCATED_VALUE_TIME : curi.isLengthTruncatedFetch() ? NAMED_FIELD_TRUNCATED_VALUE_LENGTH : curi.isHeaderTruncatedFetch() ? NAMED_FIELD_TRUNCATED_VALUE_HEAD : // TODO: Add this to spec. TRUNCATED_VALUE_UNSPECIFIED; headers.addLabelValue(HEADER_KEY_TRUNCATED, value); } rid = writeResponse(w, timestamp, HTTP_RESPONSE_MIMETYPE, baseid, curi, headers); } headers = new ANVLRecord(1); headers.addLabelValue(HEADER_KEY_CONCURRENT_TO, '<' + rid.toString() + '>'); if (((Boolean) getUncheckedAttribute(curi, ATTR_WRITE_REQUESTS))) { writeRequest(w, timestamp, HTTP_REQUEST_MIMETYPE, baseid, curi, headers); } if (((Boolean) getUncheckedAttribute(curi, ATTR_WRITE_METADATA))) { writeMetadata(w, timestamp, baseid, curi, headers); } } else if (lowerCaseScheme.equals("dns")) { ANVLRecord headers = null; String ip = curi.getString(A_DNS_SERVER_IP_LABEL); if (ip != null && ip.length() > 0) { headers = new ANVLRecord(1); headers.addLabelValue(HEADER_KEY_IP, ip); } writeResponse(w, timestamp, curi.getContentType(), baseid, curi, headers); } else { logger.warning("No handler for scheme " + lowerCaseScheme); } } catch (IOException e) { // Invalidate this file (It gets a '.invalid' suffix). getPool().invalidateFile(writer); // Set the writer to null otherwise the pool accounting // of how many active writers gets skewed if we subsequently // do a returnWriter call on this object in the finally block. writer = null; throw e; } finally { if (writer != null) { setTotalBytesWritten(getTotalBytesWritten() + (writer.getPosition() - position)); getPool().returnFile(writer); } } checkBytesWritten(); } protected URI writeRequest(final WARCWriter w, final String timestamp, final String mimetype, final URI baseid, final CrawlURI curi, final ANVLRecord namedFields) throws IOException { final URI uid = qualifyRecordID(baseid, TYPE, REQUEST); ReplayInputStream ris = curi.getHttpRecorder().getRecordedOutput().getReplayInputStream(); try { w.writeRequestRecord(curi.toString(), timestamp, mimetype, uid, namedFields, ris, curi.getHttpRecorder().getRecordedOutput().getSize()); } finally { if (ris != null) { ris.close(); } } return uid; } protected URI writeResponse(final WARCWriter w, final String timestamp, final String mimetype, final URI baseid, final CrawlURI curi, final ANVLRecord namedFields) throws IOException { ReplayInputStream ris = curi.getHttpRecorder().getRecordedInput().getReplayInputStream(); try { w.writeResponseRecord(curi.toString(), timestamp, mimetype, baseid, namedFields, ris, curi.getHttpRecorder().getRecordedInput().getSize()); } finally { if (ris != null) { ris.close(); } } return baseid; } protected URI writeResource(final WARCWriter w, final String timestamp, final String mimetype, final URI baseid, final CrawlURI curi, final ANVLRecord namedFields) throws IOException { ReplayInputStream ris = curi.getHttpRecorder().getRecordedInput().getReplayInputStream(); try { w.writeResourceRecord(curi.toString(), timestamp, mimetype, baseid, namedFields, ris, curi.getHttpRecorder().getRecordedInput().getSize()); } finally { if (ris != null) { ris.close(); } } return baseid; } protected URI writeRevisitDigest(final WARCWriter w, final String timestamp, final String mimetype, final URI baseid, final CrawlURI curi, final ANVLRecord namedFields) throws IOException { long revisedLength = curi.getHttpRecorder().getRecordedInput().getContentBegin(); revisedLength = revisedLength > 0 ? revisedLength : curi.getHttpRecorder().getRecordedInput().getSize(); namedFields.addLabelValue(HEADER_KEY_PROFILE, PROFILE_REVISIT_IDENTICAL_DIGEST); namedFields.addLabelValue(HEADER_KEY_TRUNCATED, NAMED_FIELD_TRUNCATED_VALUE_LENGTH); ReplayInputStream ris = curi.getHttpRecorder().getRecordedInput().getReplayInputStream(); try { w.writeRevisitRecord(curi.toString(), timestamp, mimetype, baseid, namedFields, ris, revisedLength); } finally { if (ris != null) { ris.close(); } } curi.addAnnotation("warcRevisit:digest"); return baseid; } protected URI writeRevisitNotModified(final WARCWriter w, final String timestamp, final URI baseid, final CrawlURI curi, final ANVLRecord namedFields) throws IOException { namedFields.addLabelValue(HEADER_KEY_PROFILE, PROFILE_REVISIT_NOT_MODIFIED); // save just enough context to understand basis of not-modified if (curi.containsKey(A_HTTP_TRANSACTION)) { HttpMethodBase method = (HttpMethodBase) curi.getObject(A_HTTP_TRANSACTION); saveHeader(A_ETAG_HEADER, method, namedFields, HEADER_KEY_ETAG); saveHeader(A_LAST_MODIFIED_HEADER, method, namedFields, HEADER_KEY_LAST_MODIFIED); } // truncate to zero-length (all necessary info is above) namedFields.addLabelValue(HEADER_KEY_TRUNCATED, NAMED_FIELD_TRUNCATED_VALUE_LENGTH); ReplayInputStream ris = curi.getHttpRecorder().getRecordedInput().getReplayInputStream(); try { w.writeRevisitRecord(curi.toString(), timestamp, null, baseid, namedFields, ris, 0); } finally { if (ris != null) { ris.close(); } } curi.addAnnotation("warcRevisit:notModified"); return baseid; } /** * Save a header from the given HTTP operation into the * provider headers under a new name * * @param origName header name to get if present * @param method http operation containing headers */ protected void saveHeader(String origName, HttpMethodBase method, ANVLRecord headers, String newName) { Header header = method.getResponseHeader(origName); if (header != null) { headers.addLabelValue(newName, header.getValue()); } } protected URI writeMetadata(final WARCWriter w, final String timestamp, final URI baseid, final CrawlURI curi, final ANVLRecord namedFields) throws IOException { final URI uid = qualifyRecordID(baseid, TYPE, METADATA); // Get some metadata from the curi. // TODO: Get all curi metadata. // TODO: Use other than ANVL (or rename ANVL as NameValue or use // RFC822 (commons-httpclient?). ANVLRecord r = new ANVLRecord(); if (curi.isSeed()) { r.addLabel("seed"); } else { if (curi.forceFetch()) { r.addLabel("force-fetch"); } r.addLabelValue("via", curi.flattenVia()); r.addLabelValue("hopsFromSeed", curi.getPathFromSeed()); if (curi.containsKey(A_SOURCE_TAG)) { r.addLabelValue("sourceTag", curi.getString(A_SOURCE_TAG)); } } long duration = curi.getFetchDuration(); if (duration > -1) { r.addLabelValue("fetchTimeMs", Long.toString(duration)); } // Add outlinks though they are effectively useless without anchor text. Collection<Link> links = curi.getOutLinks(); if (links != null && links.size() > 0) { for (Link link : links) { r.addLabelValue("outlink", link.toString()); } } // TODO: Other curi fields to write to metadata. // // Credentials // // fetch-began-time: 1154569278774 // fetch-completed-time: 1154569281816 // // Annotations. byte[] b = r.getUTF8Bytes(); w.writeMetadataRecord(curi.toString(), timestamp, ANVLRecord.MIMETYPE, uid, namedFields, new ByteArrayInputStream(b), b.length); return uid; } protected URI getRecordID() throws IOException { URI result; try { result = GeneratorFactory.getFactory().getRecordID(); } catch (URISyntaxException e) { throw new IOException(e.toString()); } return result; } protected URI qualifyRecordID(final URI base, final String key, final String value) throws IOException { URI result; Map<String, String> qualifiers = new HashMap<String, String>(1); qualifiers.put(key, value); try { result = GeneratorFactory.getFactory().qualifyRecordID(base, qualifiers); } catch (URISyntaxException e) { throw new IOException(e.toString()); } return result; } @Override protected String getFirstrecordStylesheet() { return "/warcinfobody.xsl"; } /** * Return relevant values as header-like fields (here ANVLRecord, but * spec-defined "application/warc-fields" type when written). Field * names from from DCMI Terms and the WARC/0.17 specification. * * @see com.cyberway.issue.crawler.framework.WriterPoolProcessor#getFirstrecordBody(java.io.File) */ @Override protected String getFirstrecordBody(File orderFile) { ANVLRecord record = new ANVLRecord(7); record.addLabelValue("software", "Heritrix/" + Heritrix.getVersion() + " http://crawler.archive.org"); try { InetAddress host = InetAddress.getLocalHost(); record.addLabelValue("ip", host.getHostAddress()); record.addLabelValue("hostname", host.getHostName()); } catch (UnknownHostException e) { logger.log(Level.WARNING, "unable top obtain local crawl engine host", e); } record.addLabelValue("format", "WARC File Format 0.17"); record.addLabelValue("conformsTo", "http://crawler.archive.org/warc/0.17/WARC0.17ISO.doc"); // Get other values from order.xml try { Document doc = XmlUtils.getDocument(orderFile); addIfNotBlank(record, "operator", XmlUtils.xpathOrNull(doc, "//meta/operator")); addIfNotBlank(record, "publisher", XmlUtils.xpathOrNull(doc, "//meta/organization")); addIfNotBlank(record, "audience", XmlUtils.xpathOrNull(doc, "//meta/audience")); addIfNotBlank(record, "isPartOf", XmlUtils.xpathOrNull(doc, "//meta/name")); String rawDate = XmlUtils.xpathOrNull(doc, "//meta/date"); if (StringUtils.isNotBlank(rawDate)) { Date date; try { date = ArchiveUtils.parse14DigitDate(rawDate); addIfNotBlank(record, "created", ArchiveUtils.getLog14Date(date)); } catch (ParseException e) { logger.log(Level.WARNING, "obtaining warc created date", e); } } addIfNotBlank(record, "description", XmlUtils.xpathOrNull(doc, "//meta/description")); addIfNotBlank(record, "robots", XmlUtils.xpathOrNull(doc, "//newObject[@name='robots-honoring-policy']/string[@name='type']")); addIfNotBlank(record, "http-header-user-agent", XmlUtils.xpathOrNull(doc, "//map[@name='http-headers']/string[@name='user-agent']")); addIfNotBlank(record, "http-header-from", XmlUtils.xpathOrNull(doc, "//map[@name='http-headers']/string[@name='from']")); } catch (IOException e) { logger.log(Level.WARNING, "obtaining warcinfo", e); } // really ugly to return as string, when it may just be merged with // a couple other fields at write time, but changing would require // larger refactoring return record.toString(); } protected void addIfNotBlank(ANVLRecord record, String label, String value) { if (StringUtils.isNotBlank(value)) { record.addLabelValue(label, value); } } }