Java tutorial
/* * This file is part of the Heritrix web crawler (crawler.archive.org). * * Licensed to the Internet Archive (IA) by one or more individual * contributors. * * The IA licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.archive.util; import java.io.BufferedInputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.OutputStream; import java.nio.charset.Charset; import java.util.HashSet; import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; import java.util.zip.DeflaterInputStream; import java.util.zip.GZIPInputStream; import org.apache.commons.httpclient.ChunkedInputStream; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.archive.io.GenericReplayCharSequence; import org.archive.io.RecordingInputStream; import org.archive.io.RecordingOutputStream; import org.archive.io.ReplayCharSequence; import org.archive.io.ReplayInputStream; import com.google.common.base.Charsets; /** * Pairs together a RecordingInputStream and RecordingOutputStream * to capture exactly a single HTTP transaction. * * Initially only supports HTTP/1.0 (one request, one response per stream) * * Call {@link #markContentBegin()} to demarc the transition between HTTP * header and body. * * @author gojomo */ public class Recorder { protected static Logger logger = Logger.getLogger("org.archive.util.HttpRecorder"); private static final int DEFAULT_OUTPUT_BUFFER_SIZE = 16384; private static final int DEFAULT_INPUT_BUFFER_SIZE = 524288; private RecordingInputStream ris = null; private RecordingOutputStream ros = null; /** * Backing file basename. * * Keep it around so can clean up backing files left on disk. */ private String backingFileBasename = null; /** * Backing file output stream suffix. */ private static final String RECORDING_OUTPUT_STREAM_SUFFIX = ".ros"; /** * Backing file input stream suffix. */ private static final String RECORDING_INPUT_STREAM_SUFFIX = ".ris"; /** * recording-input (ris) content character encoding. */ protected String characterEncoding = null; /** * Charset to use for CharSequence provision. Will be UTF-8 if no * encoding ever requested; a Charset matching above characterEncoding * if possible; ISO_8859 if above characterEncoding is unsatisfiable. * TODO: unify to UTF-8 for unspecified and bad-specified cases? * (current behavior is for consistency with our prior but perhaps not * optimal behavior) */ protected Charset charset = Charsets.UTF_8; /** whether recording-input (ris) message-body is chunked */ protected boolean inputIsChunked = false; /** recording-input (ris) entity content-encoding (eg gzip, deflate), if any */ protected String contentEncoding = null; private ReplayCharSequence replayCharSequence; /** * Create an HttpRecorder. * * @param tempDir Directory into which we drop backing files for * recorded input and output. * @param backingFilenameBase Backing filename base to which we'll append * suffices <code>ris</code> for recorded input stream and * <code>ros</code> for recorded output stream. * @param outBufferSize Size of output buffer to use. * @param inBufferSize Size of input buffer to use. */ public Recorder(File tempDir, String backingFilenameBase, int outBufferSize, int inBufferSize) { this(new File(ensure(tempDir), backingFilenameBase), outBufferSize, inBufferSize); } private static File ensure(File tempDir) { try { org.archive.util.FileUtils.ensureWriteableDirectory(tempDir); } catch (IOException e) { throw new IllegalStateException(e); } return tempDir; } public Recorder(File file, int outBufferSize, int inBufferSize) { super(); this.backingFileBasename = file.getAbsolutePath(); this.ris = new RecordingInputStream(inBufferSize, this.backingFileBasename + RECORDING_INPUT_STREAM_SUFFIX); this.ros = new RecordingOutputStream(outBufferSize, this.backingFileBasename + RECORDING_OUTPUT_STREAM_SUFFIX); } /** * Create an HttpRecorder. * * @param tempDir * Directory into which we drop backing files for recorded input * and output. * @param backingFilenameBase * Backing filename base to which we'll append suffices * <code>ris</code> for recorded input stream and * <code>ros</code> for recorded output stream. */ public Recorder(File tempDir, String backingFilenameBase) { this(tempDir, backingFilenameBase, DEFAULT_INPUT_BUFFER_SIZE, DEFAULT_OUTPUT_BUFFER_SIZE); } /** * Wrap the provided stream with the internal RecordingInputStream * * open() throws an exception if RecordingInputStream is already open. * * @param is InputStream to wrap. * * @return The input stream wrapper which itself is an input stream. * Pass this in place of the passed stream so input can be recorded. * * @throws IOException */ public InputStream inputWrap(InputStream is) throws IOException { logger.fine(Thread.currentThread().getName() + " wrapping input"); // discard any state from previously-recorded input this.characterEncoding = null; this.inputIsChunked = false; this.contentEncoding = null; this.ris.open(is); return this.ris; } /** * Wrap the provided stream with the internal RecordingOutputStream * * open() throws an exception if RecordingOutputStream is already open. * * @param os The output stream to wrap. * * @return The output stream wrapper which is itself an output stream. * Pass this in place of the passed stream so output can be recorded. * * @throws IOException */ public OutputStream outputWrap(OutputStream os) throws IOException { this.ros.open(os); return this.ros; } /** * Close all streams. */ public void close() { logger.fine(Thread.currentThread().getName() + " closing"); try { this.ris.close(); } catch (IOException e) { // TODO: Can we not let the exception out of here and report it // higher up in the caller? DevUtils.logger.log(Level.SEVERE, "close() ris" + DevUtils.extraInfo(), e); } try { this.ros.close(); } catch (IOException e) { DevUtils.logger.log(Level.SEVERE, "close() ros" + DevUtils.extraInfo(), e); } } /** * Return the internal RecordingInputStream * * @return A RIS. */ public RecordingInputStream getRecordedInput() { return this.ris; } /** * @return The RecordingOutputStream. */ public RecordingOutputStream getRecordedOutput() { return this.ros; } /** * Mark current position as the point where the HTTP headers end. */ public void markContentBegin() { this.ris.markContentBegin(); } public long getResponseContentLength() { return this.ris.getResponseContentLength(); } /** * Close both input and output recorders. * * Recorders are the output streams to which we are recording. * {@link #close()} closes the stream that is being recorded and the * recorder. This method explicitly closes the recorder only. */ public void closeRecorders() { try { this.ris.closeRecorder(); this.ros.closeRecorder(); } catch (IOException e) { DevUtils.warnHandle(e, "Convert to runtime exception?"); } } /** * Cleanup backing files. * * Call when completely done w/ recorder. Removes any backing files that * may have been dropped. */ public void cleanup() { this.close(); this.delete(this.backingFileBasename + RECORDING_OUTPUT_STREAM_SUFFIX); this.delete(this.backingFileBasename + RECORDING_INPUT_STREAM_SUFFIX); } /** * Delete file if exists. * * @param name Filename to delete. */ private void delete(String name) { File f = new File(name); if (f.exists()) { f.delete(); } } protected static ThreadLocal<Recorder> currentRecorder = new ThreadLocal<Recorder>(); public static void setHttpRecorder(Recorder httpRecorder) { currentRecorder.set(httpRecorder); } /** * Get the current threads' HttpRecorder. * * @return This threads' HttpRecorder. Returns null if can't find a * HttpRecorder in current instance. */ public static Recorder getHttpRecorder() { return currentRecorder.get(); } /** * @param characterEncoding Character encoding of input recording. * @return actual charset in use after attempt to set */ public void setCharset(Charset cs) { this.charset = cs; } /** * @return effective Charset of input recording */ public Charset getCharset() { return this.charset; } /** * @param characterEncoding Character encoding of input recording. */ public void setInputIsChunked(boolean chunked) { this.inputIsChunked = chunked; } protected static Set<String> SUPPORTED_ENCODINGS = new HashSet<String>(); static { SUPPORTED_ENCODINGS.add("gzip"); SUPPORTED_ENCODINGS.add("x-gzip"); SUPPORTED_ENCODINGS.add("deflate"); SUPPORTED_ENCODINGS.add("identity"); SUPPORTED_ENCODINGS.add("none"); // unofficial but common } /** * @param contentEncoding declared content-encoding of input recording. */ public void setContentEncoding(String contentEncoding) { String lowerCoding = contentEncoding.toLowerCase(); if (!SUPPORTED_ENCODINGS.contains(contentEncoding.toLowerCase())) { throw new IllegalArgumentException("contentEncoding unsupported: " + contentEncoding); } this.contentEncoding = lowerCoding; } /** * @return Returns the characterEncoding. */ public String getContentEncoding() { return this.contentEncoding; } /** * @return * @throws IOException * @deprecated use getContentReplayCharSequence */ public ReplayCharSequence getReplayCharSequence() throws IOException { return getContentReplayCharSequence(); } /** * @return A ReplayCharSequence. Caller may call * {@link ReplayCharSequence#close()} when finished. However, in * heritrix, the ReplayCharSequence is closed automatically when url * processing has finished; in that context it's preferable not * to close, so that processors can reuse the same instance. * @throws IOException * @see {@link #endReplays()} */ public ReplayCharSequence getContentReplayCharSequence() throws IOException { if (replayCharSequence == null || !replayCharSequence.isOpen() || !replayCharSequence.getCharset().equals(charset)) { if (replayCharSequence != null && replayCharSequence.isOpen()) { // existing sequence must not have matched now-configured Charset; close replayCharSequence.close(); } replayCharSequence = getContentReplayCharSequence(this.charset); } return replayCharSequence; } /** * @param characterEncoding Encoding of recorded stream. * @return A ReplayCharSequence Will return null if an IOException. Call * close on returned RCS when done. * @throws IOException */ public ReplayCharSequence getContentReplayCharSequence(Charset requestedCharset) throws IOException { // raw data overflows to disk; use temp file InputStream ris = getContentReplayInputStream(); ReplayCharSequence rcs = new GenericReplayCharSequence(ris, calcRecommendedCharBufferSize(this.getRecordedInput()), this.backingFileBasename + RECORDING_OUTPUT_STREAM_SUFFIX, requestedCharset); ris.close(); return rcs; } /** * Calculate a recommended size for an in-memory decoded-character buffer * of this content. We seek a size that is itself no larger (in 2-byte chars) * than the memory already used by the RecordingInputStream's internal raw * byte buffer, and also no larger than likely necessary. So, we take the * minimum of the actual recorded byte size and the RecordingInputStream's * max buffer size. * * @param inStream * @return int length for in-memory decoded-character buffer */ static protected int calcRecommendedCharBufferSize(RecordingInputStream inStream) { return (int) Math.min(inStream.getRecordedBufferLength() / 2, inStream.getSize()); } /** * Get a raw replay of all recorded data (including, for example, HTTP * protocol headers) * * @return A replay input stream. * @throws IOException */ public ReplayInputStream getReplayInputStream() throws IOException { return getRecordedInput().getReplayInputStream(); } /** * Get a raw replay of the 'message-body'. For the common case of * HTTP, this is the raw, possibly chunked-transfer-encoded message * contents not including the leading headers. * * @return A replay input stream. * @throws IOException */ public ReplayInputStream getMessageBodyReplayInputStream() throws IOException { return getRecordedInput().getMessageBodyReplayInputStream(); } /** * Get a raw replay of the 'entity'. For the common case of * HTTP, this is the message-body after any (usually-unnecessary) * transfer-decoding but before any content-encoding (eg gzip) decoding * * @return A replay input stream. * @throws IOException */ public InputStream getEntityReplayInputStream() throws IOException { if (inputIsChunked) { return new ChunkedInputStream(getRecordedInput().getMessageBodyReplayInputStream()); } else { return getRecordedInput().getMessageBodyReplayInputStream(); } } /** * Get a replay cued up for the 'content' (after all leading headers) * * @return A replay input stream. * @throws IOException */ public InputStream getContentReplayInputStream() throws IOException { InputStream entityStream = getEntityReplayInputStream(); if (StringUtils.isEmpty(contentEncoding)) { return entityStream; } else if ("gzip".equalsIgnoreCase(contentEncoding) || "x-gzip".equalsIgnoreCase(contentEncoding)) { try { return new GZIPInputStream(entityStream); } catch (IOException ioe) { logger.log(Level.WARNING, "gzip problem; using raw entity instead", ioe); IOUtils.closeQuietly(entityStream); // close partially-read stream return getEntityReplayInputStream(); } } else if ("deflate".equalsIgnoreCase(contentEncoding)) { return new DeflaterInputStream(entityStream); } else if ("identity".equalsIgnoreCase(contentEncoding) || "none".equalsIgnoreCase(contentEncoding)) { return entityStream; } else { // shouldn't be reached given check on setContentEncoding logger.log(Level.INFO, "Unknown content-encoding '" + contentEncoding + "' declared; using raw entity instead"); return entityStream; } } /** * Return a short prefix of the presumed-textual content as a String. * * @param size max length of String to return * @return String prefix, or empty String (with logged exception) on any error */ public String getContentReplayPrefixString(int size) { return getContentReplayPrefixString(size, this.charset); } /** * Return a short prefix of the presumed-textual content as a String. * * @param size max length of String to return * @return String prefix, or empty String (with logged exception) on any error */ public String getContentReplayPrefixString(int size, Charset cs) { try { InputStreamReader isr = new InputStreamReader(getContentReplayInputStream(), cs); char[] chars = new char[size]; int count = isr.read(chars); isr.close(); if (count > 0) { return new String(chars, 0, count); } else { return ""; } } catch (IOException e) { logger.log(Level.SEVERE, "unable to get replay prefix string", e); return ""; } } /** * @param tempFile * @throws IOException */ public void copyContentBodyTo(File tempFile) throws IOException { InputStream inStream = null; OutputStream outStream = null; try { inStream = getContentReplayInputStream(); outStream = FileUtils.openOutputStream(tempFile); IOUtils.copy(inStream, outStream); } finally { IOUtils.closeQuietly(inStream); IOUtils.closeQuietly(outStream); } } /** * Record the input stream for later playback by an extractor, etc. * This is convenience method used to setup an artificial HttpRecorder * scenario used in unit tests, etc. * @param dir Directory to write backing file to. * @param basename of what we're recording. * @param in Stream to read. * @param encoding Stream encoding. * @throws IOException * @return An {@link org.archive.util.Recorder}. */ public static Recorder wrapInputStreamWithHttpRecord(File dir, String basename, InputStream in, String encoding) throws IOException { Recorder rec = new Recorder(dir, basename); if (encoding != null && encoding.length() > 0) { rec.setCharset(Charset.forName(encoding)); } // Do not use FastBufferedInputStream here. It does not // support mark. InputStream is = rec.inputWrap(new BufferedInputStream(in)); final int BUFFER_SIZE = 1024 * 4; byte[] buffer = new byte[BUFFER_SIZE]; while (true) { // Just read it all down. int x = is.read(buffer); if (x == -1) { break; } } is.close(); return rec; } public void endReplays() { ArchiveUtils.closeQuietly(replayCharSequence); replayCharSequence = null; // like closeQuietly try { ris.clearForReuse(); } catch (IOException ioe) { } // like closeQuietly try { ros.clearForReuse(); } catch (IOException e) { } } }