Java tutorial
/* * This file is part of the Heritrix web crawler (crawler.archive.org). * * Licensed to the Internet Archive (IA) by one or more individual * contributors. * * The IA licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.archive.io.arc; import java.io.ByteArrayOutputStream; import java.io.Closeable; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Logger; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.commons.cli.PosixParser; import org.archive.io.ArchiveReader; import org.archive.io.ArchiveRecord; import org.archive.io.ArchiveRecordHeader; import org.archive.io.RecoverableIOException; import org.archive.io.WriterPoolMember; import org.archive.util.ArchiveUtils; /** * Get an iterator on an ARC file or get a record by absolute position. * * ARC files are described here: * <a href="http://www.archive.org/web/researcher/ArcFileFormat.php">Arc * File Format</a>. * * <p>This class knows how to parse an ARC file. Pass it a file path * or an URL to an ARC. It can parse ARC Version 1 and 2. * * <p>Iterator returns <code>ARCRecord</code> * though {@link Iterator#next()} is returning * java.lang.Object. Cast the return. * * <p>Profiling java.io vs. memory-mapped ByteBufferInputStream shows the * latter slightly slower -- but not by much. TODO: Test more. Just * change {@link #getInputStream(File, long)}. * * @author stack * @version $Date$ $Revision$ */ public abstract class ARCReader extends ArchiveReader implements ARCConstants, Closeable { private final Logger logger = Logger.getLogger(ARCReader.class.getName()); /** * Set to true if we are aligned on first record of Archive file. * We used depend on offset. If offset was zero, then we were * aligned on first record. This is no longer necessarily the case when * Reader is created at an offset into an Archive file: The offset is zero * but its relative to where we started reading. */ private boolean alignedOnFirstRecord = true; private boolean parseHttpHeaders = true; protected ARCReader() { super(); } /** * Skip over any trailing new lines at end of the record so we're lined up * ready to read the next. * @param record * @throws IOException */ protected void gotoEOR(ArchiveRecord record) throws IOException { if (getIn().available() <= 0) { return; } // Remove any trailing LINE_SEPARATOR int c = -1; while (getIn().available() > 0) { if (getIn().markSupported()) { getIn().mark(1); } c = getIn().read(); if (c != -1) { if (c == LINE_SEPARATOR) { continue; } if (getIn().markSupported()) { // We've overread. We're probably in next record. There is // no way of telling for sure. It may be dross at end of // current record. Backup. getIn().reset(); break; } ArchiveRecordHeader h = (getCurrentRecord() != null) ? record.getHeader() : null; throw new IOException("Read " + (char) c + " when only " + LINE_SEPARATOR + " expected. " + getReaderIdentifier() + ((h != null) ? h.getHeaderFields().toString() : "")); } } } /** * Create new arc record. * * Encapsulate housekeeping that has to do w/ creating a new record. * * <p>Call this method at end of constructor to read in the * arcfile header. Will be problems reading subsequent arc records * if you don't since arcfile header has the list of metadata fields for * all records that follow. * * <p>When parsing through ARCs writing out CDX info, we spend about * 38% of CPU in here -- about 30% of which is in getTokenizedHeaderLine * -- of which 16% is reading. * * @param is InputStream to use. * @param offset Absolute offset into arc file. * @return An arc record. * @throws IOException */ protected ARCRecord createArchiveRecord(InputStream is, long offset) throws IOException { try { String version = super.getVersion(); ARCRecord record = new ARCRecord(is, getReaderIdentifier(), offset, isDigest(), isStrict(), isParseHttpHeaders(), isAlignedOnFirstRecord(), version); if (version != null && super.getVersion() == null) super.setVersion(version); currentRecord(record); } catch (IOException e) { if (e instanceof RecoverableIOException) { // Don't mess with RecoverableIOExceptions. Let them out. throw e; } IOException newE = new IOException(e.getMessage() + " (Offset " + offset + ")."); newE.setStackTrace(e.getStackTrace()); throw newE; } return (ARCRecord) getCurrentRecord(); } /** * Returns version of this ARC file. Usually read from first record of ARC. * If we're reading without having first read the first record -- e.g. * random access into middle of an ARC -- then version will not have been * set. For now, we return a default, version 1.1. Later, if more than * just one version of ARC, we could look at such as the meta line to see * what version of ARC this is. * @return Version of this ARC file. */ public String getVersion() { return (super.getVersion() == null) ? "1.1" : super.getVersion(); } protected boolean isAlignedOnFirstRecord() { return alignedOnFirstRecord; } protected void setAlignedOnFirstRecord(boolean alignedOnFirstRecord) { this.alignedOnFirstRecord = alignedOnFirstRecord; } /** * @return Returns the parseHttpHeaders. */ public boolean isParseHttpHeaders() { return this.parseHttpHeaders; } /** * @param parse The parseHttpHeaders to set. */ public void setParseHttpHeaders(boolean parse) { this.parseHttpHeaders = parse; } public String getFileExtension() { return ARC_FILE_EXTENSION; } public String getDotFileExtension() { return DOT_ARC_FILE_EXTENSION; } protected boolean output(final String format) throws IOException, java.text.ParseException { boolean result = super.output(format); if (!result && (format.equals(NOHEAD) || format.equals(HEADER))) { throw new IOException(format + " format only supported for single Records"); } return result; } public boolean outputRecord(final String format) throws IOException { boolean result = super.outputRecord(format); if (result) { return result; } if (format.equals(NOHEAD)) { // No point digesting if dumping content. setDigest(false); ARCRecord r = (ARCRecord) get(); r.skipHttpHeader(); r.dump(); result = true; } else if (format.equals(HEADER)) { // No point digesting if dumping content. setDigest(false); ARCRecord r = (ARCRecord) get(); r.dumpHttpHeader(); result = true; } return result; } public void dump(final boolean compress) throws IOException, java.text.ParseException { // No point digesting if we're doing a dump. setDigest(false); boolean firstRecord = true; ARCWriter writer = null; for (Iterator<ArchiveRecord> ii = iterator(); ii.hasNext();) { ARCRecord r = (ARCRecord) ii.next(); // We're to dump the arc on stdout. // Get the first record's data if any. ARCRecordMetaData meta = r.getMetaData(); if (firstRecord) { firstRecord = false; // Get an ARCWriter. ByteArrayOutputStream baos = new ByteArrayOutputStream(r.available()); // This is slow but done only once at top of ARC. while (r.available() > 0) { baos.write(r.read()); } List<String> listOfMetadata = new ArrayList<String>(); listOfMetadata.add(baos.toString(WriterPoolMember.UTF8)); // Assume getArc returns full path to file. ARCWriter // or new File will complain if it is otherwise. List<File> outDirs = new ArrayList<File>(); WriterPoolSettingsData settings = new WriterPoolSettingsData("", "", -1L, compress, outDirs, listOfMetadata); writer = new ARCWriter(new AtomicInteger(), System.out, new File(meta.getArc()), settings); continue; } writer.write(meta.getUrl(), meta.getMimetype(), meta.getIp(), ArchiveUtils.parse14DigitDate(meta.getDate()).getTime(), (int) meta.getLength(), r); } // System.out.println(System.currentTimeMillis() - start); } /** * @return an ArchiveReader that will delete a local file on close. Used * when we bring Archive files local and need to clean up afterward. */ public ARCReader getDeleteFileOnCloseReader(final File f) { final ARCReader d = this; return new ARCReader() { private final ARCReader delegate = d; private File archiveFile = f; public void close() throws IOException { this.delegate.close(); if (this.archiveFile != null) { if (archiveFile.exists()) { archiveFile.delete(); } this.archiveFile = null; } } public ArchiveRecord get(long o) throws IOException { return this.delegate.get(o); } public boolean isDigest() { return this.delegate.isDigest(); } public boolean isStrict() { return this.delegate.isStrict(); } public Iterator<ArchiveRecord> iterator() { return this.delegate.iterator(); } public void setDigest(boolean d) { this.delegate.setDigest(d); } public void setStrict(boolean s) { this.delegate.setStrict(s); } public List<ArchiveRecordHeader> validate() throws IOException { return this.delegate.validate(); } @Override public ArchiveRecord get() throws IOException { return this.delegate.get(); } @Override public String getVersion() { return this.delegate.getVersion(); } @Override public List<ArchiveRecordHeader> validate(int noRecords) throws IOException { return this.delegate.validate(noRecords); } @Override protected ARCRecord createArchiveRecord(InputStream is, long offset) throws IOException { return this.delegate.createArchiveRecord(is, offset); } @Override protected void gotoEOR(ArchiveRecord record) throws IOException { this.delegate.gotoEOR(record); } @Override public void dump(boolean compress) throws IOException, java.text.ParseException { this.delegate.dump(compress); } @Override public String getDotFileExtension() { return this.delegate.getDotFileExtension(); } @Override public String getFileExtension() { return this.delegate.getFileExtension(); } }; } // Static methods follow. /** * * @param formatter Help formatter instance. * @param options Usage options. * @param exitCode Exit code. */ private static void usage(HelpFormatter formatter, Options options, int exitCode) { formatter.printHelp("java org.archive.io.arc.ARCReader" + " [--digest=true|false] \\\n" + " [--format=cdx|cdxfile|dump|gzipdump|header|nohead]" + " [--offset=#] \\\n[--strict] [--parse] ARC_FILE|ARC_URL", options); System.exit(exitCode); } /** * Write out the arcfile. * * @param reader * @param format Format to use outputting. * @throws IOException * @throws java.text.ParseException */ protected static void output(ARCReader reader, String format) throws IOException, java.text.ParseException { if (!reader.output(format)) { throw new IOException("Unsupported format: " + format); } } /** * Generate a CDX index file for an ARC file. * * @param urlOrPath The ARC file to generate a CDX index for * @throws IOException * @throws java.text.ParseException */ public static void createCDXIndexFile(String urlOrPath) throws IOException, java.text.ParseException { ARCReader r = ARCReaderFactory.get(urlOrPath); r.setStrict(false); r.setParseHttpHeaders(true); r.setDigest(true); output(r, CDX_FILE); } /** * Command-line interface to ARCReader. * * Here is the command-line interface: * <pre> * usage: java org.archive.io.arc.ARCReader [--offset=#] ARCFILE * -h,--help Prints this message and exits. * -o,--offset Outputs record at this offset into arc file.</pre> * * <p>See in <code>$HERITRIX_HOME/bin/arcreader</code> for a script that'll * take care of classpaths and the calling of ARCReader. * * <p>Outputs using a pseudo-CDX format as described here: * <a href="http://www.archive.org/web/researcher/cdx_legend.php">CDX * Legent</a> and here * <a href="http://www.archive.org/web/researcher/example_cdx.php">Example</a>. * Legend used in below is: 'CDX b e a m s c V (or v if uncompressed) n g'. * Hash is hard-coded straight SHA-1 hash of content. * * @param args Command-line arguments. * @throws ParseException Failed parse of the command line. * @throws IOException * @throws java.text.ParseException */ @SuppressWarnings("unchecked") public static void main(String[] args) throws ParseException, IOException, java.text.ParseException { Options options = getOptions(); options.addOption(new Option("p", "parse", false, "Parse headers.")); PosixParser parser = new PosixParser(); CommandLine cmdline = parser.parse(options, args, false); List<String> cmdlineArgs = cmdline.getArgList(); Option[] cmdlineOptions = cmdline.getOptions(); HelpFormatter formatter = new HelpFormatter(); // If no args, print help. if (cmdlineArgs.size() <= 0) { usage(formatter, options, 0); } // Now look at options passed. long offset = -1; boolean digest = false; boolean strict = false; boolean parse = false; String format = CDX; for (int i = 0; i < cmdlineOptions.length; i++) { switch (cmdlineOptions[i].getId()) { case 'h': usage(formatter, options, 0); break; case 'o': offset = Long.parseLong(cmdlineOptions[i].getValue()); break; case 's': strict = true; break; case 'p': parse = true; break; case 'd': digest = getTrueOrFalse(cmdlineOptions[i].getValue()); break; case 'f': format = cmdlineOptions[i].getValue().toLowerCase(); boolean match = false; // List of supported formats. final String[] supportedFormats = { CDX, DUMP, GZIP_DUMP, HEADER, NOHEAD, CDX_FILE }; for (int ii = 0; ii < supportedFormats.length; ii++) { if (supportedFormats[ii].equals(format)) { match = true; break; } } if (!match) { usage(formatter, options, 1); } break; default: throw new RuntimeException("Unexpected option: " + +cmdlineOptions[i].getId()); } } if (offset >= 0) { if (cmdlineArgs.size() != 1) { System.out.println("Error: Pass one arcfile only."); usage(formatter, options, 1); } ARCReader arc = ARCReaderFactory.get((String) cmdlineArgs.get(0), offset); arc.setStrict(strict); // We must parse headers if we need to skip them. if (format.equals(NOHEAD) || format.equals(HEADER)) { parse = true; } arc.setParseHttpHeaders(parse); outputRecord(arc, format); } else { for (String urlOrPath : cmdlineArgs) { try { ARCReader r = ARCReaderFactory.get(urlOrPath); r.setStrict(strict); r.setParseHttpHeaders(parse); r.setDigest(digest); output(r, format); } catch (RuntimeException e) { // Write out name of file we failed on to help with // debugging. Then print stack trace and try to keep // going. We do this for case where we're being fed // a bunch of ARCs; just note the bad one and move // on to the next. System.err.println("Exception processing " + urlOrPath + ": " + e.getMessage()); e.printStackTrace(System.err); System.exit(1); } } } } }