Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.tika.parser.rtf; import java.io.IOException; import java.io.InputStream; import java.io.PushbackInputStream; import java.nio.ByteBuffer; import java.nio.CharBuffer; import java.nio.charset.Charset; import java.nio.charset.CharsetDecoder; import java.nio.charset.CoderResult; import java.nio.charset.CodingErrorAction; import java.util.Calendar; import java.util.HashMap; import java.util.LinkedList; import java.util.Locale; import java.util.Map; import java.util.TimeZone; import org.apache.commons.io.IOUtils; import org.apache.tika.exception.TikaException; import org.apache.tika.extractor.EmbeddedDocumentUtil; import org.apache.tika.metadata.Metadata; import org.apache.tika.metadata.Office; import org.apache.tika.metadata.OfficeOpenXMLCore; import org.apache.tika.metadata.OfficeOpenXMLExtended; import org.apache.tika.metadata.Property; import org.apache.tika.metadata.TikaCoreProperties; import org.apache.tika.sax.XHTMLContentHandler; import org.apache.tika.utils.CharsetUtils; import org.xml.sax.SAXException; /* Tokenizes and performs a "shallow" parse of the RTF * document, just enough to properly decode the text. * * TODO: we should cutover to a "real" tokenizer (eg JFlex); * it should give better perf, by replacing the excessive * "else if" string compares with FSA traversal. */ final class TextExtractor { private static final Charset ASCII = Charset.forName("US-ASCII"); private static final Charset WINDOWS_1252 = getCharset("WINDOWS-1252"); private static final Charset MAC_ROMAN = getCharset("MacRoman"); private static final Charset SHIFT_JIS = getCharset("Shift_JIS"); private static final Charset WINDOWS_57011 = getCharset("windows-57011"); private static final Charset WINDOWS_57010 = getCharset("windows-57010"); private static final Charset WINDOWS_57009 = getCharset("windows-57009"); private static final Charset WINDOWS_57008 = getCharset("windows-57008"); private static final Charset WINDOWS_57007 = getCharset("windows-57007"); private static final Charset WINDOWS_57006 = getCharset("windows-57006"); private static final Charset WINDOWS_57005 = getCharset("windows-57005"); private static final Charset WINDOWS_57004 = getCharset("windows-57004"); private static final Charset WINDOWS_57003 = getCharset("windows-57003"); private static final Charset X_ISCII91 = getCharset("x-ISCII91"); private static final Charset X_MAC_CENTRAL_EUROPE = getCharset("x-MacCentralEurope"); private static final Charset MAC_CYRILLIC = getCharset("MacCyrillic"); private static final Charset X_JOHAB = getCharset("x-Johab"); private static final Charset CP12582 = getCharset("CP1258"); private static final Charset CP12572 = getCharset("CP1257"); private static final Charset CP12562 = getCharset("CP1256"); private static final Charset CP12552 = getCharset("CP1255"); private static final Charset CP12542 = getCharset("CP1254"); private static final Charset CP12532 = getCharset("CP1253"); private static final Charset CP1252 = getCharset("CP1252"); private static final Charset CP12512 = getCharset("CP1251"); private static final Charset CP12502 = getCharset("CP1250"); private static final Charset CP950 = getCharset("CP950"); private static final Charset CP949 = getCharset("CP949"); private static final Charset MS9362 = getCharset("MS936"); private static final Charset MS8742 = getCharset("MS874"); private static final Charset CP866 = getCharset("CP866"); private static final Charset CP865 = getCharset("CP865"); private static final Charset CP864 = getCharset("CP864"); private static final Charset CP863 = getCharset("CP863"); private static final Charset CP862 = getCharset("CP862"); private static final Charset CP860 = getCharset("CP860"); private static final Charset CP852 = getCharset("CP852"); private static final Charset CP8502 = getCharset("CP850"); private static final Charset CP819 = getCharset("CP819"); private static final Charset WINDOWS_720 = getCharset("windows-720"); private static final Charset WINDOWS_711 = getCharset("windows-711"); private static final Charset WINDOWS_710 = getCharset("windows-710"); private static final Charset WINDOWS_709 = getCharset("windows-709"); private static final Charset ISO_8859_6 = getCharset("ISO-8859-6"); private static final Charset CP4372 = getCharset("CP437"); private static final Charset CP850 = getCharset("cp850"); private static final Charset CP437 = getCharset("cp437"); private static final Charset MS874 = getCharset("ms874"); private static final Charset CP1257 = getCharset("cp1257"); private static final Charset CP1256 = getCharset("cp1256"); private static final Charset CP1255 = getCharset("cp1255"); private static final Charset CP1258 = getCharset("cp1258"); private static final Charset CP1254 = getCharset("cp1254"); private static final Charset CP1253 = getCharset("cp1253"); private static final Charset MS950 = getCharset("ms950"); private static final Charset MS936 = getCharset("ms936"); private static final Charset MS1361 = getCharset("ms1361"); private static final Charset MS932 = getCharset("MS932"); private static final Charset CP1251 = getCharset("cp1251"); private static final Charset CP1250 = getCharset("cp1250"); private static final Charset MAC_THAI = getCharset("MacThai"); private static final Charset MAC_TURKISH = getCharset("MacTurkish"); private static final Charset MAC_GREEK = getCharset("MacGreek"); private static final Charset MAC_ARABIC = getCharset("MacArabic"); private static final Charset MAC_HEBREW = getCharset("MacHebrew"); private static final Charset JOHAB = getCharset("johab"); private static final Charset BIG5 = getCharset("Big5"); private static final Charset GB2312 = getCharset("GB2312"); private static final Charset MS949 = getCharset("ms949"); // The RTF doc has a "font table" that assigns ords // (f0, f1, f2, etc.) to fonts and charsets, using the // \fcharsetN control word. This mapping maps from the // N to corresponding Java charset: private static final Map<Integer, Charset> FCHARSET_MAP = new HashMap<Integer, Charset>(); // The RTF may specify the \ansicpgN charset in the // header; this maps the N to the corresponding Java // character set: private static final Map<Integer, Charset> ANSICPG_MAP = new HashMap<Integer, Charset>(); static { FCHARSET_MAP.put(0, WINDOWS_1252); // ANSI // charset 1 is Default // charset 2 is Symbol FCHARSET_MAP.put(77, MAC_ROMAN); // Mac Roman FCHARSET_MAP.put(78, SHIFT_JIS); // Mac Shift Jis FCHARSET_MAP.put(79, MS949); // Mac Hangul FCHARSET_MAP.put(80, GB2312); // Mac GB2312 FCHARSET_MAP.put(81, BIG5); // Mac Big5 FCHARSET_MAP.put(82, JOHAB); // Mac Johab (old) FCHARSET_MAP.put(83, MAC_HEBREW); // Mac Hebrew FCHARSET_MAP.put(84, MAC_ARABIC); // Mac Arabic FCHARSET_MAP.put(85, MAC_GREEK); // Mac Greek FCHARSET_MAP.put(86, MAC_TURKISH); // Mac Turkish FCHARSET_MAP.put(87, MAC_THAI); // Mac Thai FCHARSET_MAP.put(88, CP1250); // Mac East Europe FCHARSET_MAP.put(89, CP1251); // Mac Russian FCHARSET_MAP.put(128, MS932); // Shift JIS FCHARSET_MAP.put(129, MS949); // Hangul FCHARSET_MAP.put(130, MS1361); // Johab FCHARSET_MAP.put(134, MS936); // GB2312 FCHARSET_MAP.put(136, MS950); // Big5 FCHARSET_MAP.put(161, CP1253); // Greek FCHARSET_MAP.put(162, CP1254); // Turkish FCHARSET_MAP.put(163, CP1258); // Vietnamese FCHARSET_MAP.put(177, CP1255); // Hebrew FCHARSET_MAP.put(178, CP1256); // Arabic // FCHARSET_MAP.put( 179, "" ); // Arabic Traditional // FCHARSET_MAP.put( 180, "" ); // Arabic user // FCHARSET_MAP.put( 181, "" ); // Hebrew user FCHARSET_MAP.put(186, CP1257); // Baltic FCHARSET_MAP.put(204, CP1251); // Russian FCHARSET_MAP.put(222, MS874); // Thai FCHARSET_MAP.put(238, CP1250); // Eastern European FCHARSET_MAP.put(254, CP437); // PC 437 FCHARSET_MAP.put(255, CP850); // OEM } static { ANSICPG_MAP.put(437, CP4372); // US IBM ANSICPG_MAP.put(708, ISO_8859_6); // Arabic (ASMO 708) ANSICPG_MAP.put(709, WINDOWS_709); // Arabic (ASMO 449+, BCON V4) ANSICPG_MAP.put(710, WINDOWS_710); // Arabic (transparent Arabic) ANSICPG_MAP.put(710, WINDOWS_711); // Arabic (Nafitha Enhanced) ANSICPG_MAP.put(710, WINDOWS_720); // Arabic (transparent ASMO) ANSICPG_MAP.put(819, CP819); // Windows 3.1 (US & Western Europe) ANSICPG_MAP.put(819, CP819); // Windows 3.1 (US & Western Europe) ANSICPG_MAP.put(819, CP819); // Windows 3.1 (US & Western Europe) ANSICPG_MAP.put(850, CP8502); // IBM Multilingual ANSICPG_MAP.put(852, CP852); // Eastern European ANSICPG_MAP.put(860, CP860); // Portuguese ANSICPG_MAP.put(862, CP862); // Hebrew ANSICPG_MAP.put(863, CP863); // French Canadian ANSICPG_MAP.put(864, CP864); // Arabic ANSICPG_MAP.put(865, CP865); // Norwegian ANSICPG_MAP.put(866, CP866); // Soviet Union ANSICPG_MAP.put(874, MS8742); // Thai ANSICPG_MAP.put(932, MS932); // Japanese ANSICPG_MAP.put(936, MS9362); // Simplified Chinese ANSICPG_MAP.put(949, CP949); // Korean ANSICPG_MAP.put(950, CP950); // Traditional Chinese ANSICPG_MAP.put(1250, CP12502); // Eastern European ANSICPG_MAP.put(1251, CP12512); // Cyrillic ANSICPG_MAP.put(1252, CP1252); // Western European ANSICPG_MAP.put(1253, CP12532); // Greek ANSICPG_MAP.put(1254, CP12542); // Turkish ANSICPG_MAP.put(1255, CP12552); // Hebrew ANSICPG_MAP.put(1256, CP12562); // Arabic ANSICPG_MAP.put(1257, CP12572); // Baltic ANSICPG_MAP.put(1258, CP12582); // Vietnamese ANSICPG_MAP.put(1361, X_JOHAB); // Johab ANSICPG_MAP.put(10000, MAC_ROMAN); // Mac Roman ANSICPG_MAP.put(10001, SHIFT_JIS); // Mac Japan ANSICPG_MAP.put(10004, MAC_ARABIC); // Mac Arabic ANSICPG_MAP.put(10005, MAC_HEBREW); // Mac Hebrew ANSICPG_MAP.put(10006, MAC_GREEK); // Mac Hebrew ANSICPG_MAP.put(10007, MAC_CYRILLIC); // Mac Cyrillic ANSICPG_MAP.put(10029, X_MAC_CENTRAL_EUROPE); // MAC Latin2 ANSICPG_MAP.put(10081, MAC_TURKISH); // Mac Turkish ANSICPG_MAP.put(57002, X_ISCII91); // Devanagari // TODO: in theory these other charsets are simple // shifts off of Devanagari, so we could impl that // here: ANSICPG_MAP.put(57003, WINDOWS_57003); // Bengali ANSICPG_MAP.put(57004, WINDOWS_57004); // Tamil ANSICPG_MAP.put(57005, WINDOWS_57005); // Telugu ANSICPG_MAP.put(57006, WINDOWS_57006); // Assamese ANSICPG_MAP.put(57007, WINDOWS_57007); // Oriya ANSICPG_MAP.put(57008, WINDOWS_57008); // Kannada ANSICPG_MAP.put(57009, WINDOWS_57009); // Malayalam ANSICPG_MAP.put(57010, WINDOWS_57010); // Gujariti ANSICPG_MAP.put(57011, WINDOWS_57011); // Punjabi } // Used when we decode bytes -> chars using CharsetDecoder: private final char[] outputArray = new char[128]; private final CharBuffer outputBuffer = CharBuffer.wrap(outputArray); // Holds the font table from this RTF doc, mapping // the font number (from \fN control word) to the // corresponding charset: private final Map<Integer, Charset> fontToCharset = new HashMap<Integer, Charset>(); // Group stack: when we open a new group, we push // the previous group state onto the stack; when we // close the group, we restore it private final LinkedList<GroupState> groupStates = new LinkedList<GroupState>(); private final StringBuilder pendingBuffer = new StringBuilder(); private final XHTMLContentHandler out; private final Metadata metadata; private final RTFEmbObjHandler embObjHandler; // How many next ansi chars we should skip; this // is 0 except when we are still in the "ansi // shadow" after seeing a unicode escape, at which // point it's set to the last ucN skip we had seen: int ansiSkip = 0; private int written = 0; // Hold pending bytes (encoded in the current charset) // for text output: private byte[] pendingBytes = new byte[16]; private int pendingByteCount; private ByteBuffer pendingByteBuffer = ByteBuffer.wrap(pendingBytes); // Holds pending chars for text output private char[] pendingChars = new char[10]; private int pendingCharCount; // Holds chars for a still-being-tokenized control word private byte[] pendingControl = new byte[10]; private int pendingControlCount; // Reused when possible: private CharsetDecoder decoder; private Charset lastCharset; private Charset globalCharset = WINDOWS_1252; private int globalDefaultFont = -1; private int curFontID = -1; // Current group state; in theory this initial // GroupState is unused because the RTF doc should // immediately open the top group (start with {): private GroupState groupState = new GroupState(); private boolean inHeader = true; private int fontTableState; private int fontTableDepth; // Non null if we are processing metadata (title, // keywords, etc.) inside the info group: private Property nextMetaData; private boolean inParagraph; // Non-zero if we are processing inside a field destination: private int fieldState; // Non-zero list index private int pendingListEnd; private Map<Integer, ListDescriptor> listTable = new HashMap<Integer, ListDescriptor>(); private Map<Integer, ListDescriptor> listOverrideTable = new HashMap<Integer, ListDescriptor>(); private Map<Integer, ListDescriptor> currentListTable; private ListDescriptor currentList; private int listTableLevel = -1; private boolean ignoreLists; // Non-null if we've seen the url for a HYPERLINK but not yet // its text: private String pendingURL; // Used to process the sub-groups inside the upr // group: private int uprState = -1; // Used when extracting CREATION date: private int year, month, day, hour, minute; public TextExtractor(XHTMLContentHandler out, Metadata metadata, RTFEmbObjHandler embObjHandler) { this.metadata = metadata; this.out = out; this.embObjHandler = embObjHandler; } private static Charset getCharset(String name) { try { return CharsetUtils.forName(name); } catch (Exception e) { return ASCII; } } protected static boolean isHexChar(int ch) { return (ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F'); } private static boolean isAlpha(int ch) { return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z'); } private static boolean isDigit(int ch) { return ch >= '0' && ch <= '9'; } protected static int hexValue(int ch) { if (ch >= '0' && ch <= '9') { return ch - '0'; } else if (ch >= 'a' && ch <= 'z') { return 10 + (ch - 'a'); } else { assert ch >= 'A' && ch <= 'Z'; return 10 + (ch - 'A'); } } public boolean isIgnoringLists() { return ignoreLists; } public void setIgnoreLists(boolean ignore) { this.ignoreLists = ignore; } // Push pending bytes or pending chars: private void pushText() throws IOException, SAXException, TikaException { if (pendingByteCount != 0) { assert pendingCharCount == 0; pushBytes(); } else { pushChars(); } } // Buffers the byte (unit in the current charset) for // output: private void addOutputByte(int b) throws IOException, SAXException, TikaException { assert b >= 0 && b < 256 : "byte value out of range: " + b; if (pendingCharCount != 0) { pushChars(); } if (groupState.pictDepth > 0) { embObjHandler.writeMetadataChar((char) b); } else { // Save the byte in pending buffer: if (pendingByteCount == pendingBytes.length) { // Gradual but exponential growth: final byte[] newArray = new byte[(int) (pendingBytes.length * 1.25)]; System.arraycopy(pendingBytes, 0, newArray, 0, pendingBytes.length); pendingBytes = newArray; pendingByteBuffer = ByteBuffer.wrap(pendingBytes); } pendingBytes[pendingByteCount++] = (byte) b; } } // Buffers a byte as part of a control word: private void addControl(int b) { assert isAlpha(b); // Save the byte in pending buffer: if (pendingControlCount == pendingControl.length) { // Gradual but exponential growth: final byte[] newArray = new byte[(int) (pendingControl.length * 1.25)]; System.arraycopy(pendingControl, 0, newArray, 0, pendingControl.length); pendingControl = newArray; } pendingControl[pendingControlCount++] = (byte) b; } // Buffers a UTF16 code unit for output private void addOutputChar(char ch) throws IOException, SAXException, TikaException { if (pendingByteCount != 0) { pushBytes(); } if (inHeader || fieldState == 1) { pendingBuffer.append(ch); } else if (groupState.sn == true || groupState.sv == true) { embObjHandler.writeMetadataChar(ch); } else { if (pendingCharCount == pendingChars.length) { // Gradual but exponential growth: final char[] newArray = new char[(int) (pendingChars.length * 1.25)]; System.arraycopy(pendingChars, 0, newArray, 0, pendingChars.length); pendingChars = newArray; } pendingChars[pendingCharCount++] = ch; } } // Shallow parses the entire doc, writing output to // this.out and this.metadata public void extract(InputStream in) throws IOException, SAXException, TikaException { // in = new FilterInputStream(in) { // public int read() throws IOException { // int r = super.read(); // System.out.write(r); // System.out.flush(); // return r; // } // public int read(byte b[], int off, int len) throws IOException { // int r = super.read(b, off, len); // System.out.write(b, off, r); // System.out.flush(); // return r; // } // }; extract(new PushbackInputStream(in, 2)); } private void extract(PushbackInputStream in) throws IOException, SAXException, TikaException { out.startDocument(); while (true) { final int b = in.read(); if (b == -1) { break; } else if (b == '\\') { parseControlToken(in); } else if (b == '{') { pushText(); processGroupStart(in); } else if (b == '}') { pushText(); processGroupEnd(); if (groupStates.isEmpty()) { // parsed document closing brace break; } } else if (groupState.objdata == true || groupState.pictDepth == 1) { embObjHandler.writeHexChar(b); } else if (b != '\r' && b != '\n' && (!groupState.ignore || nextMetaData != null || groupState.sn == true || groupState.sv == true)) { // Linefeed and carriage return are not // significant if (ansiSkip != 0) { ansiSkip--; } else { addOutputByte(b); } } } endParagraph(false); out.endDocument(); } private void parseControlToken(PushbackInputStream in) throws IOException, SAXException, TikaException { int b = in.read(); if (b == '\'') { // escaped hex char parseHexChar(in); } else if (isAlpha(b)) { // control word parseControlWord((char) b, in); } else if (b == '{' || b == '}' || b == '\\' || b == '\r' || b == '\n') { // escaped char addOutputByte(b); } else if (b != -1) { // control symbol, eg \* or \~ processControlSymbol((char) b); } } private void parseHexChar(PushbackInputStream in) throws IOException, SAXException, TikaException { int hex1 = in.read(); if (!isHexChar(hex1)) { // DOC ERROR (malformed hex escape): ignore in.unread(hex1); return; } int hex2 = in.read(); if (!isHexChar(hex2)) { // TODO: log a warning here, somehow? // DOC ERROR (malformed hex escape): // ignore in.unread(hex2); return; } if (ansiSkip != 0) { // Skip this ansi char since we are // still in the shadow of a unicode // escape: ansiSkip--; } else { // Unescape: addOutputByte(16 * hexValue(hex1) + hexValue(hex2)); } } private void parseControlWord(int firstChar, PushbackInputStream in) throws IOException, SAXException, TikaException { addControl(firstChar); int b = in.read(); while (isAlpha(b)) { addControl(b); b = in.read(); } boolean hasParam = false; boolean negParam = false; if (b == '-') { negParam = true; hasParam = true; b = in.read(); } int param = 0; while (isDigit(b)) { param *= 10; param += (b - '0'); hasParam = true; b = in.read(); } // space is consumed as part of the // control word, but is not added to the // control word if (b != ' ') { in.unread(b); } if (hasParam) { if (negParam) { param = -param; } processControlWord(param, in); } else { processControlWord(); } pendingControlCount = 0; } private void lazyStartParagraph() throws IOException, SAXException, TikaException { if (!inParagraph) { // Ensure </i></b> order if (groupState.italic) { end("i"); } if (groupState.bold) { end("b"); } if (pendingListEnd != 0 && groupState.list != pendingListEnd) { endList(pendingListEnd); pendingListEnd = 0; } if (inList() && pendingListEnd != groupState.list) { startList(groupState.list); } if (inList()) { out.startElement("li"); } else { out.startElement("p"); } // Ensure <b><i> order if (groupState.bold) { start("b"); } if (groupState.italic) { start("i"); } inParagraph = true; } } private void endParagraph(boolean preserveStyles) throws IOException, SAXException, TikaException { pushText(); //maintain consecutive new lines if (!inParagraph) { lazyStartParagraph(); } if (inParagraph) { if (groupState.italic) { end("i"); groupState.italic = preserveStyles; } if (groupState.bold) { end("b"); groupState.bold = preserveStyles; } if (inList()) { out.endElement("li"); } else { out.endElement("p"); } if (preserveStyles && (groupState.bold || groupState.italic)) { start("p"); if (groupState.bold) { start("b"); } if (groupState.italic) { start("i"); } inParagraph = true; } else { inParagraph = false; } } // Ensure closing the list at document end if (!preserveStyles && pendingListEnd != 0) { endList(pendingListEnd); pendingListEnd = 0; } } // Push pending UTF16 units to out ContentHandler private void pushChars() throws IOException, SAXException, TikaException { if (pendingCharCount != 0) { lazyStartParagraph(); out.characters(pendingChars, 0, pendingCharCount); pendingCharCount = 0; } } // Decodes the buffered bytes in pendingBytes // into UTF16 code units, and sends the characters // to the out ContentHandler, if we are in the body, // else appends the characters to the pendingBuffer private void pushBytes() throws IOException, SAXException, TikaException { if (pendingByteCount > 0 && (!groupState.ignore || nextMetaData != null)) { final CharsetDecoder decoder = getDecoder(); pendingByteBuffer.limit(pendingByteCount); assert pendingByteBuffer.position() == 0; assert outputBuffer.position() == 0; while (true) { // We pass true for endOfInput because, when // we are called, we should have seen a // complete sequence of characters for this // charset: final CoderResult result = decoder.decode(pendingByteBuffer, outputBuffer, true); final int pos = outputBuffer.position(); if (pos > 0) { if (inHeader || fieldState == 1) { pendingBuffer.append(outputArray, 0, pos); } else { lazyStartParagraph(); out.characters(outputArray, 0, pos); } outputBuffer.position(0); } if (result == CoderResult.UNDERFLOW) { break; } } while (true) { final CoderResult result = decoder.flush(outputBuffer); final int pos = outputBuffer.position(); if (pos > 0) { if (inHeader || fieldState == 1) { pendingBuffer.append(outputArray, 0, pos); } else { lazyStartParagraph(); out.characters(outputArray, 0, pos); } outputBuffer.position(0); } if (result == CoderResult.UNDERFLOW) { break; } } // Reset for next decode decoder.reset(); pendingByteBuffer.position(0); } pendingByteCount = 0; } // NOTE: s must be ascii alpha only private boolean equals(String s) { if (pendingControlCount != s.length()) { return false; } for (int idx = 0; idx < pendingControlCount; idx++) { assert isAlpha(s.charAt(idx)); if (((byte) s.charAt(idx)) != pendingControl[idx]) { return false; } } return true; } private void processControlSymbol(char ch) throws IOException, SAXException, TikaException { switch (ch) { case '~': // Non-breaking space -> unicode NON-BREAKING SPACE addOutputChar('\u00a0'); break; case '*': // Ignorable destination (control words defined after // the 1987 RTF spec). These are already handled by // processGroupStart() break; case '-': // Optional hyphen -> unicode SOFT HYPHEN addOutputChar('\u00ad'); break; case '_': // Non-breaking hyphen -> unicode NON-BREAKING HYPHEN addOutputChar('\u2011'); break; default: break; } } private CharsetDecoder getDecoder() throws TikaException { Charset charset = getCharset(); // Common case: charset is same as last time, so // just reuse it: if (lastCharset == null || !charset.equals(lastCharset)) { decoder = charset.newDecoder(); decoder.onMalformedInput(CodingErrorAction.REPLACE); decoder.onUnmappableCharacter(CodingErrorAction.REPLACE); lastCharset = charset; } return decoder; } // Return current charset in-use private Charset getCharset() throws TikaException { // If a specific font (fN) was set, use its charset if (groupState.fontCharset != null) { return groupState.fontCharset; } // Else, if global default font (defN) was set, use that one if (globalDefaultFont != -1 && !inHeader) { Charset cs = fontToCharset.get(globalDefaultFont); if (cs != null) { return cs; } } // Else, use the global charset if (globalCharset == null) { throw new TikaException("unable to determine charset"); } return globalCharset; } // Handle control word that takes a parameter: private void processControlWord(int param, PushbackInputStream in) throws IOException, SAXException, TikaException { // TODO: afN? (associated font number) // TODO: do these alter text output...? /* } else if (equals("stshfdbch")) { // font to be used by default in // style sheet for East Asian chars // arg N is font table entry } else if (equals("stshfloch")) { // font to be used by default in // style sheet for ASCII chars // arg N is font table entry } else if (equals("stshfhich")) { // font to be used by default in // style sheet for High Ansi chars // arg N is font table entry } else if (equals("stshfbi")) { // style sheet for Complex Scripts (BIDI) chars // arg N is font table entry */ // TODO: inefficient that we check equals N times; // we'd get better perf w/ real lexer (eg // JFlex), which uses single-pass FSM to do cmp: if (inHeader) { if (equals("ansicpg")) { // ANSI codepage Charset cs = ANSICPG_MAP.get(param); if (cs != null) { globalCharset = cs; } } else if (equals("deff")) { // Default font globalDefaultFont = param; } else if (equals("nofpages")) { metadata.add(Office.PAGE_COUNT, Integer.toString(param)); } else if (equals("nofwords")) { metadata.add(Office.WORD_COUNT, Integer.toString(param)); } else if (equals("nofchars")) { metadata.add(Office.CHARACTER_COUNT, Integer.toString(param)); } else if (equals("yr")) { year = param; } else if (equals("mo")) { month = param; } else if (equals("dy")) { day = param; } else if (equals("hr")) { hour = param; } else if (equals("min")) { minute = param; } if (fontTableState == 1) { // Still inside font table -- record the // mappings of fN to the fcharset: if (groupState.depth < fontTableDepth) { fontTableState = 2; } else { if (equals("f")) { // Start new font definition curFontID = param; } else if (equals("fcharset")) { Charset cs = FCHARSET_MAP.get(param); if (cs != null) { fontToCharset.put(curFontID, cs); } } } } if (currentList != null) { if (equals("listid")) { currentList.id = param; currentListTable.put(currentList.id, currentList); } else if (equals("listtemplateid")) { currentList.templateID = param; } else if (equals("levelnfc") || equals("levelnfcn")) { //sanity check to make sure list information isn't corrupt if (listTableLevel > -1 && listTableLevel < currentList.numberType.length) { currentList.numberType[listTableLevel] = param; } } } } else { // In document if (equals("b")) { // b0 assert param == 0; if (groupState.bold) { pushText(); if (groupState.italic) { end("i"); } end("b"); if (groupState.italic) { start("i"); } groupState.bold = false; } } else if (equals("i")) { // i0 assert param == 0; if (groupState.italic) { pushText(); end("i"); groupState.italic = false; } } else if (equals("f")) { // Change current font Charset fontCharset = fontToCharset.get(param); // Push any buffered text before changing // font: pushText(); if (fontCharset != null) { groupState.fontCharset = fontCharset; } else { // DOC ERROR: font change referenced a // non-table'd font number // TODO: log a warning? Throw an exc? groupState.fontCharset = null; } } else if (equals("ls")) { groupState.list = param; } else if (equals("lslvl")) { groupState.listLevel = param; } } // Process unicode escape. This can appear in doc // or in header, since the metadata (info) fields // in the header can be unicode escaped as well: if (equals("u")) { // Unicode escape if (!groupState.ignore || groupState.sv || groupState.sn) { final char utf16CodeUnit = (char) (param & 0xffff); addOutputChar(utf16CodeUnit); } // After seeing a unicode escape we must // skip the next ucSkip ansi chars (the // "unicode shadow") ansiSkip = groupState.ucSkip; } else if (equals("uc")) { // Change unicode shadow length groupState.ucSkip = param; } else if (equals("bin")) { if (param >= 0) { if (groupState.pictDepth == 1) { try { embObjHandler.writeBytes(in, param); } catch (IOException | TikaException e) { EmbeddedDocumentUtil.recordEmbeddedStreamException(e, metadata); embObjHandler.reset(); } } else { IOUtils.skipFully(in, param); } } else { // log some warning? } } } private boolean inList() { return !ignoreLists && groupState.list != 0; } /** * Marks the current list as pending to end. This is done to be able to merge list items of * the same list within the same enclosing list tag (ie. either <code>"ul"</code>, or * <code>"ol"</code>). */ private void pendingListEnd() { pendingListEnd = groupState.list; groupState.list = 0; } /** * Emits the end tag of a list. Uses {@link #isUnorderedList(int)} to determine the list * type for the given <code>listID</code>. * * @param listID The ID of the list. * @throws IOException * @throws SAXException * @throws TikaException */ private void endList(int listID) throws IOException, SAXException, TikaException { if (!ignoreLists) { out.endElement(isUnorderedList(listID) ? "ul" : "ol"); } } /** * Emits the start tag of a list. Uses {@link #isUnorderedList(int)} to determine the list * type for the given <code>listID</code>. * * @param listID The ID of the list. * @throws IOException * @throws SAXException * @throws TikaException */ private void startList(int listID) throws IOException, SAXException, TikaException { if (!ignoreLists) { out.startElement(isUnorderedList(listID) ? "ul" : "ol"); } } private boolean isUnorderedList(int listID) { ListDescriptor list = listTable.get(listID); if (list != null) { return list.isUnordered(groupState.listLevel); } return true; } private void end(String tag) throws IOException, SAXException, TikaException { out.endElement(tag); } private void start(String tag) throws IOException, SAXException, TikaException { out.startElement(tag); } // Handle non-parameter control word: private void processControlWord() throws IOException, SAXException, TikaException { if (inHeader) { if (equals("ansi")) { globalCharset = WINDOWS_1252; } else if (equals("pca")) { globalCharset = CP850; } else if (equals("pc")) { globalCharset = CP437; } else if (equals("mac")) { globalCharset = MAC_ROMAN; } if (equals("colortbl") || equals("stylesheet") || equals("fonttbl")) { groupState.ignore = true; } else if (equals("listtable")) { currentListTable = listTable; } else if (equals("listoverridetable")) { currentListTable = listOverrideTable; } if (uprState == -1) { // TODO: we can also parse \creatim, \revtim, // \printim, \version, etc. if (equals("author")) { nextMetaData = TikaCoreProperties.CREATOR; } else if (equals("title")) { nextMetaData = TikaCoreProperties.TITLE; } else if (equals("subject")) { // TODO: Move to OO subject in Tika 2.0 nextMetaData = TikaCoreProperties.TRANSITION_SUBJECT_TO_OO_SUBJECT; } else if (equals("keywords")) { nextMetaData = TikaCoreProperties.TRANSITION_KEYWORDS_TO_DC_SUBJECT; } else if (equals("category")) { nextMetaData = OfficeOpenXMLCore.CATEGORY; } else if (equals("comment")) { nextMetaData = TikaCoreProperties.COMMENTS; } else if (equals("company")) { nextMetaData = OfficeOpenXMLExtended.COMPANY; } else if (equals("manager")) { nextMetaData = OfficeOpenXMLExtended.MANAGER; } else if (equals("template")) { nextMetaData = OfficeOpenXMLExtended.TEMPLATE; } else if (equals("creatim")) { nextMetaData = TikaCoreProperties.CREATED; } } if (fontTableState == 0) { // Didn't see font table yet if (equals("fonttbl")) { fontTableState = 1; fontTableDepth = groupState.depth; } } else if (fontTableState == 1) { // Inside font table if (groupState.depth < fontTableDepth) { fontTableState = 2; } } // List table handling if (currentListTable != null) { if (equals("list") || equals("listoverride")) { currentList = new ListDescriptor(); listTableLevel = -1; } else if (currentList != null) { if (equals("liststylename")) { currentList.isStyle = true; } else if (equals("listlevel")) { listTableLevel++; } } } if (!groupState.ignore && (equals("par") || equals("pard") || equals("sect") || equals("sectd") || equals("plain") || equals("ltrch") || equals("rtlch"))) { inHeader = false; } } else { if (equals("b")) { if (!groupState.bold) { pushText(); lazyStartParagraph(); if (groupState.italic) { // Make sure nesting is always <b><i> end("i"); } groupState.bold = true; start("b"); if (groupState.italic) { start("i"); } } } else if (equals("i")) { if (!groupState.italic) { pushText(); lazyStartParagraph(); groupState.italic = true; start("i"); } } } final boolean ignored = groupState.ignore; if (equals("pard")) { // Reset styles pushText(); if (groupState.italic) { end("i"); groupState.italic = false; } if (groupState.bold) { end("b"); groupState.bold = false; } if (inList()) { // && (groupStates.size() == 1 || groupStates.peekLast().list < 0)) pendingListEnd(); } } else if (equals("par")) { if (!ignored) { endParagraph(true); } } else if (equals("shptxt")) { pushText(); // Text inside a shape groupState.ignore = false; } else if (equals("atnid")) { pushText(); // Annotation ID groupState.ignore = false; } else if (equals("atnauthor")) { pushText(); // Annotation author groupState.ignore = false; } else if (equals("annotation")) { pushText(); // Annotation groupState.ignore = false; } else if (equals("listtext")) { groupState.ignore = true; } else if (equals("cell")) { // TODO: we should produce a table output here? //addOutputChar(' '); endParagraph(true); } else if (equals("sp")) { groupState.sp = true; } else if (equals("sn")) { embObjHandler.startSN(); groupState.sn = true; } else if (equals("sv")) { embObjHandler.startSV(); groupState.sv = true; } else if (equals("object")) { pushText(); embObjHandler.setInObject(true); groupState.object = true; } else if (equals("objdata")) { groupState.objdata = true; embObjHandler.startObjData(); } else if (equals("pict")) { pushText(); // TODO: create img tag? but can that support // embedded image data? groupState.pictDepth = 1; embObjHandler.startPict(); } else if (equals("line")) { if (!ignored) { addOutputChar('\n'); } } else if (equals("column")) { if (!ignored) { addOutputChar(' '); } } else if (equals("page")) { if (!ignored) { addOutputChar('\n'); } } else if (equals("softline")) { if (!ignored) { addOutputChar('\n'); } } else if (equals("softcolumn")) { if (!ignored) { addOutputChar(' '); } } else if (equals("softpage")) { if (!ignored) { addOutputChar('\n'); } } else if (equals("tab")) { if (!ignored) { addOutputChar('\t'); } } else if (equals("upr")) { uprState = 0; } else if (equals("ud") && uprState == 1) { uprState = -1; // 2nd group inside the upr destination, which // contains the unicode encoding of the text, so // we want to keep that: groupState.ignore = false; } else if (equals("bullet")) { if (!ignored) { // unicode BULLET addOutputChar('\u2022'); } } else if (equals("endash")) { if (!ignored) { // unicode EN DASH addOutputChar('\u2013'); } } else if (equals("emdash")) { if (!ignored) { // unicode EM DASH addOutputChar('\u2014'); } } else if (equals("enspace")) { if (!ignored) { // unicode EN SPACE addOutputChar('\u2002'); } } else if (equals("qmspace")) { if (!ignored) { // quarter em space -> unicode FOUR-PER-EM SPACE addOutputChar('\u2005'); } } else if (equals("emspace")) { if (!ignored) { // unicode EM SPACE addOutputChar('\u2003'); } } else if (equals("lquote")) { if (!ignored) { // unicode LEFT SINGLE QUOTATION MARK addOutputChar('\u2018'); } } else if (equals("rquote")) { if (!ignored) { // unicode RIGHT SINGLE QUOTATION MARK addOutputChar('\u2019'); } } else if (equals("ldblquote")) { if (!ignored) { // unicode LEFT DOUBLE QUOTATION MARK addOutputChar('\u201C'); } } else if (equals("rdblquote")) { if (!ignored) { // unicode RIGHT DOUBLE QUOTATION MARK addOutputChar('\u201D'); } } else if (equals("fldinst")) { fieldState = 1; groupState.ignore = false; } else if (equals("fldrslt") && fieldState == 2) { assert pendingURL != null; lazyStartParagraph(); out.startElement("a", "href", pendingURL); pendingURL = null; fieldState = 3; groupState.ignore = false; } } // Push new GroupState private void processGroupStart(PushbackInputStream in) throws IOException { ansiSkip = 0; // Push current groupState onto the stack groupStates.add(groupState); // Make new GroupState groupState = new GroupState(groupState); assert groupStates.size() == groupState.depth : "size=" + groupStates.size() + " depth=" + groupState.depth; if (uprState == 0) { uprState = 1; groupState.ignore = true; } // Check for ignorable groups. Note that // sometimes we un-ignore within this group, eg // when handling upr escape. int b2 = in.read(); if (b2 == '\\') { int b3 = in.read(); if (b3 == '*') { groupState.ignore = true; } in.unread(b3); } in.unread(b2); } // Pop current GroupState private void processGroupEnd() throws IOException, SAXException, TikaException { if (inHeader) { if (nextMetaData != null) { if (nextMetaData == TikaCoreProperties.CREATED) { Calendar cal = Calendar.getInstance(TimeZone.getDefault(), Locale.ROOT); cal.set(year, month - 1, day, hour, minute, 0); metadata.set(nextMetaData, cal.getTime()); } else if (nextMetaData.isMultiValuePermitted()) { metadata.add(nextMetaData, pendingBuffer.toString()); } else { metadata.set(nextMetaData, pendingBuffer.toString()); } nextMetaData = null; } pendingBuffer.setLength(0); } assert groupState.depth > 0; ansiSkip = 0; if (groupState.objdata == true) { try { embObjHandler.handleCompletedObject(); } catch (TikaException | IOException e) { EmbeddedDocumentUtil.recordException(e, metadata); } groupState.objdata = false; } else if (groupState.pictDepth > 0) { if (groupState.sn == true) { embObjHandler.endSN(); } else if (groupState.sv == true) { embObjHandler.endSV(); } else if (groupState.sp == true) { embObjHandler.endSP(); } else if (groupState.pictDepth == 1) { embObjHandler.handleCompletedObject(); } } if (groupState.object == true) { embObjHandler.setInObject(false); } // Be robust if RTF doc is corrupt (has too many // closing }s): // TODO: log a warning? if (groupStates.size() > 0) { // Restore group state: final GroupState outerGroupState = groupStates.removeLast(); // Close italic, if outer does not have italic or // bold changed: if (groupState.italic) { if (!outerGroupState.italic || groupState.bold != outerGroupState.bold) { end("i"); groupState.italic = false; } } // Close bold if (groupState.bold && !outerGroupState.bold) { end("b"); } // Open bold if (!groupState.bold && outerGroupState.bold) { start("b"); } // Open italic if (!groupState.italic && outerGroupState.italic) { start("i"); } groupState = outerGroupState; } assert groupStates.size() == groupState.depth; if (fieldState == 1) { String s = pendingBuffer.toString().trim(); pendingBuffer.setLength(0); if (s.startsWith("HYPERLINK")) { s = s.substring(9).trim(); // TODO: what other instructions can be in a // HYPERLINK destination? final boolean isLocalLink = s.contains("\\l "); int idx = s.indexOf('"'); if (idx != -1) { int idx2 = s.indexOf('"', 1 + idx); if (idx2 != -1) { s = s.substring(1 + idx, idx2); } } pendingURL = (isLocalLink ? "#" : "") + s; fieldState = 2; } else { fieldState = 0; } // TODO: we could process the other known field // types. Right now, we will extract their text // inlined, but fail to record them in metadata // as a field value. } else if (fieldState == 3) { out.endElement("a"); fieldState = 0; } } }