Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.tika.server.resource; import static java.nio.charset.StandardCharsets.UTF_8; import javax.mail.internet.ContentDisposition; import javax.mail.internet.ParseException; import javax.ws.rs.Consumes; import javax.ws.rs.GET; import javax.ws.rs.POST; import javax.ws.rs.PUT; import javax.ws.rs.Path; import javax.ws.rs.Produces; import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.Context; import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.MultivaluedMap; import javax.ws.rs.core.Response; import javax.ws.rs.core.StreamingOutput; import javax.ws.rs.core.UriInfo; import javax.xml.transform.OutputKeys; import javax.xml.transform.TransformerConfigurationException; import javax.xml.transform.sax.SAXTransformerFactory; import javax.xml.transform.sax.TransformerHandler; import javax.xml.transform.stream.StreamResult; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.Writer; import java.lang.reflect.Field; import java.lang.reflect.Method; import java.util.Locale; import java.util.Map; import java.util.Set; import org.apache.commons.lang.StringUtils; import org.apache.cxf.jaxrs.ext.multipart.Attachment; import org.apache.poi.extractor.ExtractorFactory; import org.apache.tika.Tika; import org.apache.tika.config.TikaConfig; import org.apache.tika.detect.Detector; import org.apache.tika.exception.EncryptedDocumentException; import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; import org.apache.tika.metadata.TikaMetadataKeys; import org.apache.tika.mime.MediaType; import org.apache.tika.parser.AutoDetectParser; import org.apache.tika.parser.DigestingParser; import org.apache.tika.parser.ParseContext; import org.apache.tika.parser.Parser; import org.apache.tika.parser.ParserDecorator; import org.apache.tika.parser.PasswordProvider; import org.apache.tika.parser.html.BoilerpipeContentHandler; import org.apache.tika.parser.html.HtmlParser; import org.apache.tika.parser.ocr.TesseractOCRConfig; import org.apache.tika.parser.pdf.PDFParserConfig; import org.apache.tika.sax.BodyContentHandler; import org.apache.tika.sax.ExpandedTitleContentHandler; import org.apache.tika.sax.RichTextContentHandler; import org.apache.tika.server.InputStreamFactory; import org.apache.tika.server.TikaServerParseException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.xml.sax.ContentHandler; import org.xml.sax.SAXException; @Path("/tika") public class TikaResource { public static final String GREETING = "This is Tika Server (" + new Tika().toString() + "). Please PUT\n"; public static final String X_TIKA_OCR_HEADER_PREFIX = "X-Tika-OCR"; public static final String X_TIKA_PDF_HEADER_PREFIX = "X-Tika-PDF"; private static final Logger LOG = LoggerFactory.getLogger(TikaResource.class); private static TikaConfig tikaConfig; private static DigestingParser.Digester digester = null; private static InputStreamFactory inputStreamFactory = null; public static void init(TikaConfig config, DigestingParser.Digester digestr, InputStreamFactory iSF) { tikaConfig = config; digester = digestr; inputStreamFactory = iSF; } static { ExtractorFactory.setAllThreadsPreferEventExtractors(true); } @SuppressWarnings("serial") public static Parser createParser() { final Parser parser = new AutoDetectParser(tikaConfig); Map<MediaType, Parser> parsers = ((AutoDetectParser) parser).getParsers(); parsers.put(MediaType.APPLICATION_XML, new HtmlParser()); ((AutoDetectParser) parser).setParsers(parsers); ((AutoDetectParser) parser).setFallback(new Parser() { public Set<MediaType> getSupportedTypes(ParseContext parseContext) { return parser.getSupportedTypes(parseContext); } public void parse(InputStream inputStream, ContentHandler contentHandler, Metadata metadata, ParseContext parseContext) { throw new WebApplicationException(Response.Status.UNSUPPORTED_MEDIA_TYPE); } }); if (digester != null) { return new DigestingParser(parser, digester); } return parser; } public static TikaConfig getConfig() { return tikaConfig; } public static String detectFilename(MultivaluedMap<String, String> httpHeaders) { String disposition = httpHeaders.getFirst("Content-Disposition"); if (disposition != null) { try { ContentDisposition c = new ContentDisposition(disposition); // only support "attachment" dispositions if ("attachment".equals(c.getDisposition())) { String fn = c.getParameter("filename"); if (fn != null) { return fn; } } } catch (ParseException e) { // not a valid content-disposition field LOG.warn("Parse exception {} determining content disposition", e.getMessage(), e); } } // this really should not be used, since it's not an official field return httpHeaders.getFirst("File-Name"); } public static void fillParseContext(ParseContext parseContext, MultivaluedMap<String, String> httpHeaders, Parser embeddedParser) { TesseractOCRConfig ocrConfig = new TesseractOCRConfig(); PDFParserConfig pdfParserConfig = new PDFParserConfig(); for (String key : httpHeaders.keySet()) { if (StringUtils.startsWith(key, X_TIKA_OCR_HEADER_PREFIX)) { processHeaderConfig(httpHeaders, ocrConfig, key, X_TIKA_OCR_HEADER_PREFIX); } else if (StringUtils.startsWith(key, X_TIKA_PDF_HEADER_PREFIX)) { processHeaderConfig(httpHeaders, pdfParserConfig, key, X_TIKA_PDF_HEADER_PREFIX); } } parseContext.set(TesseractOCRConfig.class, ocrConfig); parseContext.set(PDFParserConfig.class, pdfParserConfig); if (embeddedParser != null) { parseContext.set(Parser.class, embeddedParser); } } public static InputStream getInputStream(InputStream is, HttpHeaders headers) { try { return inputStreamFactory.getInputSteam(is, headers); } catch (IOException e) { throw new TikaServerParseException(e); } } /** * Utility method to set a property on a class via reflection. * * @param httpHeaders the HTTP headers set. * @param object the <code>Object</code> to set the property on. * @param key the key of the HTTP Header. * @param prefix the name of the HTTP Header prefix used to find property. * @throws WebApplicationException thrown when field cannot be found. */ private static void processHeaderConfig(MultivaluedMap<String, String> httpHeaders, Object object, String key, String prefix) { try { String property = StringUtils.removeStart(key, prefix); Field field = object.getClass().getDeclaredField(StringUtils.uncapitalize(property)); field.setAccessible(true); if (field.getType() == String.class) { field.set(object, httpHeaders.getFirst(key)); } else if (field.getType() == int.class) { field.setInt(object, Integer.parseInt(httpHeaders.getFirst(key))); } else if (field.getType() == double.class) { field.setDouble(object, Double.parseDouble(httpHeaders.getFirst(key))); } else if (field.getType() == boolean.class) { field.setBoolean(object, Boolean.parseBoolean(httpHeaders.getFirst(key))); } else { //couldn't find a directly accessible field //try for setX(String s) String setter = StringUtils.uncapitalize(property); setter = "set" + setter.substring(0, 1).toUpperCase(Locale.US) + setter.substring(1); Method m = null; try { m = object.getClass().getMethod(setter, String.class); } catch (NoSuchMethodException e) { //swallow } if (m != null) { m.invoke(object, httpHeaders.getFirst(key)); } } } catch (Throwable ex) { throw new WebApplicationException( String.format(Locale.ROOT, "%s is an invalid %s header", key, X_TIKA_OCR_HEADER_PREFIX)); } } @SuppressWarnings("serial") public static void fillMetadata(Parser parser, Metadata metadata, ParseContext context, MultivaluedMap<String, String> httpHeaders) { String fileName = detectFilename(httpHeaders); if (fileName != null) { metadata.set(TikaMetadataKeys.RESOURCE_NAME_KEY, fileName); } String contentTypeHeader = httpHeaders.getFirst(HttpHeaders.CONTENT_TYPE); javax.ws.rs.core.MediaType mediaType = contentTypeHeader == null ? null : javax.ws.rs.core.MediaType.valueOf(contentTypeHeader); if (mediaType != null && "xml".equals(mediaType.getSubtype())) { mediaType = null; } if (mediaType != null && mediaType.equals(javax.ws.rs.core.MediaType.APPLICATION_OCTET_STREAM_TYPE)) { mediaType = null; } if (mediaType != null) { metadata.add(org.apache.tika.metadata.HttpHeaders.CONTENT_TYPE, mediaType.toString()); final Detector detector = getDetector(parser); setDetector(parser, new Detector() { public MediaType detect(InputStream inputStream, Metadata metadata) throws IOException { String ct = metadata.get(org.apache.tika.metadata.HttpHeaders.CONTENT_TYPE); //make sure never to return null -- TIKA-1845 MediaType type = null; if (ct != null) { //this can return null if ct is not a valid mime type type = MediaType.parse(ct); } if (type != null) { return type; } else { return detector.detect(inputStream, metadata); } } }); } final String password = httpHeaders.getFirst("Password"); if (password != null) { context.set(PasswordProvider.class, new PasswordProvider() { @Override public String getPassword(Metadata metadata) { return password; } }); } } public static void setDetector(Parser p, Detector detector) { AutoDetectParser adp = getAutoDetectParser(p); adp.setDetector(detector); } public static Detector getDetector(Parser p) { AutoDetectParser adp = getAutoDetectParser(p); return adp.getDetector(); } private static AutoDetectParser getAutoDetectParser(Parser p) { //bit stinky if (p instanceof AutoDetectParser) { return (AutoDetectParser) p; } else if (p instanceof ParserDecorator) { Parser wrapped = ((ParserDecorator) p).getWrappedParser(); if (wrapped instanceof AutoDetectParser) { return (AutoDetectParser) wrapped; } throw new RuntimeException("Couldn't find AutoDetectParser within: " + wrapped.getClass()); } throw new RuntimeException("Couldn't find AutoDetectParser within: " + p.getClass()); } public static void parse(Parser parser, Logger logger, String path, InputStream inputStream, ContentHandler handler, Metadata metadata, ParseContext parseContext) throws IOException { try (TikaInputStream tikaInputStream = TikaInputStream.get(inputStream)) { parser.parse(tikaInputStream, handler, metadata, parseContext); } catch (SAXException e) { throw new TikaServerParseException(e); } catch (EncryptedDocumentException e) { logger.warn("{}: Encrypted document", path, e); throw new TikaServerParseException(e); } catch (Exception e) { logger.warn("{}: Text extraction failed", path, e); throw new TikaServerParseException(e); } } public static void logRequest(Logger logger, UriInfo info, Metadata metadata) { if (metadata.get(org.apache.tika.metadata.HttpHeaders.CONTENT_TYPE) == null) { logger.info("{} (autodetecting type)", info.getPath()); } else { logger.info("{} ({})", info.getPath(), metadata.get(org.apache.tika.metadata.HttpHeaders.CONTENT_TYPE)); } } @GET @Produces("text/plain") public String getMessage() { return GREETING; } @POST @Consumes("multipart/form-data") @Produces("text/plain") @Path("form") public StreamingOutput getTextFromMultipart(Attachment att, @Context final UriInfo info) { return produceText(att.getObject(InputStream.class), att.getHeaders(), info); } //this is equivalent to text-main in tika-app @PUT @Consumes("*/*") @Produces("text/plain") @Path("main") public StreamingOutput getTextMain(final InputStream is, @Context HttpHeaders httpHeaders, @Context final UriInfo info) { return produceTextMain(is, httpHeaders.getRequestHeaders(), info); } //this is equivalent to text-main (Boilerpipe handler) in tika-app @POST @Consumes("multipart/form-data") @Produces("text/plain") @Path("form/main") public StreamingOutput getTextMainFromMultipart(final Attachment att, @Context final UriInfo info) { return produceTextMain(att.getObject(InputStream.class), att.getHeaders(), info); } public StreamingOutput produceTextMain(final InputStream is, @Context MultivaluedMap<String, String> httpHeaders, @Context final UriInfo info) { final Parser parser = createParser(); final Metadata metadata = new Metadata(); final ParseContext context = new ParseContext(); fillMetadata(parser, metadata, context, httpHeaders); fillParseContext(context, httpHeaders, parser); logRequest(LOG, info, metadata); return new StreamingOutput() { public void write(OutputStream outputStream) throws IOException, WebApplicationException { Writer writer = new OutputStreamWriter(outputStream, UTF_8); ContentHandler handler = new BoilerpipeContentHandler(writer); try (InputStream inputStream = is) { parse(parser, LOG, info.getPath(), inputStream, handler, metadata, context); } } }; } @PUT @Consumes("*/*") @Produces("text/plain") public StreamingOutput getText(final InputStream is, @Context HttpHeaders httpHeaders, @Context final UriInfo info) { return produceText(getInputStream(is, httpHeaders), httpHeaders.getRequestHeaders(), info); } public StreamingOutput produceText(final InputStream is, MultivaluedMap<String, String> httpHeaders, final UriInfo info) { final Parser parser = createParser(); final Metadata metadata = new Metadata(); final ParseContext context = new ParseContext(); fillMetadata(parser, metadata, context, httpHeaders); fillParseContext(context, httpHeaders, parser); logRequest(LOG, info, metadata); return new StreamingOutput() { public void write(OutputStream outputStream) throws IOException, WebApplicationException { Writer writer = new OutputStreamWriter(outputStream, UTF_8); BodyContentHandler body = new BodyContentHandler(new RichTextContentHandler(writer)); try (InputStream inputStream = is) { parse(parser, LOG, info.getPath(), inputStream, body, metadata, context); } } }; } @POST @Consumes("multipart/form-data") @Produces("text/html") @Path("form") public StreamingOutput getHTMLFromMultipart(Attachment att, @Context final UriInfo info) { return produceOutput(att.getObject(InputStream.class), att.getHeaders(), info, "html"); } @PUT @Consumes("*/*") @Produces("text/html") public StreamingOutput getHTML(final InputStream is, @Context HttpHeaders httpHeaders, @Context final UriInfo info) { return produceOutput(getInputStream(is, httpHeaders), httpHeaders.getRequestHeaders(), info, "html"); } @POST @Consumes("multipart/form-data") @Produces("text/xml") @Path("form") public StreamingOutput getXMLFromMultipart(Attachment att, @Context final UriInfo info) { return produceOutput(att.getObject(InputStream.class), att.getHeaders(), info, "xml"); } @PUT @Consumes("*/*") @Produces("text/xml") public StreamingOutput getXML(final InputStream is, @Context HttpHeaders httpHeaders, @Context final UriInfo info) { return produceOutput(getInputStream(is, httpHeaders), httpHeaders.getRequestHeaders(), info, "xml"); } private StreamingOutput produceOutput(final InputStream is, final MultivaluedMap<String, String> httpHeaders, final UriInfo info, final String format) { final Parser parser = createParser(); final Metadata metadata = new Metadata(); final ParseContext context = new ParseContext(); fillMetadata(parser, metadata, context, httpHeaders); fillParseContext(context, httpHeaders, parser); logRequest(LOG, info, metadata); return new StreamingOutput() { public void write(OutputStream outputStream) throws IOException, WebApplicationException { Writer writer = new OutputStreamWriter(outputStream, UTF_8); ContentHandler content; try { SAXTransformerFactory factory = (SAXTransformerFactory) SAXTransformerFactory.newInstance(); TransformerHandler handler = factory.newTransformerHandler(); handler.getTransformer().setOutputProperty(OutputKeys.METHOD, format); handler.getTransformer().setOutputProperty(OutputKeys.INDENT, "yes"); handler.getTransformer().setOutputProperty(OutputKeys.ENCODING, UTF_8.name()); handler.setResult(new StreamResult(writer)); content = new ExpandedTitleContentHandler(handler); } catch (TransformerConfigurationException e) { throw new WebApplicationException(e); } parse(parser, LOG, info.getPath(), is, content, metadata, context); } }; } }