Java tutorial
/******************************************************************************* * Copyright 2013 * TU Darmstadt, FG Sprachtechnologie * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. ******************************************************************************/ package org.dkpro.bigdata.io.hadoop; import java.io.IOException; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.compress.CompressionCodec; import org.apache.hadoop.io.compress.CompressionCodecFactory; import org.apache.hadoop.io.compress.SplittableCompressionCodec; import org.apache.hadoop.mapred.FileInputFormat; import org.apache.hadoop.mapred.FileSplit; import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.JobConfigurable; import org.apache.hadoop.mapred.RecordReader; import org.apache.hadoop.mapred.Reporter; import org.apache.uima.cas.CAS; import de.tudarmstadt.ukp.dkpro.core.api.metadata.type.DocumentMetaData; /** * Input format for generating CAS instances from <Text, Text> key/value pairs * * By default, the value of the <Text, Text> key/value lines in the input files * is used as CAS document text. See {@link setDocumentTextExtractorClass} to * change this behavior. * * @author Johannes Simon * */ public class Text2CASInputFormat extends FileInputFormat<Text, CASWritable> implements JobConfigurable { /** * Provide a custom implementation of this interface if you want the * generated CAS instances to contain text different from the value of * <Text, Text> key/value lines in the input files.<br> * <br> * This is useful e.g. if you want to remove HTML markup or if you want to * extract text from the <b>key</b>, not from the <b>value</b>.<br> * <br> * By default, i.e. if you do not set a custom DocumentTextExtractor, the * input line's value is used as the CAS document text. */ public interface DocumentTextExtractor { Text extractDocumentText(Text key, Text value); } /** * Tells Text2CASInputFormat to use a custom implementation of * DocumentTextExtractor. <br> * By default, i.e. if you do not set a custom DocumentTextExtractor, the * input line's value is used as the CAS document text. * * @param conf * Configuration object * @param extractorClass * Implementation of DocumentTextExtractor */ public static void setDocumentTextExtractorClass(Configuration conf, Class<? extends DocumentTextExtractor> extractorClass) { conf.set("dkpro.uima.text2casinputformat.documenttextextractor", extractorClass.getName()); } /** * Provide a custom implementation of this interface if you want the * generated CAS instances to have metadata different from the value of * <Text, Text> key/value lines in the input files.<br> * <br> * This is useful e.g. if you want to set the language or the URI of the * document <br> * By default, i.e. if you do not set a custom DocumentMetadataExtractor, * the documentId and documentTitle are generated from the key/value pair */ public interface DocumentMetadataExtractor { void extractDocumentMetaData(Text key, Text value, DocumentMetaData metadata); } /** * Tells Text2CASInputFormat to use a custom implementation of * DocumentMetadataExtractor. <br> * By default, i.e. if you do not set a custom DocumentMetadataExtractor, * the documentId and documentTitle are generated from the key/value pair * * @param conf * Configuration object * @param extractorClass * Implementation of DocumentMetadataExtractor */ public static void setDocumentMetadataExtractorClass(Configuration conf, Class<? extends DocumentMetadataExtractor> extractorClass) { conf.set("dkpro.uima.text2casinputformat.documentmetadataextractor", extractorClass.getName()); } /** * Reads in CAS from JSON, XML or plain text. * * <p>This is an internal class. It is only visible to hadoop.io package.</p> * * @author Johannes Simon * */ private class Text2CASRecordReader extends GenericKeyValueLineRecordReader<Text, CASWritable> { private final DocumentTextExtractor textExtractor; private final DocumentMetadataExtractor metadataExtractor; public Text2CASRecordReader(FileSplit fileSplit, JobConf jobConf, Reporter reporter, DocumentTextExtractor textExtractor, DocumentMetadataExtractor metadataExtractor) throws IOException { super(fileSplit, jobConf, reporter); this.textExtractor = textExtractor; this.metadataExtractor = metadataExtractor; } @Override public Text createKey() { return new Text(); } @Override public CASWritable createValue() { return new CASWritable(); } @Override public void convertKey(Text keyFrom, Text valueFrom, Text keyTo) { keyTo.set(keyFrom); } @Override public void convertValue(Text keyFrom, Text valueFrom, CASWritable valueTo) { CAS cas = valueTo.getCAS(); cas.reset(); Text doc = valueFrom; if (textExtractor != null) { doc = textExtractor.extractDocumentText(keyFrom, valueFrom); } cas.setDocumentText(doc.toString()); try { // add some simple metadata String key_as_str = keyFrom.toString(); String key_abbrev = StringUtils.abbreviate(key_as_str, 50); DocumentMetaData metadata = DocumentMetaData.create(cas); metadata.setDocumentTitle(key_abbrev); metadata.setDocumentId(String.format("<%d>%s", key_as_str.hashCode(), key_abbrev)); if (metadataExtractor != null) { metadataExtractor.extractDocumentMetaData(keyFrom, valueFrom, metadata); } } catch (Exception e) { System.err.println("DocumentMetaData already present."); } } } @Override public RecordReader<Text, CASWritable> getRecordReader(InputSplit split, JobConf jobConf, Reporter reporter) throws IOException { DocumentTextExtractor textConverter = null; String textConverterClass = jobConf.get("dkpro.uima.text2casinputformat.documenttextextractor"); if (textConverterClass != null) { try { textConverter = (DocumentTextExtractor) Class.forName(textConverterClass).newInstance(); } catch (InstantiationException e) { e.printStackTrace(); } catch (IllegalAccessException e) { e.printStackTrace(); } catch (ClassNotFoundException e) { e.printStackTrace(); } } DocumentMetadataExtractor metadataConverter = null; String metadataConverterClass = jobConf.get("dkpro.uima.text2casinputformat.documentmetadataextractor"); if (metadataConverterClass != null) { try { metadataConverter = (DocumentMetadataExtractor) Class.forName(metadataConverterClass).newInstance(); } catch (InstantiationException e) { e.printStackTrace(); } catch (IllegalAccessException e) { e.printStackTrace(); } catch (ClassNotFoundException e) { e.printStackTrace(); } } return new Text2CASRecordReader((FileSplit) split, jobConf, reporter, textConverter, metadataConverter); } /** * The following compression-codec logic was copied from TextInputFormat */ private CompressionCodecFactory compressionCodecs = null; @Override public void configure(JobConf conf) { compressionCodecs = new CompressionCodecFactory(conf); } @Override protected boolean isSplitable(FileSystem fs, Path file) { final CompressionCodec codec = compressionCodecs.getCodec(file); if (null == codec) { return true; } return codec instanceof SplittableCompressionCodec; } }