Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.stanbol.commons.opennlp; import java.io.IOException; import java.io.InputStream; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.nio.charset.Charset; import java.security.AccessController; import java.security.PrivilegedActionException; import java.security.PrivilegedExceptionAction; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import opennlp.tools.chunker.Chunker; import opennlp.tools.chunker.ChunkerME; import opennlp.tools.chunker.ChunkerModel; import opennlp.tools.namefind.NameFinderME; import opennlp.tools.namefind.TokenNameFinder; import opennlp.tools.namefind.TokenNameFinderModel; import opennlp.tools.postag.POSModel; import opennlp.tools.postag.POSTagger; import opennlp.tools.postag.POSTaggerME; import opennlp.tools.sentdetect.SentenceDetector; import opennlp.tools.sentdetect.SentenceDetectorME; import opennlp.tools.sentdetect.SentenceModel; import opennlp.tools.tokenize.SimpleTokenizer; import opennlp.tools.tokenize.Tokenizer; import opennlp.tools.tokenize.TokenizerME; import opennlp.tools.tokenize.TokenizerModel; import opennlp.tools.util.InvalidFormatException; import org.apache.commons.io.IOUtils; import org.apache.felix.scr.annotations.Component; import org.apache.felix.scr.annotations.Reference; import org.apache.felix.scr.annotations.Service; import org.apache.stanbol.commons.stanboltools.datafileprovider.DataFileProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * OSGI service that let you load OpenNLP Models via the Stanbol * {@link DataFileProvider} infrastructure. This allows users to copy models * to the 'datafiles' directory or developer to provide models via via OSGI * bundles.<p> * This service also provides methods that directly return the OpenNLP component * wrapping the model. */ @Component(immediate = true) @Service(value = OpenNLP.class) public class OpenNLP { /** * added as link to the download location for requested model files * Will show up in the DataFilePorivder tab in the Apache Felix Web Console */ private static final String DOWNLOAD_ROOT = "http://opennlp.sourceforge.net/models-1.5/"; /** * The logger */ private final Logger log = LoggerFactory.getLogger(getClass()); @Reference private DataFileProvider dataFileProvider; /** * Map holding the already built models * TODO: change to use a WeakReferenceMap */ protected Map<String, Object> models = new HashMap<String, Object>(); /** * used to sync access to the {@link #models} and {@link #modelCreationLock} */ protected ReadWriteLock modelLock = new ReentrantReadWriteLock(); /** * used to avoid loading the same model multiple times in parallel. * The value is a int array with an single element. The int at index zero is * used as reference count. When it reaches zero the mapping can be deleted * from the map. */ protected Map<String, int[]> modelCreationLock = new HashMap<String, int[]>(); /** * Default constructor */ public OpenNLP() { super(); } /** * Constructor intended to be used when running outside an OSGI environment * (e.g. when used for UnitTests) * @param dataFileProvider the dataFileProvider used to load Model data. */ public OpenNLP(DataFileProvider dataFileProvider) { this(); this.dataFileProvider = dataFileProvider; } /** * Getter for the sentence detection model of the parsed language. * If the model is not yet available a new one is built. The required data * are loaded by using the {@link DataFileProvider} service. * @param language the language * @return the model or <code>null</code> if no model data are found * @throws InvalidFormatException in case the found model data are in the wrong format * @throws IOException on any error while reading the model data */ public SentenceModel getSentenceModel(String language) throws InvalidFormatException, IOException { return initModel(String.format("%s-sent.bin", language), SentenceModel.class); } /** * Getter for the sentence detector of the parsed language. * @param language the language * @return the model or <code>null</code> if no model data are found * @throws InvalidFormatException in case the found model data are in the wrong format * @throws IOException on any error while reading the model data */ public SentenceDetector getSentenceDetector(String language) throws IOException { SentenceModel sentModel = getSentenceModel(language); if (sentModel != null) { return new SentenceDetectorME(sentModel); } else { log.debug("No Sentence Detection Model for language '{}'", language); return null; } } /** * Getter for the named entity finder model for the parsed entity type and language. * If the model is not yet available a new one is built. The required data * are loaded by using the {@link DataFileProvider} service. * @param type the type of the named entities to find (person, organization) * @param language the language * @return the model or <code>null</code> if no model data are found * @throws InvalidFormatException in case the found model data are in the wrong format * @throws IOException on any error while reading the model data */ public TokenNameFinderModel getNameModel(String type, String language) throws InvalidFormatException, IOException { return initModel(String.format("%s-ner-%s.bin", language, type), TokenNameFinderModel.class); } /** * Getter for the {@link TokenNameFinder} for the parsed entity type and language. * @param type the type of the named entities to find (person, organization) * @param language the language * @return the model or <code>null</code> if no model data are found * @throws InvalidFormatException in case the found model data are in the wrong format * @throws IOException on any error while reading the model data */ public TokenNameFinder getNameFinder(String type, String language) throws IOException { TokenNameFinderModel model = getNameModel(type, language); if (model != null) { return new NameFinderME(model); } else { log.debug("TokenNameFinder model for type {} and langauge {} not present", type, language); return null; } } /** * Getter for the tokenizer model for the parsed language. * If the model is not yet available a new one is built. The required data * are loaded by using the {@link DataFileProvider} service. * @param language the language * @return the model or <code>null</code> if no model data are found * @throws InvalidFormatException in case the found model data are in the wrong format * @throws IOException on any error while reading the model data */ public TokenizerModel getTokenizerModel(String language) throws InvalidFormatException, IOException { return initModel(String.format("%s-token.bin", language), TokenizerModel.class); } /** * Getter for the Tokenizer of a given language. This first tries to * create an {@link TokenizerME} instance if the required * {@link TokenizerModel} for the parsed language is available. if such a * model is not available it returns the {@link SimpleTokenizer} instance. * @param language the language or <code>null</code> to build a * {@link SimpleTokenizer} * @return the {@link Tokenizer} for the parsed language. */ public Tokenizer getTokenizer(String language) { Tokenizer tokenizer = null; if (language != null) { try { TokenizerModel model = getTokenizerModel(language); if (model != null) { tokenizer = new TokenizerME(model); } } catch (InvalidFormatException e) { log.warn("Unable to load Tokenizer Model for " + language + ": " + "Will use Simple Tokenizer instead", e); } catch (IOException e) { log.warn("Unable to load Tokenizer Model for " + language + ": " + "Will use Simple Tokenizer instead", e); } } if (tokenizer == null) { log.debug("Use Simple Tokenizer for language {}", language); tokenizer = SimpleTokenizer.INSTANCE; } else { log.debug("Use ME Tokenizer for language {}", language); } return tokenizer; } /** * Getter for the "part-of-speech" model for the parsed language. * If the model is not yet available a new one is built. The required data * are loaded by using the {@link DataFileProvider} service. * @param language the language * @return the model or <code>null</code> if no model data are found * @throws InvalidFormatException in case the found model data are in the wrong format * @throws IOException on any error while reading the model data */ public POSModel getPartOfSpeechModel(String language) throws IOException, InvalidFormatException { //typically there are two versions //we prefer the perceptron variant but if not available try to build the other IOException first = null; POSModel model; try { model = initModel(String.format("%s-pos-perceptron.bin", language), POSModel.class); } catch (IOException e) { first = e; log.warn("Unable to laod preceptron based POS model for " + language, e); model = null; } if (model == null) { log.debug("No perceptron based POS model for language " + language + "available. Will try to load maxent model"); try { model = initModel(String.format("%s-pos-maxent.bin", language), POSModel.class); } catch (IOException e) { if (first != null) { throw first; } else { throw e; } } } return model; } /** * Getter for the "part-of-speech" tagger for the parsed language. * @param language the language * @return the model or <code>null</code> if no model data are found * @throws InvalidFormatException in case the found model data are in the wrong format * @throws IOException on any error while reading the model data */ public POSTagger getPartOfSpeechTagger(String language) throws IOException { POSModel posModel = getPartOfSpeechModel(language); if (posModel != null) { return new POSTaggerME(posModel); } else { log.debug("No POS Model for language '{}'", language); return null; } } /** * Getter for the Model with the parsed type, name and properties. * @param modelType the type of the Model (e.g. {@link ChunkerModel}) * @param modelName the name of the model file. MUST BE available via the * {@link DataFileProvider}. * @param properties additional properties about the model (parsed to the * {@link DataFileProvider}. NOTE that "Description", "Model Type" and * "Download Location" are set to default values if not defined in the * parsed value. * @return the loaded (or cached) model * @throws InvalidFormatException in case the found model data are in the wrong format * @throws IOException on any error while reading the model data */ public <T> T getModel(Class<T> modelType, String modelName, Map<String, String> properties) throws InvalidFormatException, IOException { return initModel(modelName, modelType, properties); } /** * Getter for the chunker model for the parsed language. * If the model is not yet available a new one is built. The required data * are loaded by using the {@link DataFileProvider} service. * @param language the language * @return the model or <code>null</code> if no model data are present * @throws InvalidFormatException in case the found model data are in the wrong format * @throws IOException on any error while reading the model data */ public ChunkerModel getChunkerModel(String language) throws InvalidFormatException, IOException { return initModel(String.format("%s-chunker.bin", language), ChunkerModel.class); } /** * Getter for the {@link Chunker} for a given language * @param language the language * @return the {@link Chunker} or <code>null</code> if no model is present * @throws InvalidFormatException in case the found model data are in the wrong format * @throws IOException on any error while reading the model data */ public Chunker getChunker(String language) throws IOException { ChunkerModel chunkerModel = getChunkerModel(language); if (chunkerModel != null) { return new ChunkerME(chunkerModel); } else { log.debug("No Chunker Model for language {}", language); return null; } } // /** // * Activates the component and re-enables all {@link DataFileProvider}s // * previously {@link #registerModelLocation(BundleContext, String...) registered}. // * @param context the context // */ // @Activate // protected void activate(ComponentContext context){ // synchronized (modelLocations) { // for(ModelLocation modelLocation : modelLocations.values()){ // if(modelLocation.provider == null){ // modelLocation.provider = new BundleResourceProvider( // modelLocation.bundleContext, // modelLocation.paths == null ? null : Arrays.asList(modelLocation.paths)); // } // still registered -> should never happen unless activate is called twice // } // } // } // /** // * Deactivates this component. Deactivates all {@link DataFileProvider}s for // * {@link #registerModelLocation(BundleContext, String...) registered} // * locations to search for OpenNLP models and also // * {@link Map#clear() clears} the {@link #models model cache}. // * @param context the context // */ // @Deactivate // protected void deactivate(ComponentContext context){ // synchronized (modelLocations) { // for(ModelLocation modelLocation : modelLocations.values()){ // if(modelLocation.provider != null){ // modelLocation.provider.close(); // modelLocation.provider = null; // } // } // } // //clear the model cache // models.clear(); // } // /** // * Registers the parsed paths as locations to lookup openNLP models.<p> // * This Method is a convenience for manually registering a // * {@link DataFileProvider} that provides the openNLP model classes such as: // * <pre><code> // * protected void activate(ComponentContext context){ // * this.modelProvider = new BundleResourceProvider( // * context.getBundleContext, Arrays.asList("openNLP/models")); // * ... // * } // * // * protected void deactivate(ComponentContext context){ // * if(this.modelProvider != null){ // * modelProvider.close(); // * modelProvider = null; // * } // * ... // * } // * </code></pre><p> // * Note that multiple calls with the same bundleContext will cause previous // * registration for the same {@link BundleContext} to be removed.<p> // * {@link DataFileProvider}s created by this will be removed/added as this // * Component is activated/deactivated. However registrations are not // * persisted and will be gone after an restart of the OSGI environment // * @param bundleContext The context of the bundle used to load openNLP models // * @param searchPaths The paths used to search openNLP models (via the // * bundles classpath). // */ // public void registerModelLocation(BundleContext bundleContext, String...searchPaths){ // if(bundleContext == null){ // throw new IllegalArgumentException("The parsed BundleContext MUST NOT be NULL!"); // } // String bundleSymbolicName = bundleContext.getBundle().getSymbolicName(); // synchronized (modelLocations) { // ModelLocation current = modelLocations.get(bundleSymbolicName); // if(current != null){ // if(Arrays.equals(searchPaths, current.paths)) { // log.debug("ModelLocations for Bundle {} and Paths {} already registered"); // return; // } else { //remove current registration // log.debug("remove existing ModelLocations for Bundle {} and Paths {}", // bundleSymbolicName,current.paths); // if(current.provider != null){ // current.provider.close(); // } // } // } else { // current = new ModelLocation(); // current.bundleContext = bundleContext; // } // current.paths = searchPaths; // current.provider = new BundleResourceProvider(bundleContext, // searchPaths == null ? null : Arrays.asList(searchPaths)); // modelLocations.put(bundleSymbolicName, current); // } // // } // /** // * Removes previously registerd openNLP model locations for the parsed bundle // * context. // * @param bundleContext // */ // public void unregisterModelLocation(BundleContext bundleContext){ // if(bundleContext == null){ // throw new IllegalArgumentException("The parsed BundleContext MUST NOT be NULL!"); // } // String bundleSymbolicName = bundleContext.getBundle().getSymbolicName(); // synchronized (modelLocations) { // ModelLocation current = modelLocations.remove(bundleSymbolicName); // if(current != null){ // log.debug("remove modelLocation for Bundle {} and paths {}", // bundleSymbolicName,current.paths); // if(current.provider != null){ // current.provider.close(); // } // } // } // } /** * Uses generics to build models of the parsed type. The {@link #models} * map is used to lookup already created models. * @param <T> the type of the model to create * @param name the name of the file with the model data * @param modelType the class object representing the model to create * @return the model or <code>null</code> if the model data where not found * @throws InvalidFormatException if the model data are in an invalid format * @throws IOException on any error while loading the model data * @throws IllegalStateException on any Exception while creating the model */ private <T> T initModel(String name, Class<T> modelType) throws InvalidFormatException, IOException { return initModel(name, modelType, null); } /** * Uses generics to build models of the parsed type. The {@link #models} * map is used to lookup already created models. * @param <T> the type of the model to create * @param name the name of the file with the model data * @param modelType the class object representing the model to create * @param modelProperties additional metadata about the requested model * @return the model or <code>null</code> if the model data where not found * @throws InvalidFormatException if the model data are in an invalid format * @throws IOException on any error while loading the model data * @throws IllegalStateException on any Exception while creating the model */ private <T> T initModel(String name, Class<T> modelType, Map<String, String> modelProperties) throws InvalidFormatException, IOException { T model = getCachedModel(name, modelType); if (model != null) { return model; } //else create the model //We need to avoid creating a model twice in parallel modelLock.writeLock().lock(); int[] lock; try { lock = modelCreationLock.get(name); if (lock == null) { lock = new int[] { 0 }; modelCreationLock.put(name, lock); } lock[0]++; } finally { modelLock.writeLock().unlock(); } try { //create only one model with the same name in parallel synchronized (lock) { //now we have the lock ... // first check if it was created while we where waiting for the lock model = getCachedModel(name, modelType); if (model != null) { return model; } //not created in the meantime ... we need to create it! T built = loadModel(name, modelType, modelProperties); //register the model modelLock.writeLock().lock(); try { models.put(name, built); } finally { modelLock.writeLock().unlock(); } return built; } } finally { //we do no longer need the lock lock[0]--; //check if we need to clean up the modelCreationLock map if (lock[0] == 0) { modelLock.writeLock().lock(); try { if (lock[0] == 0) { modelCreationLock.remove(name); } } finally { modelLock.writeLock().unlock(); } } } } private <T> T loadModel(String name, Class<T> modelType, Map<String, String> modelProperties) throws InvalidFormatException, IOException { if (modelProperties != null) { //copy the data to avoid external modifications modelProperties = new HashMap<String, String>(modelProperties); } else { modelProperties = new HashMap<String, String>(); } if (!modelProperties.containsKey("Description")) { modelProperties.put("Description", "Statistical model for OpenNLP"); } if (!modelProperties.containsKey("Model Type")) { modelProperties.put("Model Type", modelType.getSimpleName()); } if (!modelProperties.containsKey("Download Location")) { modelProperties.put("Download Location", DOWNLOAD_ROOT + name); } InputStream modelDataStream; try { modelDataStream = lookupModelStream(name, modelProperties); } catch (IOException e) { log.debug("Unable to load Resource {} via the DataFileProvider", name); return null; } if (modelDataStream == null) { log.debug("Unable to load Resource {} via the DataFileProvider", name); return null; } T built; try { Constructor<T> constructor; constructor = modelType.getConstructor(InputStream.class); built = constructor.newInstance(modelDataStream); } catch (SecurityException e) { throw new IllegalStateException( String.format("Unable to create %s for %s!", modelType.getSimpleName(), name), e); } catch (NoSuchMethodException e) { throw new IllegalStateException( String.format("Unable to create %s for %s!", modelType.getSimpleName(), name), e); } catch (IllegalArgumentException e) { throw new IllegalStateException( String.format("Unable to create %s for %s!", modelType.getSimpleName(), name), e); } catch (InstantiationException e) { throw new IllegalStateException( String.format("Unable to create %s for %s!", modelType.getSimpleName(), name), e); } catch (IllegalAccessException e) { throw new IllegalStateException( String.format("Unable to create %s for %s!", modelType.getSimpleName(), name), e); } catch (InvocationTargetException e) { //this indicates an exception while creating the instance //for InvalidFormatException and IO Exceptions we shall //directly throw the cause. for all others wrap the thrown one //in an IllegalStateException Throwable checked = e.getCause(); if (checked instanceof InvalidFormatException) { throw (InvalidFormatException) checked; } else if (checked instanceof IOException) { throw (IOException) checked; } else { throw new IllegalStateException( String.format("Unable to create %s for %s!", modelType.getSimpleName(), name), e); } } finally { IOUtils.closeQuietly(modelDataStream); } return built; } /** * Used to retrieve a model of the parsed model type from the internal cache * @param name the name of the model * @param modelType the type of the model * @return the model or <code>null</code> if not cached * @throws IllegalStateException if the cached model does not have the * expected type */ private <T> T getCachedModel(String name, Class<T> modelType) { modelLock.readLock().lock(); try { Object model = models.get(name); if (model != null) { if (modelType.isAssignableFrom(model.getClass())) { return modelType.cast(model); } else { throw new IllegalStateException( String.format("Incompatible Model Types for name '%s': present=%s | requested=%s", name, model.getClass(), modelType)); } } else { return null; } } finally { modelLock.readLock().unlock(); } } /** * Lookup an openNLP data file via the {@link #dataFileProvider} * @param modelName the name of the model * @return the stream or <code>null</code> if not found * @throws IOException an any error while opening the model file */ protected InputStream lookupModelStream(final String modelName, final Map<String, String> properties) throws IOException { try { return AccessController.doPrivileged(new PrivilegedExceptionAction<InputStream>() { public InputStream run() throws IOException { return dataFileProvider.getInputStream(null, modelName, properties); } }); } catch (PrivilegedActionException pae) { Exception e = pae.getException(); if (e instanceof IOException) { throw (IOException) e; } else { throw RuntimeException.class.cast(e); } } } /** * Remove non UTF-8 compliant characters (typically control characters) so has to avoid polluting the * annotation graph with snippets that are not serializable as XML. */ protected static String removeNonUtf8CompliantCharacters(final String text) { if (null == text) { return null; } Charset UTF8 = Charset.forName("UTF-8"); byte[] bytes = text.getBytes(UTF8); for (int i = 0; i < bytes.length; i++) { byte ch = bytes[i]; // remove any characters outside the valid UTF-8 range as well as all control characters // except tabs and new lines if (!((ch > 31 && ch < 253) || ch == '\t' || ch == '\n' || ch == '\r')) { bytes[i] = ' '; } } return new String(bytes, UTF8); } }