Java tutorial
/******************************************************************************* * Copyright 2015-2016 - CNRS (Centre National de Recherche Scientifique) * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. * *******************************************************************************/ /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright 2, 2015nership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package eu.project.ttc.resources; import java.io.IOException; import java.io.InputStream; import java.util.Collection; import java.util.Scanner; import java.util.Set; import org.apache.commons.io.IOUtils; import org.apache.uima.resource.DataResource; import org.apache.uima.resource.ResourceInitializationException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.base.Objects; import com.google.common.collect.HashMultimap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Multimap; import com.google.common.collect.Sets; public class GeneralLanguageResource implements GeneralLanguage { private static final Logger LOGGER = LoggerFactory.getLogger(GeneralLanguageResource.class); /** * The number of WordAnnotation in the corpus that has been used * to produced this general language resource. * * This field is used as the reference for frequency normalization. */ private int nbCorpusWords = -1; private boolean cumulatedFrequencyMode = false; private int cumulatedFrequency; private Multimap<String, Entry> frequencies; private Set<String> words; public class Entry { private String lemma; private String pattern; private int frequency; public Entry(String lemma, String pattern, int frequency) { super(); this.lemma = lemma; this.pattern = pattern; this.frequency = frequency; } public String getLemma() { return lemma; } public String getPattern() { return pattern; } public int getFrequency() { return frequency; } @Override public int hashCode() { return Objects.hashCode(this.lemma, this.pattern); } @Override public boolean equals(Object obj) { if (obj instanceof Entry) return Objects.equal(((Entry) obj).lemma, ((Entry) obj).pattern); else return false; } } public GeneralLanguageResource() { this.cumulatedFrequency = 0; this.frequencies = HashMultimap.create(); } @Override public void load(DataResource data) throws ResourceInitializationException { try { load(data.getInputStream()); } catch (IOException e) { throw new ResourceInitializationException(e); } } public void load(InputStream inputStream) throws ResourceInitializationException { words = Sets.newHashSet(); Scanner scanner = null; try { scanner = new Scanner(inputStream, "UTF-8"); scanner.useDelimiter("\n"); int index = 0; while (scanner.hasNext()) { index++; String line = scanner.next(); String[] items = line.split("::"); if (items.length == 3) { String key = items[0].trim(); if (!key.contains(" ")) this.words.add(key); Integer value = Integer.valueOf(items[2].trim()); this.cumulatedFrequency += value.intValue(); String lemma = key; this.frequencies.put(lemma, new Entry(lemma, items[1], new Integer(value.intValue()))); } else { throw new IOException("Wrong general language format at line " + index + ": " + line); } } this.words = ImmutableSet.copyOf(this.words); if (this.frequencies.containsKey(PARAM_NB_CORPUS_WORDS)) this.nbCorpusWords = this.frequencies.get(PARAM_NB_CORPUS_WORDS).iterator().next().getFrequency(); else { LOGGER.warn("No such key for in GeneralLanguage resource {}", PARAM_NB_CORPUS_WORDS); LOGGER.warn("Switch to cumulatedFrequency mode"); this.cumulatedFrequencyMode = true; } } catch (Exception e) { throw new ResourceInitializationException(e); } finally { IOUtils.closeQuietly(scanner); } } @Override public int getFrequency(String lemma, String pattern) { Entry e = getEntry(lemma, pattern); return e == null ? 1 : e.getFrequency(); } public Entry getEntry(String lemma, String pattern) { for (Entry e : this.frequencies.get(lemma.toLowerCase())) if (e.getPattern().equals(pattern)) return e; return null; } @Override public int getNbCorpusWords() { return nbCorpusWords; } @Override @Deprecated public double getNormalizedFrequency(String entry) { if (cumulatedFrequencyMode) { Collection<Entry> l = this.frequencies.get(entry.toLowerCase()); if (l.isEmpty()) { return 1.0 / this.cumulatedFrequency; } else { double freq = new Double(l.iterator().next().getFrequency()).doubleValue(); return freq / this.cumulatedFrequency; } } else throw new IllegalStateException("Not in cumulatedFrequencyMode. Should call #getFrequency() instead"); } @Override public boolean isCumulatedFrequencyMode() { return cumulatedFrequencyMode; } @Override public boolean findSingleWord(String word) { return this.words.contains(word); } @Override public Set<String> getWords() { return words; } }