Java tutorial
/* * KEAModelBuilder.java * Copyright (C) 2001 Eibe Frank * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ package kea; import gate.util.BomStrippingInputStreamReader; import java.io.*; import java.util.*; import weka.core.Attribute; import weka.core.Instance; import weka.core.Instances; import weka.core.OptionHandler; import weka.core.Utils; import weka.core.FastVector; import weka.core.Option; /** * Builds a keyphrase extraction model from the documents in a given * directory. Assumes that the file names for the documents end with * ".txt". Assumes that files containing corresponding * author-assigned keyphrases end with ".key". Optionally an encoding * for the documents/keyphrases can be defined (e.g. for Chinese * text). * * Valid options are:<p> * * -l "directory name"<br> * Specifies name of directory.<p> * * -m "model name"<br> * Specifies name of model.<p> * * -e "encoding"<br> * Specifies encoding.<p> * * -d<br> * Turns debugging mode on.<p> * * -k<br> * Use keyphrase frequency statistic.<p> * * -p<br> * Disallow internal periods.<p> * * -x "length"<br> * Sets maximum phrase length (default: 3).<p> * * -y "length"<br> * Sets minimum phrase length (default: 1).<p> * * -o "number"<br> * The minimum number of times a phrase needs to occur (default: 2). <p> * * -s "name of class implementing list of stop words"<br> * Sets list of stop words to used (default: StopwordsEnglish).<p> * * -t "name of class implementing stemmer"<br> * Sets stemmer to use (default: IteratedLovinsStemmer). <p> * * -n<br> * Do not check for proper nouns. <p> * * @author Eibe Frank (eibe@cs.waikato.ac.nz) * @version 1.0 */ @SuppressWarnings({ "rawtypes", "unchecked", "cast", "resource" }) public class KEAModelBuilder implements OptionHandler { /** Name of directory */ String m_dirName = null; /** Name of model */ String m_modelName = null; /** Encoding */ String m_encoding = "default"; /** Debugging mode? */ boolean m_debug = false; /** Use keyphrase frequency attribute? */ boolean m_useKFrequency = false; /** Disallow internal periods? */ boolean m_disallowIPeriods = false; /** The maximum length of phrases */ private int m_MaxPhraseLength = 3; /** The minimum length of phrases */ private int m_MinPhraseLength = 1; /** The minimum number of occurences of a phrase */ private int m_MinNumOccur = 2; /** The KEA filter object */ KEAFilter m_KEAFilter = null; /** The stemmer to be used */ private Stemmer m_Stemmer = new IteratedLovinsStemmer(); /** The list of stop words to be used */ private Stopwords m_Stopwords = new StopwordsEnglish(); /** Determines whether check for proper nouns is performed */ private boolean m_CheckForProperNouns = true; /** * Get the M_CheckProperNouns value. * @return the M_CheckProperNouns value. */ public boolean getCheckForProperNouns() { return m_CheckForProperNouns; } /** * Set the M_CheckProperNouns value. * @param newM_CheckProperNouns The new M_CheckProperNouns value. */ public void setCheckForProperNouns(boolean newM_CheckProperNouns) { this.m_CheckForProperNouns = newM_CheckProperNouns; } /** * Get the M_Stopwords value. * @return the M_Stopwords value. */ public Stopwords getStopwords() { return m_Stopwords; } /** * Set the M_Stopwords value. * @param newM_Stopwords The new M_Stopwords value. */ public void setStopwords(Stopwords newM_Stopwords) { this.m_Stopwords = newM_Stopwords; } /** * Get the Stemmer value. * @return the Stemmer value. */ public Stemmer getStemmer() { return m_Stemmer; } /** * Set the Stemmer value. * @param newStemmer The new Stemmer value. */ public void setStemmer(Stemmer newStemmer) { this.m_Stemmer = newStemmer; } /** * Get the value of MinNumOccur. * * @return Value of MinNumOccur. */ public int getMinNumOccur() { return m_MinNumOccur; } /** * Set the value of MinNumOccur. * * @param newMinNumOccur Value to assign to MinNumOccur. */ public void setMinNumOccur(int newMinNumOccur) { m_MinNumOccur = newMinNumOccur; } /** * Get the value of MaxPhraseLength. * * @return Value of MaxPhraseLength. */ public int getMaxPhraseLength() { return m_MaxPhraseLength; } /** * Set the value of MaxPhraseLength. * * @param newMaxPhraseLength Value to assign to MaxPhraseLength. */ public void setMaxPhraseLength(int newMaxPhraseLength) { m_MaxPhraseLength = newMaxPhraseLength; } /** * Get the value of MinPhraseLength. * * @return Value of MinPhraseLength. */ public int getMinPhraseLength() { return m_MinPhraseLength; } /** * Set the value of MinPhraseLength. * * @param newMinPhraseLength Value to assign to MinPhraseLength. */ public void setMinPhraseLength(int newMinPhraseLength) { m_MinPhraseLength = newMinPhraseLength; } /** * Get the value of disallowIPeriods. * * @return Value of disallowIPeriods. */ public boolean getDisallowIPeriods() { return m_disallowIPeriods; } /** * Set the value of disallowIPeriods. * * @param newdisallowIPeriods Value to assign to disallowIPeriods. */ public void setDisallowIPeriods(boolean newdisallowIPeriods) { m_disallowIPeriods = newdisallowIPeriods; } /** * Get the value of useKFrequency. * * @return Value of useKFrequency. */ public boolean getUseKFrequency() { return m_useKFrequency; } /** * Set the value of useKFrequency. * * @param newuseKFrequency Value to assign to useKFrequency. */ public void setUseKFrequency(boolean newuseKFrequency) { m_useKFrequency = newuseKFrequency; } /** * Get the value of debug. * * @return Value of debug. */ public boolean getDebug() { return m_debug; } /** * Set the value of debug. * * @param newdebug Value to assign to debug. */ public void setDebug(boolean newdebug) { m_debug = newdebug; } /** * Get the value of encoding. * * @return Value of encoding. */ public String getEncoding() { return m_encoding; } /** * Set the value of encoding. * * @param newencoding Value to assign to encoding. */ public void setEncoding(String newencoding) { m_encoding = newencoding; } /** * Get the value of modelName. * * @return Value of modelName. */ public String getModelName() { return m_modelName; } /** * Set the value of modelName. * * @param newmodelName Value to assign to modelName. */ public void setModelName(String newmodelName) { m_modelName = newmodelName; } /** * Get the value of dirName. * * @return Value of dirName. */ public String getDirName() { return m_dirName; } /** * Set the value of dirName. * * @param newdirName Value to assign to dirName. */ public void setDirName(String newdirName) { m_dirName = newdirName; } /** * Parses a given list of options controlling the behaviour of this object. * Valid options are:<p> * * -l "directory name" <br> * Specifies name of directory.<p> * * -m "model name" <br> * Specifies name of model.<p> * * -e "encoding" <br> * Specifies encoding.<p> * * -d<br> * Turns debugging mode on.<p> * * -k<br> * Use keyphrase frequency statistic.<p> * * -p<br> * Disallow internal periods. <p> * * -x "length"<br> * Sets maximum phrase length (default: 3).<p> * * -y "length"<br> * Sets minimum phrase length (default: 3).<p> * * -o "number"<br> * The minimum number of times a phrase needs to occur (default: 2). <p> * * -s "name of class implementing list of stop words"<br> * Sets list of stop words to used (default: StopwordsEnglish).<p> * * -t "name of class implementing stemmer"<br> * Sets stemmer to use (default: IteratedLovinsStemmer). <p> * * -n<br> * Do not check for proper nouns. <p> * * @param options the list of options as an array of strings * @exception Exception if an option is not supported */ public void setOptions(String[] options) throws Exception { String dirName = Utils.getOption('l', options); if (dirName.length() > 0) { setDirName(dirName); } else { setDirName(null); throw new Exception("Name of directory required argument."); } String modelName = Utils.getOption('m', options); if (modelName.length() > 0) { setModelName(modelName); } else { setModelName(null); throw new Exception("Name of model required argument."); } String encoding = Utils.getOption('e', options); if (encoding.length() > 0) { setEncoding(encoding); } else { setEncoding("default"); } String maxPhraseLengthString = Utils.getOption('x', options); if (maxPhraseLengthString.length() > 0) { setMaxPhraseLength(Integer.parseInt(maxPhraseLengthString)); } else { setMaxPhraseLength(3); } String minPhraseLengthString = Utils.getOption('y', options); if (minPhraseLengthString.length() > 0) { setMinPhraseLength(Integer.parseInt(minPhraseLengthString)); } else { setMinPhraseLength(1); } String minNumOccurString = Utils.getOption('o', options); if (minNumOccurString.length() > 0) { setMinNumOccur(Integer.parseInt(minNumOccurString)); } else { setMinNumOccur(2); } String stopwordsString = Utils.getOption('s', options); if (stopwordsString.length() > 0) { setStopwords((Stopwords) Class.forName(stopwordsString).newInstance()); } String stemmerString = Utils.getOption('t', options); if (stemmerString.length() > 0) { setStemmer((Stemmer) Class.forName(stemmerString).newInstance()); } setDebug(Utils.getFlag('d', options)); setUseKFrequency(Utils.getFlag('k', options)); setDisallowIPeriods(Utils.getFlag('p', options)); setCheckForProperNouns(!Utils.getFlag('n', options)); Utils.checkForRemainingOptions(options); } /** * Gets the current option settings. * * @return an array of strings suitable for passing to setOptions */ public String[] getOptions() { String[] options = new String[20]; int current = 0; options[current++] = "-l"; options[current++] = "" + (getDirName()); options[current++] = "-m"; options[current++] = "" + (getModelName()); options[current++] = "-e"; options[current++] = "" + (getEncoding()); if (getUseKFrequency()) { options[current++] = "-k"; } if (getDebug()) { options[current++] = "-d"; } if (getDisallowIPeriods()) { options[current++] = "-p"; } options[current++] = "-x"; options[current++] = "" + (getMaxPhraseLength()); options[current++] = "-y"; options[current++] = "" + (getMinPhraseLength()); options[current++] = "-o"; options[current++] = "" + (getMinNumOccur()); options[current++] = "-s"; options[current++] = "" + (getStopwords().getClass().getName()); options[current++] = "-t"; options[current++] = "" + (getStemmer().getClass().getName()); if (getCheckForProperNouns()) { options[current++] = "-n"; } while (current < options.length) { options[current++] = ""; } return options; } /** * Returns an enumeration describing the available options. * * @return an enumeration of all the available options */ public Enumeration listOptions() { Vector newVector = new Vector(12); newVector.addElement(new Option("\tSpecifies name of directory.", "l", 1, "-l <directory name>")); newVector.addElement(new Option("\tSpecifies name of model.", "m", 1, "-m <model name>")); newVector.addElement(new Option("\tSpecifies encoding.", "e", 1, "-e <encoding>")); newVector.addElement(new Option("\tTurns debugging mode on.", "d", 0, "-d")); newVector.addElement(new Option("\tUse keyphrase frequency statistic.", "k", 0, "-k")); newVector.addElement(new Option("\tDisallow internal periods.", "p", 0, "-p")); newVector.addElement(new Option("\tSets the maximum phrase length (default: 3).", "x", 1, "-x <length>")); newVector.addElement(new Option("\tSets the minimum phrase length (default: 1).", "y", 1, "-y <length>")); newVector.addElement(new Option("\tSet the minimum number of occurences (default: 2).", "o", 1, "-o")); newVector.addElement(new Option("\tSets the list of stopwords to use (default: StopwordsEnglish).", "s", 1, "-s <name of stopwords class>")); newVector.addElement(new Option("\tSet the stemmer to use (default: IteratedLovinsStemmer).", "t", 1, "-t <name of stemmer class>")); newVector.addElement(new Option("\tDo not check for proper nouns.", "n", 0, "-n")); return newVector.elements(); } /** * Collects the stems of the file names. */ public Hashtable collectStems() throws Exception { Hashtable stems = new Hashtable(); try { File dir = new File(m_dirName); String[] files = dir.list(); for (int i = 0; i < files.length; i++) { if (files[i].endsWith(".key") || files[i].endsWith(".txt")) { String stem = files[i].substring(0, files[i].length() - 4); if (!stems.containsKey(stem)) { stems.put(stem, new Double(0)); } } } } catch (Exception e) { throw new Exception("Problem opening directory " + m_dirName); } return stems; } /** * Builds the model from the files */ public void buildModel(Hashtable stems) throws Exception { // Check whether there is actually any data if (stems.size() == 0) { throw new Exception("Couldn't find any data!"); } FastVector atts = new FastVector(2); atts.addElement(new Attribute("doc", (FastVector) null)); atts.addElement(new Attribute("keyphrases", (FastVector) null)); Instances data = new Instances("keyphrase_training_data", atts, 0); // Build model m_KEAFilter = new KEAFilter(); m_KEAFilter.setDebug(m_debug); m_KEAFilter.setDisallowInternalPeriods(getDisallowIPeriods()); m_KEAFilter.setKFused(getUseKFrequency()); m_KEAFilter.setMaxPhraseLength(getMaxPhraseLength()); m_KEAFilter.setMinPhraseLength(getMinPhraseLength()); m_KEAFilter.setMinNumOccur(getMinNumOccur()); m_KEAFilter.setInputFormat(data); m_KEAFilter.setStemmer(getStemmer()); m_KEAFilter.setStopwords(getStopwords()); m_KEAFilter.setCheckForProperNouns(getCheckForProperNouns()); Enumeration elem = stems.keys(); while (elem.hasMoreElements()) { String str = (String) elem.nextElement(); double[] newInst = new double[2]; try { File txt = new File(m_dirName + "/" + str + ".txt"); BufferedReader is; if (!m_encoding.equals("default")) { is = new BomStrippingInputStreamReader(new FileInputStream(txt), m_encoding); } else { is = new BomStrippingInputStreamReader(new FileInputStream(txt)); } StringBuffer txtStr = new StringBuffer(); int c; while ((c = is.read()) != -1) { txtStr.append((char) c); } newInst[0] = (double) data.attribute(0).addStringValue(txtStr.toString()); } catch (Exception e) { if (m_debug) { System.err.println("Can't find document for stem " + str + "."); } newInst[0] = Instance.missingValue(); } try { File key = new File(m_dirName + "/" + str + ".key"); BufferedReader is; if (!m_encoding.equals("default")) { is = new BomStrippingInputStreamReader(new FileInputStream(key), m_encoding); } else { is = new BomStrippingInputStreamReader(new FileInputStream(key)); } StringBuffer keyStr = new StringBuffer(); int c; while ((c = is.read()) != -1) { keyStr.append((char) c); } newInst[1] = (double) data.attribute(1).addStringValue(keyStr.toString()); } catch (Exception e) { if (m_debug) { System.err.println("Can't find keyphrases for stem " + str + "."); } newInst[1] = Instance.missingValue(); } data.add(new Instance(1.0, newInst)); m_KEAFilter.input(data.instance(0)); data = data.stringFreeStructure(); } m_KEAFilter.batchFinished(); // Get rid of instances in filter while (m_KEAFilter.output() != null) { } ; } /** * Saves the extraction model to the file. */ public void saveModel() throws Exception { BufferedOutputStream bufferedOut = new BufferedOutputStream(new FileOutputStream(m_modelName)); ObjectOutputStream out = new ObjectOutputStream(bufferedOut); out.writeObject(m_KEAFilter); out.flush(); out.close(); } /** * The main method. */ public static void main(String[] ops) { KEAModelBuilder kmb = new KEAModelBuilder(); try { kmb.setOptions(ops); System.err.print("Building model with options: "); String[] optionSettings = kmb.getOptions(); for (int i = 0; i < optionSettings.length; i++) { System.err.print(optionSettings[i] + " "); } System.err.println(); kmb.buildModel(kmb.collectStems()); kmb.saveModel(); } catch (Exception e) { e.printStackTrace(); System.err.println(e.getMessage()); System.err.println("\nOptions:\n"); Enumeration enumeration = kmb.listOptions(); while (enumeration.hasMoreElements()) { Option option = (Option) enumeration.nextElement(); System.err.println(option.synopsis()); System.err.println(option.description()); } } } }