org.apache.nutch.analysis.lang.NGramProfile.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.nutch.analysis.lang.NGramProfile.java

Source

/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.nutch.analysis.lang;

// JDK imports
import java.io.File;
import java.io.InputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStreamReader;
import java.io.BufferedInputStream;
import java.util.Date;
import java.util.List;
import java.util.Iterator;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;

// Commons Logging imports
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

// Lucene imports
import org.apache.lucene.analysis.Token;

// Nutch imports
import org.apache.nutch.util.LogUtil;

/**
 * This class runs a ngram analysis over submitted text, results might be used
 * for automatic language identifiaction.
 * 
 * The similarity calculation is at experimental level. You have been warned.
 * 
 * Methods are provided to build new NGramProfiles profiles.
 * 
 * @author Sami Siren
 * @author Jerome Charron - http://frutch.free.fr/
 */
public class NGramProfile {

    public static final Log LOG = LogFactory.getLog(NGramProfile.class);

    /** The minimum length allowed for a ngram. */
    final static int ABSOLUTE_MIN_NGRAM_LENGTH = 1;

    /** The maximum length allowed for a ngram. */
    final static int ABSOLUTE_MAX_NGRAM_LENGTH = 4;

    /** The default min length of ngram */
    final static int DEFAULT_MIN_NGRAM_LENGTH = 3;

    /** The default max length of ngram */
    final static int DEFAULT_MAX_NGRAM_LENGTH = 3;

    /** The ngram profile file extension */
    static final String FILE_EXTENSION = "ngp";

    /** The profile max size (number of ngrams of the same size) */
    static final int MAX_SIZE = 1000;

    /** separator char */
    static final char SEPARATOR = '_';
    /** The String form of the separator char */
    private final static String SEP_CHARSEQ = new String(new char[] { SEPARATOR });

    /** The profile's name */
    private String name = null;

    /** The NGrams of this profile sorted on the number of occurences */
    private List<NGramEntry> sorted = null;

    /** The min length of ngram */
    private int minLength = DEFAULT_MIN_NGRAM_LENGTH;

    /** The max length of ngram */
    private int maxLength = DEFAULT_MAX_NGRAM_LENGTH;

    /** The total number of ngrams occurences */
    private int[] ngramcounts = null;

    /** An index of the ngrams of the profile */
    private Map<CharSequence, NGramEntry> ngrams = null;

    /** A StringBuffer used during analysis */
    private QuickStringBuffer word = new QuickStringBuffer();

    /**
     * Construct a new ngram profile
     * 
     * @param name is the name of the profile
     * @param minlen is the min length of ngram sequences
     * @param maxlen is the max length of ngram sequences
     */
    public NGramProfile(String name, int minlen, int maxlen) {
        // TODO: Compute the initial capacity using minlen and maxlen.
        this.ngrams = new HashMap<CharSequence, NGramEntry>(4000);
        this.minLength = minlen;
        this.maxLength = maxlen;
        this.name = name;
    }

    /**
     * @return Returns the name.
     */
    public String getName() {
        return name;
    }

    /**
     * Add ngrams from a token to this profile
     * 
     * @param t is the Token to be added
     */
    public void add(Token t) {
        add(new StringBuffer().append(SEPARATOR).append(t.term()).append(SEPARATOR));
    }

    /**
     * Add ngrams from a single word to this profile
     * 
     * @param word is the word to add
     */
    public void add(StringBuffer word) {
        for (int i = minLength; (i <= maxLength) && (i < word.length()); i++) {
            add(word, i);
        }
    }

    /**
     * Add the last NGrams from the specified word.
     */
    private void add(QuickStringBuffer word) {
        int wlen = word.length();
        if (wlen >= minLength) {
            int max = Math.min(maxLength, wlen);
            for (int i = minLength; i <= max; i++) {
                add(word.subSequence(wlen - i, wlen));
            }
        }
    }

    /**
     * Add ngrams from a single word in this profile
     *
     * @param word is the word to add
     * @param n is the ngram size
     */
    private void add(CharSequence cs) {

        if (cs.equals(SEP_CHARSEQ)) {
            return;
        }
        NGramEntry nge = ngrams.get(cs);
        if (nge == null) {
            nge = new NGramEntry(cs);
            ngrams.put(cs, nge);
        }
        nge.inc();
    }

    /**
     * Analyze a piece of text
     * 
     * @param text the text to be analyzed
     */
    public void analyze(StringBuilder text) {

        if (ngrams != null) {
            ngrams.clear();
            sorted = null;
            ngramcounts = null;
        }

        word.clear().append(SEPARATOR);
        for (int i = 0; i < text.length(); i++) {
            char c = Character.toLowerCase(text.charAt(i));

            if (Character.isLetter(c)) {
                add(word.append(c));
            } else {
                //found word boundary
                if (word.length() > 1) {
                    //we have a word!
                    add(word.append(SEPARATOR));
                    word.clear().append(SEPARATOR);
                }
            }
        }

        if (word.length() > 1) {
            //we have a word!
            add(word.append(SEPARATOR));
        }
        normalize();
    }

    /**
     * @param word
     * @param n sequence length
     */
    private void add(StringBuffer word, int n) {
        for (int i = 0; i <= word.length() - n; i++) {
            add(word.subSequence(i, i + n));
        }
    }

    /**
     * Normalize the profile (calculates the ngrams frequencies)
     */
    protected void normalize() {

        NGramEntry e = null;
        //List sorted = getSorted();
        Iterator<NGramEntry> i = ngrams.values().iterator();

        // Calculate ngramcount if not already done
        if (ngramcounts == null) {
            ngramcounts = new int[maxLength + 1];
            while (i.hasNext()) {
                e = i.next();
                ngramcounts[e.size()] += e.count;
            }
        }

        i = ngrams.values().iterator();
        while (i.hasNext()) {
            e = i.next();
            e.frequency = (float) e.count / (float) ngramcounts[e.size()];
        }
    }

    /**
     * Return a sorted list of ngrams (sort done by 1. frequency 2. sequence)
     * 
     * @return sorted vector of ngrams
     */
    public List<NGramEntry> getSorted() {
        // make sure sorting is done only once
        if (sorted == null) {
            sorted = new ArrayList<NGramEntry>(ngrams.values());
            Collections.sort(sorted);

            // trim at NGRAM_LENGTH entries
            if (sorted.size() > MAX_SIZE) {
                sorted = sorted.subList(0, MAX_SIZE);
            }
        }
        return sorted;
    }

    // Inherited JavaDoc
    public String toString() {

        StringBuffer s = new StringBuffer().append("NGramProfile: ").append(name).append("\n");

        Iterator<NGramEntry> i = getSorted().iterator();

        while (i.hasNext()) {
            NGramEntry entry = i.next();
            s.append("[").append(entry.seq).append("/").append(entry.count).append("/").append(entry.frequency)
                    .append("]\n");
        }
        return s.toString();
    }

    /**
     * Calculate a score how well NGramProfiles match each other
     * 
     * @param another
     *          ngram profile to compare against
     * @return similarity 0=exact match
     */
    public float getSimilarity(NGramProfile another) {

        float sum = 0;

        try {
            Iterator<NGramEntry> i = another.getSorted().iterator();
            while (i.hasNext()) {
                NGramEntry other = i.next();
                if (ngrams.containsKey(other.seq)) {
                    sum += Math.abs((other.frequency - ngrams.get(other.seq).frequency)) / 2;
                } else {
                    sum += other.frequency;
                }
            }
            i = getSorted().iterator();
            while (i.hasNext()) {
                NGramEntry other = i.next();
                if (another.ngrams.containsKey(other.seq)) {
                    sum += Math.abs((other.frequency - another.ngrams.get(other.seq).frequency)) / 2;
                } else {
                    sum += other.frequency;
                }
            }
        } catch (Exception e) {
            if (LOG.isFatalEnabled()) {
                LOG.fatal(e.toString());
            }
        }
        return sum;
    }

    /**
     * Loads a ngram profile from an InputStream
     * (assumes UTF-8 encoded content)
     * @param is the InputStream to read
     */
    public void load(InputStream is) throws IOException {

        ngrams.clear();
        ngramcounts = new int[maxLength + 1];
        BufferedReader reader = new BufferedReader(new InputStreamReader(is, "UTF-8"));
        String line = null;

        while ((line = reader.readLine()) != null) {

            // # starts a comment line
            if (line.charAt(0) != '#') {
                int spacepos = line.indexOf(' ');
                String ngramsequence = line.substring(0, spacepos).trim();
                int len = ngramsequence.length();
                if ((len >= minLength) && (len <= maxLength)) {
                    int ngramcount = Integer.parseInt(line.substring(spacepos + 1));
                    NGramEntry en = new NGramEntry(ngramsequence, ngramcount);
                    ngrams.put(en.getSeq(), en);
                    ngramcounts[len] += ngramcount;
                }
            }
        }
        normalize();
    }

    /**
     * Create a new Language profile from (preferably quite large) text file
     * 
     * @param name is thename of profile
     * @param is is the stream to read
     * @param encoding is the encoding of stream
     */
    public static NGramProfile create(String name, InputStream is, String encoding) {

        NGramProfile newProfile = new NGramProfile(name, ABSOLUTE_MIN_NGRAM_LENGTH, ABSOLUTE_MAX_NGRAM_LENGTH);
        BufferedInputStream bis = new BufferedInputStream(is);

        byte buffer[] = new byte[4096];
        StringBuilder text = new StringBuilder();
        int len;

        try {
            while ((len = bis.read(buffer)) != -1) {
                text.append(new String(buffer, 0, len, encoding));
            }
        } catch (IOException e) {
            e.printStackTrace(LogUtil.getWarnStream(LOG));
        }

        newProfile.analyze(text);
        return newProfile;
    }

    /**
     * Writes NGramProfile content into OutputStream, content is outputted with
     * UTF-8 encoding
     * 
     * @param os the Stream to output to
     * @throws IOException
     */
    public void save(OutputStream os) throws IOException {

        // Write header
        os.write(("# NgramProfile generated at " + new Date() + " for Nutch Language Identification\n").getBytes());

        // And then each ngram

        // First dispatch ngrams in many lists depending on their size
        // (one list for each size, in order to store MAX_SIZE ngrams for each
        // size of ngram)
        List<NGramEntry> list = new ArrayList<NGramEntry>();
        List<NGramEntry> sublist = new ArrayList<NGramEntry>();
        NGramEntry[] entries = ngrams.values().toArray(new NGramEntry[ngrams.size()]);
        for (int i = minLength; i <= maxLength; i++) {
            for (int j = 0; j < entries.length; j++) {
                if (entries[j].getSeq().length() == i) {
                    sublist.add(entries[j]);
                }
            }
            Collections.sort(sublist);
            if (sublist.size() > MAX_SIZE) {
                sublist = sublist.subList(0, MAX_SIZE);
            }
            list.addAll(sublist);
            sublist.clear();
        }
        for (int i = 0; i < list.size(); i++) {
            NGramEntry e = list.get(i);
            String line = e.toString() + " " + e.getCount() + "\n";
            os.write(line.getBytes("UTF-8"));
        }
        os.flush();
    }

    /**
     * main method used for testing only
     * 
     * @param args
     */
    public static void main(String args[]) {

        String usage = "Usage: NGramProfile " + "[-create profilename filename encoding] "
                + "[-similarity file1 file2] " + "[-score profile-name filename encoding]";
        int command = 0;

        final int CREATE = 1;
        final int SIMILARITY = 2;
        final int SCORE = 3;

        String profilename = "";
        String filename = "";
        String filename2 = "";
        String encoding = "";

        if (args.length == 0) {
            System.err.println(usage);
            System.exit(-1);
        }

        for (int i = 0; i < args.length; i++) { // parse command line
            if (args[i].equals("-create")) { // found -create option
                command = CREATE;
                profilename = args[++i];
                filename = args[++i];
                encoding = args[++i];
            }

            if (args[i].equals("-similarity")) { // found -similarity option
                command = SIMILARITY;
                filename = args[++i];
                filename2 = args[++i];
                encoding = args[++i];
            }

            if (args[i].equals("-score")) { // found -Score option
                command = SCORE;
                profilename = args[++i];
                filename = args[++i];
                encoding = args[++i];
            }
        }

        try {

            switch (command) {

            case CREATE:

                File f = new File(filename);
                FileInputStream fis = new FileInputStream(f);
                NGramProfile newProfile = NGramProfile.create(profilename, fis, encoding);
                fis.close();
                f = new File(profilename + "." + FILE_EXTENSION);
                FileOutputStream fos = new FileOutputStream(f);
                newProfile.save(fos);
                System.out.println("new profile " + profilename + "." + FILE_EXTENSION + " was created.");
                break;

            case SIMILARITY:

                f = new File(filename);
                fis = new FileInputStream(f);
                newProfile = NGramProfile.create(filename, fis, encoding);
                newProfile.normalize();

                f = new File(filename2);
                fis = new FileInputStream(f);
                NGramProfile newProfile2 = NGramProfile.create(filename2, fis, encoding);
                newProfile2.normalize();
                System.out.println("Similarity is " + newProfile.getSimilarity(newProfile2));
                break;

            case SCORE:
                f = new File(filename);
                fis = new FileInputStream(f);
                newProfile = NGramProfile.create(filename, fis, encoding);

                f = new File(profilename + "." + FILE_EXTENSION);
                fis = new FileInputStream(f);
                NGramProfile compare = new NGramProfile(profilename, DEFAULT_MIN_NGRAM_LENGTH,
                        DEFAULT_MAX_NGRAM_LENGTH);
                compare.load(fis);
                System.out.println("Score is " + compare.getSimilarity(newProfile));
                break;

            }

        } catch (Exception e) {
            if (LOG.isFatalEnabled()) {
                LOG.fatal("Caught an exception:" + e);
            }
        }
    }

    /**
     * Inner class that describes a NGram
     */
    class NGramEntry implements Comparable<NGramEntry> {

        /** The NGRamProfile this NGram is related to */
        private NGramProfile profile = null;

        /** The sequence of characters of the ngram */
        CharSequence seq = null;

        /** The number of occurences of this ngram in its profile */
        private int count = 0;

        /** The frequency of this ngram in its profile */
        private float frequency = 0.0F;

        /** 
         * Constructs a new NGramEntry
         * @param seq is the sequence of characters of the ngram
         */
        public NGramEntry(CharSequence seq) {
            this.seq = seq;
        }

        /** 
         * Constructs a new NGramEntry
         * @param seq is the sequence of characters of the ngram
         * @param count is the number of occurences of this ngram
         */
        public NGramEntry(String seq, int count) {
            this.seq = new StringBuffer(seq).subSequence(0, seq.length());
            this.count = count;
        }

        /**
         * Returns the number of occurences of this ngram in its profile
         * @return the number of occurences of this ngram in its profile
         */
        public int getCount() {
            return count;
        }

        /**
         * Returns the frequency of this ngram in its profile
         * @return the frequency of this ngram in its profile
         */
        public float getFrequency() {
            return frequency;
        }

        /**
         * Returns the sequence of characters of this ngram
         * @return the sequence of characters of this ngram
         */
        public CharSequence getSeq() {
            return seq;
        }

        /**
         * Returns the size of this ngram
         * @return the size of this ngram
         */
        public int size() {
            return seq.length();
        }

        // Inherited JavaDoc
        public int compareTo(NGramEntry ngram) {
            int diff = Float.compare(ngram.getFrequency(), frequency);
            if (diff != 0) {
                return diff;
            } else {
                return (toString().compareTo(ngram.toString()));
            }
        }

        /**
         * Increments the number of occurences of this ngram.
         */
        public void inc() {
            count++;
        }

        /**
         * Associated a profile to this ngram
         * @param profile is the profile associated to this ngram
         */
        public void setProfile(NGramProfile profile) {
            this.profile = profile;
        }

        /**
         * Returns the profile associated to this ngram
         * @return the profile associated to this ngram
         */
        public NGramProfile getProfile() {
            return profile;
        }

        // Inherited JavaDoc
        public String toString() {
            return seq.toString();
        }

        // Inherited JavaDoc
        public int hashCode() {
            return seq.hashCode();
        }

        // Inherited JavaDoc
        public boolean equals(Object obj) {

            NGramEntry ngram = null;
            try {
                ngram = (NGramEntry) obj;
                return ngram.seq.equals(seq);
            } catch (Exception e) {
                return false;
            }
        }

    }

    private class QuickStringBuffer implements CharSequence {

        private char value[];

        private int count;

        QuickStringBuffer() {
            this(16);
        }

        QuickStringBuffer(char[] value) {
            this.value = value;
            count = value.length;
        }

        QuickStringBuffer(int length) {
            value = new char[length];
        }

        QuickStringBuffer(String str) {
            this(str.length() + 16);
            append(str);
        }

        public int length() {
            return count;
        }

        private void expandCapacity(int minimumCapacity) {
            int newCapacity = (value.length + 1) * 2;
            if (newCapacity < 0) {
                newCapacity = Integer.MAX_VALUE;
            } else if (minimumCapacity > newCapacity) {
                newCapacity = minimumCapacity;
            }

            char newValue[] = new char[newCapacity];
            System.arraycopy(value, 0, newValue, 0, count);
            value = newValue;
        }

        QuickStringBuffer clear() {
            count = 0;
            return this;
        }

        public char charAt(int index) {
            return value[index];
        }

        QuickStringBuffer append(String str) {
            if (str == null) {
                str = String.valueOf(str);
            }

            int len = str.length();
            int newcount = count + len;
            if (newcount > value.length) {
                expandCapacity(newcount);
            }
            str.getChars(0, len, value, count);
            count = newcount;
            return this;
        }

        QuickStringBuffer append(char c) {
            int newcount = count + 1;
            if (newcount > value.length) {
                expandCapacity(newcount);
            }
            value[count++] = c;
            return this;
        }

        public CharSequence subSequence(int start, int end) {
            return new String(value, start, end - start);
        }

        public String toString() {
            return new String(this.value);
        }
    }

}