edu.stanford.nlp.ling.SentenceUtils.java Source code

Java tutorial

Introduction

Here is the source code for edu.stanford.nlp.ling.SentenceUtils.java

Source

package edu.stanford.nlp.ling;

import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;

import edu.stanford.nlp.util.CoreMap;

/**
 * SentenceUtils holds a couple utility methods for lists that are sentences.
 * Those include a method that nicely prints a list of words and methods that
 * construct lists of words from lists of strings.
 *
 * @author Dan Klein
 * @author Christopher Manning (generified)
 * @author John Bauer
 * @version 2010
 */
public class SentenceUtils {

    private SentenceUtils() {
    } // static methods

    /**
     * Create an ArrayList as a list of {@code TaggedWord} from two
     * lists of {@code String}, one for the words, and the second for
     * the tags.
     *
     * @param lex  a list whose items are of type {@code String} and
     *             are the words
     * @param tags a list whose items are of type {@code String} and
     *             are the tags
     * @return The Sentence
     */
    public static ArrayList<TaggedWord> toTaggedList(List<String> lex, List<String> tags) {
        ArrayList<TaggedWord> sent = new ArrayList<>();
        int ls = lex.size();
        int ts = tags.size();
        if (ls != ts) {
            throw new IllegalArgumentException("Sentence.toSentence: lengths differ");
        }
        for (int i = 0; i < ls; i++) {
            sent.add(new TaggedWord(lex.get(i), tags.get(i)));
        }
        return sent;
    }

    /**
     * Create an ArrayList as a list of {@code Word} from a
     * list of {@code String}.
     *
     * @param lex  a list whose items are of type {@code String} and
     *             are the words
     * @return The Sentence
     */
    //TODO wsg2010: This should be deprecated in favor of the method below with new labels
    public static ArrayList<Word> toUntaggedList(List<String> lex) {
        ArrayList<Word> sent = new ArrayList<>();
        for (String str : lex) {
            sent.add(new Word(str));
        }
        return sent;
    }

    /**
     * Create a Sentence as a list of {@code Word} objects from
     * an array of String objects.
     *
     * @param words  The words to make it from
     * @return The Sentence
     */
    //TODO wsg2010: This should be deprecated in favor of the method below with new labels
    public static ArrayList<Word> toUntaggedList(String... words) {
        ArrayList<Word> sent = new ArrayList<>();
        for (String str : words) {
            sent.add(new Word(str));
        }
        return sent;
    }

    public static List<HasWord> toWordList(String... words) {
        List<HasWord> sent = new ArrayList<>();
        for (String word : words) {
            CoreLabel cl = new CoreLabel();
            cl.setValue(word);
            cl.setWord(word);
            sent.add(cl);
        }
        return sent;
    }

    /**
     * Create a sentence as a List of {@code CoreLabel} objects from
     * an array (or varargs) of String objects.
     *
     * @param words The words to make it from
     * @return The Sentence
     */
    public static List<CoreLabel> toCoreLabelList(String... words) {
        List<CoreLabel> sent = new ArrayList<>(words.length);
        for (String word : words) {
            CoreLabel cl = new CoreLabel();
            cl.setValue(word);
            cl.setWord(word);
            sent.add(cl);
        }
        return sent;
    }

    /**
     * Create a sentence as a List of {@code CoreLabel} objects from
     * a List of other label objects.
     *
     * @param words The words to make it from
     * @return The Sentence
     */
    public static List<CoreLabel> toCoreLabelList(List<? extends HasWord> words) {
        List<CoreLabel> sent = new ArrayList<>(words.size());
        for (HasWord word : words) {
            CoreLabel cl = new CoreLabel();
            if (word instanceof Label) {
                cl.setValue(((Label) word).value());
            }
            cl.setWord(word.word());
            if (word instanceof HasTag) {
                cl.setTag(((HasTag) word).tag());
            }
            if (word instanceof HasLemma) {
                cl.setLemma(((HasLemma) word).lemma());
            }
            sent.add(cl);
        }
        return sent;
    }

    /**
     * Returns the sentence as a string with a space between words.
     * It prints out the {@code value()} of each item -
     * this will give the expected answer for a short form representation
     * of the "sentence" over a range of cases.  It is equivalent to
     * calling {@code toString(true)}.
     *
     * TODO: Sentence used to be a subclass of ArrayList, with this
     * method as the toString.  Therefore, there may be instances of
     * ArrayList being printed that expect this method to be used.
     *
     * @param list The tokenized sentence to print out
     * @return The tokenized sentence as a String
     */
    public static <T> String listToString(List<T> list) {
        return listToString(list, true);
    }

    /**
     * Returns the sentence as a string with a space between words.
     * Designed to work robustly, even if the elements stored in the
     * 'Sentence' are not of type Label.
     *
     * This one uses the default separators for any word type that uses
     * separators, such as TaggedWord.
     *
     * @param list The tokenized sentence to print out
     * @param justValue If {@code true} and the elements are of type
     *                  {@code Label}, return just the
     *                  {@code value()} of the {@code Label} of each word;
     *                  otherwise,
     *                  call the {@code toString()} method on each item.
     * @return The sentence in String form
     */
    public static <T> String listToString(List<T> list, final boolean justValue) {
        return listToString(list, justValue, null);
    }

    /**
     * As already described, but if separator is not null, then objects
     * such as TaggedWord
     *
     * @param separator The string used to separate Word and Tag
     *                  in TaggedWord, etc
     */
    public static <T> String listToString(List<T> list, final boolean justValue, final String separator) {
        StringBuilder s = new StringBuilder();
        for (Iterator<T> wordIterator = list.iterator(); wordIterator.hasNext();) {
            T o = wordIterator.next();
            s.append(wordToString(o, justValue, separator));
            if (wordIterator.hasNext()) {
                s.append(' ');
            }
        }
        return s.toString();
    }

    /**
     * Pretty print CoreMap classes using the same semantics as the toShorterString method.
     */
    public static <T extends CoreMap> String listToString(List<T> list, String... keys) {
        StringBuilder s = new StringBuilder();
        for (Iterator<T> wordIterator = list.iterator(); wordIterator.hasNext();) {
            T o = wordIterator.next();
            s.append(o.toShorterString(keys));
            if (wordIterator.hasNext()) {
                s.append(' ');
            }
        }
        return s.toString();
    }

    /**
     * Returns the sentence as a string, based on the original text and spacing
     * prior to tokenization.
     * This method assumes that this extra information has been encoded in CoreLabel
     * objects for each token of the sentence, which do have the original spacing
     * preserved (done with "invertible=true" for PTBTokenizer).
     * However, the method has loose typing for easier inter-operation
     * with old code that still works with a {@code List<HasWord>}.
     *
     * @param list The sentence (List of tokens) to print out
     * @return The original sentence String, which may contain newlines or other artifacts of spacing
     */
    public static <T extends HasWord> String listToOriginalTextString(List<T> list) {
        return listToOriginalTextString(list, true);
    }

    /**
     * Returns the sentence as a string, based on the original text and spacing
     * prior to tokenization.
     * This method assumes that this extra information has been encoded in CoreLabel
     * objects for each token of the sentence, which do have the original spacing
     * preserved (done with "invertible=true" for PTBTokenizer). If that information
     * is not there, you will see null outputs, and if you do not pass in a List
     * of CoreLabel objects, then the code will Exception.
     * The method has loose typing for easier inter-operation
     * with old code that still works with a {@code List<HasWord>}.
     *
     * @param list The sentence (List of tokens) to print out
     * @param printBeforeBeforeStart Whether to print the BeforeAnnotation before the first token
     *                               of the sentence. (In general, the BeforeAnnotation is the same
     *                               as the AfterAnnotation of the preceding token. So, usually this
     *                               is correct to do only for the first sentence of a text.)
     * @return The original sentence String, which may contain newlines or other artifacts of spacing
     */
    public static <T extends HasWord> String listToOriginalTextString(List<T> list,
            boolean printBeforeBeforeStart) {
        if (list == null) {
            return null;
        }
        StringBuilder s = new StringBuilder();
        for (HasWord word : list) {
            CoreLabel cl = (CoreLabel) word;
            // check if this is an MWT
            // skip all but the first MWT token in that case
            if (cl.containsKey(CoreAnnotations.IsMultiWordTokenAnnotation.class) && cl.isMWT()
                    && !cl.isMWTFirst()) {
                continue;
            }
            if (printBeforeBeforeStart) {
                // Only print Before for first token, since otherwise same as After of previous token
                // BUG: if you print a sequence of sentences, you double up between sentence spacing.
                if (cl.get(CoreAnnotations.BeforeAnnotation.class) != null) {
                    s.append(cl.get(CoreAnnotations.BeforeAnnotation.class));
                }
                printBeforeBeforeStart = false;
            }
            if (cl.containsKey(CoreAnnotations.IsMultiWordTokenAnnotation.class) && cl.isMWT()) {
                s.append(cl.get(CoreAnnotations.MWTTokenTextAnnotation.class));
            } else {
                s.append(cl.get(CoreAnnotations.OriginalTextAnnotation.class));
            }
            if (cl.get(CoreAnnotations.AfterAnnotation.class) != null) {
                s.append(cl.get(CoreAnnotations.AfterAnnotation.class));
            } else {
                s.append(' ');
            }
        }
        return s.toString();
    }

    public static <T> String wordToString(T o, final boolean justValue) {
        return wordToString(o, justValue, null);
    }

    public static <T> String wordToString(T o, final boolean justValue, final String separator) {
        if (justValue && o instanceof Label) {
            if (o instanceof CoreLabel) {
                CoreLabel l = (CoreLabel) o;
                String w = l.value();
                if (w == null)
                    w = l.word();
                return w;
            } else {
                return (((Label) o).value());
            }
        } else if (o instanceof CoreLabel) {
            CoreLabel l = ((CoreLabel) o);
            String w = l.value();
            if (w == null)
                w = l.word();
            if (l.tag() != null) {
                if (separator == null) {
                    return w + CoreLabel.TAG_SEPARATOR + l.tag();
                } else {
                    return w + separator + l.tag();
                }
            }
            return w;
            // an interface that covered these next four cases would be
            // nice, but we're moving away from these data types anyway
        } else if (separator != null && o instanceof TaggedWord) {
            return ((TaggedWord) o).toString(separator);
        } else if (separator != null && o instanceof LabeledWord) {
            return ((LabeledWord) o).toString(separator);
        } else if (separator != null && o instanceof WordLemmaTag) {
            return ((WordLemmaTag) o).toString(separator);
        } else if (separator != null && o instanceof WordTag) {
            return ((WordTag) o).toString(separator);
        } else {
            return (o.toString());
        }
    }

    /**
     * Returns the substring of the sentence from start (inclusive)
     * to end (exclusive).
     *
     * @param start Leftmost index of the substring
     * @param end Rightmost index of the ngram
     * @return The ngram as a String. Currently returns null if one of the indices is out of bounds.
     *         But maybe it should exception instead.
     */
    public static <T> String extractNgram(List<T> list, int start, int end) {
        if (start < 0 || end > list.size() || start >= end)
            return null;
        final StringBuilder sb = new StringBuilder();
        for (int i = start; i < end; i++) {
            T o = list.get(i);
            if (sb.length() != 0)
                sb.append(' ');
            sb.append((o instanceof HasWord) ? ((HasWord) o).word() : o.toString());
        }
        return sb.toString();
    }

}