Example usage for opennlp.tools.ngram NGramGenerator generate

List of usage examples for opennlp.tools.ngram NGramGenerator generate

Introduction

In this page you can find the example usage for opennlp.tools.ngram NGramGenerator generate.

Prototype

public static List<String> generate(char[] input, int n, String separator) 

Source Link

Document

Generates an nGram based on a char[] input

Usage

From source file:opennlp.tools.languagemodel.NgramLanguageModelTest.java

@Test
public void testTrigramLanguageModelCreationFromText() throws Exception {
    int ngramSize = 3;
    NGramLanguageModel languageModel = new NGramLanguageModel(ngramSize);
    InputStream stream = getClass().getResourceAsStream("/opennlp/tools/languagemodel/sentences.txt");
    for (String line : IOUtils.readLines(stream)) {
        String[] array = line.split(" ");
        List<String> split = Arrays.asList(array);
        List<String> generatedStrings = NGramGenerator.generate(split, ngramSize, " ");
        for (String generatedString : generatedStrings) {
            String[] tokens = generatedString.split(" ");
            if (tokens.length > 0) {
                languageModel.add(new StringList(tokens), 1, ngramSize);
            }/*w  ww . ja v a 2 s  .  c  om*/
        }
    }
    StringList tokens = languageModel.predictNextTokens(new StringList("neural", "network", "language"));
    Assert.assertNotNull(tokens);
    Assert.assertEquals(new StringList("models"), tokens);
    double p1 = languageModel.calculateProbability(new StringList("neural", "network", "language", "models"));
    double p2 = languageModel.calculateProbability(new StringList("neural", "network", "language", "model"));
    Assert.assertTrue(p1 > p2);
}