org.languagetool.dev.errorcorpus.PedlerCorpus.java Source code

Introduction

Here is the source code for org.languagetool.dev.errorcorpus.PedlerCorpus.java

Source

/* LanguageTool, a natural language style checker 
 * Copyright (C) 2014 Daniel Naber (http://www.danielnaber.de)
 * 
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
 * USA
 */
package org.languagetool.dev.errorcorpus;

import org.apache.commons.io.IOUtils;
import org.languagetool.markup.AnnotatedText;
import org.languagetool.markup.AnnotatedTextBuilder;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.StringTokenizer;

/**
 * Access to the Jenny Pedler's Real-word Error Corpus corpus.
 * Get it from http://www.dcs.bbk.ac.uk/~jenny/resources.html.
 * @since 2.7
 */
public class PedlerCorpus implements ErrorCorpus {

    private static final String NORMALIZE_REGEX = "\\s*<ERR targ\\s*=\\s*([^>]*?)\\s*>\\s*(.*?)\\s*</ERR>\\s*";

    private final List<String> lines = new ArrayList<>();

    private int pos;

    public PedlerCorpus(File dir) throws IOException {
        File[] files = dir.listFiles();
        if (files == null) {
            throw new RuntimeException("Directory not found or is not a directory: " + dir);
        }
        for (File file : files) {
            if (!file.getName().endsWith(".txt")) {
                System.out.println("Ignoring " + file + ", does not match *.txt");
                continue;
            }
            try (FileInputStream fis = new FileInputStream(file)) {
                lines.addAll(IOUtils.readLines(fis));
            }
        }
    }

    @Override
    public Iterator<ErrorSentence> iterator() {
        return new Iterator<ErrorSentence>() {
            @Override
            public boolean hasNext() {
                return pos < lines.size();
            }

            @Override
            public ErrorSentence next() {
                String line = lines.get(pos++);
                ErrorSentence sentence = getIncorrectSentence(line);
                return sentence;
            }

            @Override
            public void remove() {
                throw new UnsupportedOperationException();
            }
        };
    }

    private ErrorSentence getIncorrectSentence(String line) {
        String normalized = line.replaceAll(NORMALIZE_REGEX, " <ERR targ=$1>$2</ERR> ").replaceAll("\\s+", " ")
                .trim();
        List<Error> errors = new ArrayList<>();
        int startPos = 0;
        while (normalized.indexOf("<ERR targ=", startPos) != -1) {
            int startTagStart = normalized.indexOf("<ERR targ=", startPos);
            int startTagEnd = normalized.indexOf(">", startTagStart);
            int endTagStart = normalized.indexOf("</ERR>", startTagStart);
            int correctionEnd = normalized.indexOf(">", startTagStart);
            String correction = normalized.substring(startTagStart + "<ERR targ=".length(), correctionEnd);
            errors.add(new Error(startTagEnd + 1, endTagStart, correction));
            startPos = startTagStart + 1;
        }
        return new ErrorSentence(normalized, makeAnnotatedText(normalized), errors);
    }

    private AnnotatedText makeAnnotatedText(String pseudoXml) {
        AnnotatedTextBuilder builder = new AnnotatedTextBuilder();
        StringTokenizer tokenizer = new StringTokenizer(pseudoXml, "<>", true);
        boolean inMarkup = false;
        while (tokenizer.hasMoreTokens()) {
            String part = tokenizer.nextToken();
            if (part.startsWith("<")) {
                builder.addMarkup(part);
                inMarkup = true;
            } else if (part.startsWith(">")) {
                inMarkup = false;
                builder.addMarkup(part);
            } else {
                if (inMarkup) {
                    builder.addMarkup(part);
                } else {
                    builder.addText(part);
                }
            }
        }
        return builder.build();
    }

}