Example usage for org.apache.commons.io IOUtils lineIterator

List of usage examples for org.apache.commons.io IOUtils lineIterator

  1. HOME
  2. Java
  3. org.apache.commons
  4. org.apache.commons.io.*
  5. IOUtils
  6. lineIterator

Introduction

In this page you can find the example usage for org.apache.commons.io IOUtils lineIterator.

Prototype

public static LineIterator lineIterator(Reader reader)

Source Link

Document

Return an Iterator for the lines in a Reader.

Usage

From source file:mitm.common.postfix.PostfixQueueParser.java

private void parse(String queue, LineHandler lineHandler) {
    Check.notNull(lineHandler, "lineHandler");

    StringReader reader = new StringReader(queue);

    try {//from  w  ww. jav  a  2s  .  c o m
        LineIterator iterator = IOUtils.lineIterator(reader);

        /*
         * If the mail queue is empty the first line is "Mail queue is empty". If the mail queue is
         * not empty the first line should be the header. We should therefore skip the first line
         */
        if (iterator.hasNext()) {
            iterator.next();
        }

        while (iterator.hasNext()) {
            String line = iterator.nextLine();

            if (line.startsWith("--")) {
                /*
                 * The last line starts with -- so we are finished
                 */
                break;
            }

            /*
             * We need to collect all lines that belong to one queue item. Queue items use multiple lines
             * which are separated by an empty line
             */
            while (iterator.hasNext()) {
                String otherLine = iterator.nextLine();

                if (otherLine.length() == 0) {
                    break;
                }

                line = line + " " + otherLine;
            }

            boolean match = true;

            if (searchPattern != null) {
                Matcher matcher = searchPattern.matcher(line);

                if (!matcher.find()) {
                    match = false;
                }
            }

            if (match && !lineHandler.lineFound(line)) {
                break;
            }
        }
    } finally {
        IOUtils.closeQuietly(reader);
    }
}

From source file:com.cognifide.cq.cqsm.core.scripts.ScriptManagerImpl.java

private List<ActionDescriptor> parseIncludeDescriptors(Script script, Map<String, String> definitions,
        List<Script> includes, ResourceResolver resolver) throws ExecutionException {
    final List<ActionDescriptor> descriptors = new LinkedList<>();
    LineIterator lineIterator = IOUtils.lineIterator(new StringReader(script.getData()));

    while (lineIterator.hasNext()) {
        String line = lineIterator.next();
        if (ScriptUtils.isAction(line)) {
            final String command = ScriptUtils.parseCommand(line, definitions);
            final ActionDescriptor descriptor = actionFactory.evaluate(command);
            final Action action = descriptor.getAction();

            descriptors.add(descriptor);

            if (action instanceof DefinitionProvider) {
                definitions.putAll(((DefinitionProvider) action).provideDefinitions(definitions));
            } else if (action instanceof ScriptProvider) {
                getIncludes(definitions, includes, resolver, descriptors, (ScriptProvider) action);
            }/*from w  ww . j  a v  a 2 s . co m*/
        }
    }
    return descriptors;
}

From source file:de.tudarmstadt.ukp.clarin.webanno.automation.util.AutomationUtil.java

public static void addTabSepTrainDocument(MiraTemplate aTemplate, RepositoryService aRepository,
        AutomationService aAutomationService)
        throws IOException, UIMAException, ClassNotFoundException, AutomationException {
    File miraDir = aAutomationService.getMiraDir(aTemplate.getTrainFeature());
    if (!miraDir.exists()) {
        FileUtils.forceMkdir(miraDir);// w w  w.  j a  v  a2  s . co m
    }

    AutomationStatus status = aAutomationService.getAutomationStatus(aTemplate);

    boolean documentChanged = false;
    for (SourceDocument document : aAutomationService
            .listTabSepDocuments(aTemplate.getTrainFeature().getProject())) {
        if (!document.isProcessed()) {
            documentChanged = true;
            break;
        }
    }
    if (!documentChanged) {
        return;
    }

    for (SourceDocument sourceDocument : aAutomationService
            .listTabSepDocuments(aTemplate.getTrainFeature().getProject())) {
        if (sourceDocument.getFeature() != null) { // This is a target layer train document
            continue;
        }
        File trainFile = new File(miraDir,
                sourceDocument.getId() + sourceDocument.getProject().getId() + ".train");
        BufferedWriter trainOut = new BufferedWriter(new FileWriter(trainFile));
        File tabSepFile = new File(aRepository.getDocumentFolder(sourceDocument), sourceDocument.getName());
        LineIterator it = IOUtils.lineIterator(new FileReader(tabSepFile));
        while (it.hasNext()) {
            String line = it.next();
            if (line.trim().equals("")) {
                trainOut.append("\n");
            } else {
                StringTokenizer st = new StringTokenizer(line, "\t");
                if (st.countTokens() != 2) {
                    trainOut.close();
                    throw new AutomationException("This is not a valid TAB-SEP document");
                }
                trainOut.append(getMiraLineForTabSep(st.nextToken(), st.nextToken()));
            }
        }
        sourceDocument.setProcessed(false);
        status.setTrainDocs(status.getTrainDocs() - 1);
        trainOut.close();
    }

}

From source file:brut.androlib.res.AndrolibResources.java

public boolean detectWhetherAppIsFramework(File appDir) throws AndrolibException {
    File publicXml = new File(appDir, "res/values/public.xml");
    if (!publicXml.exists()) {
        return false;
    }/*  www. j  av  a 2s  . c  o m*/

    Iterator<String> it;
    try {
        it = IOUtils.lineIterator(new FileReader(new File(appDir, "res/values/public.xml")));
    } catch (FileNotFoundException ex) {
        throw new AndrolibException("Could not detect whether app is framework one", ex);
    }
    it.next();
    it.next();
    return it.next().contains("0x01");
}

From source file:de.tudarmstadt.ukp.clarin.webanno.automation.util.AutomationUtil.java

public static void generateTrainDocument(MiraTemplate aTemplate, RepositoryService aRepository,
        AnnotationService aAnnotationService, AutomationService aAutomationService, UserDao aUserDao,
        boolean aBase) throws IOException, UIMAException, ClassNotFoundException, AutomationException {
    File miraDir = aAutomationService.getMiraDir(aTemplate.getTrainFeature());
    if (!miraDir.exists()) {
        FileUtils.forceMkdir(miraDir);/*from www.j a va 2s  .  c  o  m*/
    }

    String username = SecurityContextHolder.getContext().getAuthentication().getName();
    User user = aUserDao.get(username);
    AnnotationFeature feature = aTemplate.getTrainFeature();
    boolean documentChanged = false;
    // A. training document for other train layers were changed
    for (AnnotationFeature otherrFeature : aTemplate.getOtherFeatures()) {
        for (SourceDocument document : aRepository
                .listSourceDocuments(aTemplate.getTrainFeature().getProject())) {
            if (!document.isProcessed() && document.getFeature() != null
                    && document.getFeature().equals(otherrFeature)) {
                documentChanged = true;
                break;
            }
        }
    }
    // B. Training document for the main training layer were changed
    for (SourceDocument document : aRepository.listSourceDocuments(feature.getProject())) {
        if (!document.isProcessed()
                && (document.getFeature() != null && document.getFeature().equals(feature))) {
            documentChanged = true;
            break;
        }
    }
    // C. New Curation document arrives
    for (SourceDocument document : aRepository.listSourceDocuments(feature.getProject())) {
        if (!document.isProcessed() && document.getState().equals(SourceDocumentState.CURATION_FINISHED)) {
            documentChanged = true;
            break;
        }
    }
    // D. tab-sep training documents
    for (SourceDocument document : aAutomationService
            .listTabSepDocuments(aTemplate.getTrainFeature().getProject())) {
        if (!document.isProcessed() && document.getFeature() != null && document.getFeature().equals(feature)) {
            documentChanged = true;
            break;
        }
    }
    if (!documentChanged) {
        return;
    }
    File trainFile;
    if (aBase) {
        trainFile = new File(miraDir, feature.getLayer().getId() + "-" + feature.getId() + ".train.ft");
    } else {
        trainFile = new File(miraDir, feature.getLayer().getId() + "-" + feature.getId() + ".train.base");
    }

    AutomationStatus status = aAutomationService.getAutomationStatus(aTemplate);

    BufferedWriter trainOut = new BufferedWriter(new FileWriter(trainFile));
    AutomationTypeAdapter adapter = (AutomationTypeAdapter) TypeUtil.getAdapter(aAnnotationService,
            feature.getLayer());
    // Training documents (Curated or webanno-compatible imported ones - read using UIMA)
    for (SourceDocument sourceDocument : aRepository.listSourceDocuments(feature.getProject())) {
        if ((sourceDocument.isTrainingDocument() && sourceDocument.getFeature() != null
                && sourceDocument.getFeature().equals(feature))) {
            JCas jCas = aRepository.readAnnotationCas(sourceDocument, user);
            for (Sentence sentence : select(jCas, Sentence.class)) {
                if (aBase) {// base training document
                    trainOut.append(getMiraLine(sentence, null, adapter).toString() + "\n");
                } else {// training document with other features
                    trainOut.append(getMiraLine(sentence, feature, adapter).toString() + "\n");
                }
            }
            sourceDocument.setProcessed(!aBase);
            if (!aBase) {
                status.setTrainDocs(status.getTrainDocs() - 1);
            }
        } else if (sourceDocument.getState().equals(SourceDocumentState.CURATION_FINISHED)) {
            JCas jCas = aRepository.readCurationCas(sourceDocument);
            for (Sentence sentence : select(jCas, Sentence.class)) {
                if (aBase) {// base training document
                    trainOut.append(getMiraLine(sentence, null, adapter).toString() + "\n");
                } else {// training document with other features
                    trainOut.append(getMiraLine(sentence, feature, adapter).toString() + "\n");
                }
            }
            sourceDocument.setProcessed(!aBase);
            if (!aBase) {
                status.setTrainDocs(status.getTrainDocs() - 1);
            }
        }
    }
    // Tab-sep documents to be used as a target layer train document
    for (SourceDocument document : aAutomationService.listTabSepDocuments(feature.getProject())) {
        if (document.getFormat().equals(WebAnnoConst.TAB_SEP) && document.getFeature() != null
                && document.getFeature().equals(feature)) {
            File tabSepFile = new File(aRepository.getDocumentFolder(document), document.getName());
            LineIterator it = IOUtils.lineIterator(new FileReader(tabSepFile));
            while (it.hasNext()) {
                String line = it.next();
                if (line.trim().equals("")) {
                    trainOut.append("\n");
                } else {
                    StringTokenizer st = new StringTokenizer(line, "\t");
                    if (st.countTokens() != 2) {
                        trainOut.close();
                        throw new AutomationException("This is not a valid TAB-SEP document");
                    }
                    if (aBase) {
                        trainOut.append(getMiraLineForTabSep(st.nextToken(), ""));
                    } else {
                        trainOut.append(getMiraLineForTabSep(st.nextToken(), st.nextToken()));
                    }
                }
            }
        }
    }
    trainOut.close();
}

From source file:de.tudarmstadt.ukp.clarin.webanno.automation.util.AutomationUtil.java

private static void getFeatureOtherLayer(MiraTemplate aTemplate, RepositoryService aRepository,
        AnnotationService aAnnotationService, AutomationService aAutomationService, UserDao aUserDao,
        int beamSize, boolean maxPosteriors, List<List<String>> predictions, Mira mira, File predFtFile,
        File predcitedFile, SourceDocument document)
        throws FileNotFoundException, IOException, ClassNotFoundException, UIMAException {
    // other layers as training document
    for (AnnotationFeature feature : aTemplate.getOtherFeatures()) {
        int shiftColumns = 0;
        int nbest = 1;
        String modelName = aAutomationService.getMiraModel(feature, true, null).getAbsolutePath();
        if (!new File(modelName).exists()) {
            addOtherFeatureFromAnnotation(feature, aRepository, aAnnotationService, aUserDao, predictions,
                    document);//  ww w  .ja va  2 s.  c o m
            continue;
        }
        String testName = predFtFile.getAbsolutePath();

        PrintStream stream = new PrintStream(predcitedFile);
        BufferedReader input = new BufferedReader(new InputStreamReader(System.in));
        if (testName != null) {
            input = new BufferedReader(new FileReader(testName));
        }
        mira.loadModel(modelName);
        mira.setShiftColumns(shiftColumns);
        mira.nbest = nbest;
        mira.beamSize = beamSize;
        mira.maxPosteriors = maxPosteriors;
        mira.test(input, stream);

        LineIterator it = IOUtils.lineIterator(new FileReader(predcitedFile));
        List<String> annotations = new ArrayList<String>();

        while (it.hasNext()) {
            String line = it.next();
            if (line.trim().equals("")) {
                continue;
            }
            StringTokenizer st = new StringTokenizer(line, " ");
            String tag = "";
            while (st.hasMoreTokens()) {
                tag = st.nextToken();
            }
            annotations.add(tag);
        }
        predictions.add(annotations);
    }
}

From source file:de.tudarmstadt.ukp.clarin.webanno.automation.util.AutomationUtil.java

private static void getFeaturesTabSep(MiraTemplate aTemplate, RepositoryService aRepository,
        AutomationService aAutomationService, int beamSize, boolean maxPosteriors,
        AnnotationFeature layerFeature, List<List<String>> predictions, Mira mira, File predFile,
        File predcitedFile)/*  ww w  . ja va  2s.  co m*/
        throws FileNotFoundException, IOException, ClassNotFoundException, AutomationException {
    for (SourceDocument document : aAutomationService
            .listTabSepDocuments(aTemplate.getTrainFeature().getProject())) {
        int shiftColumns = 0;
        int nbest = 1;
        String modelName = aAutomationService.getMiraModel(layerFeature, true, document).getAbsolutePath();
        if (!new File(modelName).exists()) {
            continue;
        }
        String testName = predFile.getAbsolutePath();

        PrintStream stream = new PrintStream(predcitedFile);
        BufferedReader input = new BufferedReader(new InputStreamReader(System.in));
        if (testName != null) {
            input = new BufferedReader(new FileReader(testName));
        }
        mira.loadModel(modelName);
        mira.setShiftColumns(shiftColumns);
        mira.nbest = nbest;
        mira.beamSize = beamSize;
        mira.maxPosteriors = maxPosteriors;
        try {
            mira.test(input, stream);
        } catch (Exception e) {
            throw new AutomationException(document.getName() + " is Invalid TAB-SEP file!");
        }

        LineIterator it = IOUtils.lineIterator(new FileReader(predcitedFile));
        List<String> annotations = new ArrayList<String>();

        while (it.hasNext()) {
            String line = it.next();
            if (line.trim().equals("")) {
                continue;
            }
            StringTokenizer st = new StringTokenizer(line, " ");
            String tag = "";
            while (st.hasMoreTokens()) {
                tag = st.nextToken();
            }
            annotations.add(tag);
        }
        predictions.add(annotations);
    }
}

From source file:de.tudarmstadt.ukp.clarin.webanno.automation.util.AutomationUtil.java

private static void buildTrainFile(File aBaseFile, File aTrainFile, List<List<String>> aPredictions)
        throws IOException {
    LineIterator it = IOUtils.lineIterator(new FileReader(aBaseFile));
    StringBuffer trainBuffer = new StringBuffer();
    int i = 0;//from  ww  w. jav  a  2  s . c  o  m
    while (it.hasNext()) {
        String line = it.next();
        if (line.trim().equals("")) {
            trainBuffer.append("\n");
            continue;
        }
        StringTokenizer st = new StringTokenizer(line, " ");
        String label = "";
        String feature = "";
        // Except the last token, which is the label, maintain the line
        while (st.hasMoreTokens()) {
            feature = st.nextToken();
            if (label.equals("")) { // first time
                label = feature;
                continue;
            }
            trainBuffer.append(label + " ");
            label = feature;

        }
        for (List<String> prediction : aPredictions) {
            trainBuffer.append(prediction.get(i) + " ");
        }
        // add its own label
        trainBuffer.append(label + "\n");
        i++;
    }
    IOUtils.write(trainBuffer.toString(), new FileOutputStream(aTrainFile));

}

From source file:de.tudarmstadt.ukp.clarin.webanno.automation.util.AutomationUtil.java

private static void buildPredictFile(File apredFt, File aPredFile, List<List<String>> aPredictions,
        AnnotationFeature aFeature) throws IOException {
    LineIterator it = IOUtils.lineIterator(new FileReader(apredFt));
    StringBuffer predBuffer = new StringBuffer();
    int i = 0;//from  w w  w .j a v  a 2s .c o m
    while (it.hasNext()) {
        String line = it.next();
        if (line.trim().equals("")) {
            predBuffer.append("\n");
            continue;
        }
        StringTokenizer st = new StringTokenizer(line, " ");
        // if the target feature is on multiple token, we do not need the morphological features
        // in the prediction file
        if (aFeature.getLayer().isMultipleTokens()) {
            predBuffer.append(st.nextToken() + " ");
        } else {
            while (st.hasMoreTokens()) {
                predBuffer.append(st.nextToken() + " ");
            }
        }
        for (List<String> prediction : aPredictions) {
            predBuffer.append(prediction.get(i) + " ");
        }
        // add its
        predBuffer.append("\n");
        i++;
    }
    IOUtils.write(predBuffer.toString(), new FileOutputStream(aPredFile));

}

From source file:de.tudarmstadt.ukp.clarin.webanno.automation.util.AutomationUtil.java

public static void predict(MiraTemplate aTemplate, RepositoryService aRepository,
        AutomationService aAutomationService, UserDao aUserDao)
        throws CASException, UIMAException, ClassNotFoundException, IOException, BratAnnotationException {
    AnnotationFeature layerFeature = aTemplate.getTrainFeature();

    File miraDir = aAutomationService.getMiraDir(layerFeature);
    AutomationStatus status = aAutomationService.getAutomationStatus(aTemplate);
    for (SourceDocument document : aRepository.listSourceDocuments(layerFeature.getProject())) {
        if (!document.isProcessed() && !document.isTrainingDocument()) {
            File predFile = new File(miraDir, document.getId() + ".pred");
            Mira mira = new Mira();
            int shiftColumns = 0;
            int nbest = 1;
            int beamSize = 0;
            boolean maxPosteriors = false;
            String modelName = aAutomationService.getMiraModel(layerFeature, false, null).getAbsolutePath();
            String testName = predFile.getAbsolutePath();
            File predcitedFile = new File(predFile.getAbsolutePath() + "-pred");
            PrintStream stream = new PrintStream(predcitedFile);
            BufferedReader input = new BufferedReader(new InputStreamReader(System.in));
            if (testName != null) {
                input = new BufferedReader(new FileReader(testName));
            }/*from  w w  w . j  a  v  a2s  .c  o  m*/
            mira.loadModel(modelName);
            mira.setShiftColumns(shiftColumns);
            mira.nbest = nbest;
            mira.beamSize = beamSize;
            mira.maxPosteriors = maxPosteriors;
            mira.test(input, stream);

            LOG.info("Prediction is wrtten to a MIRA File. To be done is writing back to the CAS");
            LineIterator it = IOUtils.lineIterator(new FileReader(predcitedFile));
            List<String> annotations = new ArrayList<String>();

            while (it.hasNext()) {
                String line = it.next();
                if (line.trim().equals("")) {
                    continue;
                }
                StringTokenizer st = new StringTokenizer(line, " ");
                String tag = "";
                while (st.hasMoreTokens()) {
                    tag = st.nextToken();

                }
                annotations.add(tag);
            }

            LOG.info(annotations.size() + " Predictions found to be written to the CAS");
            JCas jCas = null;
            String username = SecurityContextHolder.getContext().getAuthentication().getName();
            User user = aUserDao.get(username);
            try {
                AnnotationDocument annoDocument = aRepository.getAnnotationDocument(document, user);
                jCas = aRepository.readAnnotationCas(annoDocument);
            } catch (DataRetrievalFailureException e) {

            }
            automate(jCas, layerFeature, annotations);
            LOG.info("Predictions found are written to the CAS");
            aRepository.writeCorrectionCas(jCas, document, user);
            document.setProcessed(true);
            status.setAnnoDocs(status.getAnnoDocs() - 1);
        }
    }
}