Example usage for org.apache.commons.io IOUtils lineIterator

Introduction

In this page you can find the example usage for org.apache.commons.io IOUtils lineIterator.

Prototype

public static LineIterator lineIterator(Reader reader) 

Source Link

Document

Return an Iterator for the lines in a Reader.

Usage

From source file:io.druid.firehose.oss.StaticOSSFirehoseFactory.java

@Override
public Firehose connect(StringInputRowParser firehoseParser) throws IOException {

    Preconditions.checkNotNull(ossClient, "null ossClient");

    final LinkedList<URI> objectQueue = Lists.newLinkedList(uris);

    return new FileIteratingFirehose(new Iterator<LineIterator>() {
        @Override//w w w.  j  a  v a 2 s  . co m
        public boolean hasNext() {
            return !objectQueue.isEmpty();
        }

        @Override
        public LineIterator next() {
            final URI nextURI = objectQueue.poll();

            final String bucket = nextURI.getAuthority();
            final String key = nextURI.getPath().startsWith("/") ? nextURI.getPath().substring(1)
                    : nextURI.getPath();

            log.info("reading from bucket[%s] object[%s] (%s)", bucket, key, nextURI);

            try {
                final InputStream innerInputStream = ossClient.getObject(bucket, key).getObjectContent();

                final InputStream outerInputStream = key.endsWith(".gz")
                        ? CompressionUtils.gzipInputStream(innerInputStream)
                        : innerInputStream;

                return IOUtils.lineIterator(
                        new BufferedReader(new InputStreamReader(outerInputStream, Charsets.UTF_8)));
            } catch (Exception e) {
                log.error(e, "exception reading from bucket[%s] object[%s]", bucket, key);

                throw Throwables.propagate(e);
            }
        }

        @Override
        public void remove() {
            throw new UnsupportedOperationException();
        }
    }, firehoseParser);
}

---

From source file:io.druid.firehose.s3.StaticS3FirehoseFactory.java

@Override
public Firehose connect(StringInputRowParser firehoseParser) throws IOException {
    Preconditions.checkNotNull(s3Client, "null s3Client");

    final LinkedList<URI> objectQueue = Lists.newLinkedList(uris);

    return new FileIteratingFirehose(new Iterator<LineIterator>() {
        @Override//from ww w.java2  s  .co m
        public boolean hasNext() {
            return !objectQueue.isEmpty();
        }

        @Override
        public LineIterator next() {
            final URI nextURI = objectQueue.poll();

            final String s3Bucket = nextURI.getAuthority();
            final S3Object s3Object = new S3Object(
                    nextURI.getPath().startsWith("/") ? nextURI.getPath().substring(1) : nextURI.getPath());

            log.info("Reading from bucket[%s] object[%s] (%s)", s3Bucket, s3Object.getKey(), nextURI);

            try {
                final InputStream innerInputStream = s3Client
                        .getObject(new S3Bucket(s3Bucket), s3Object.getKey()).getDataInputStream();

                final InputStream outerInputStream = s3Object.getKey().endsWith(".gz")
                        ? CompressionUtils.gzipInputStream(innerInputStream)
                        : innerInputStream;

                return IOUtils.lineIterator(
                        new BufferedReader(new InputStreamReader(outerInputStream, Charsets.UTF_8)));
            } catch (Exception e) {
                log.error(e, "Exception reading from bucket[%s] object[%s]", s3Bucket, s3Object.getKey());

                throw Throwables.propagate(e);
            }
        }

        @Override
        public void remove() {
            throw new UnsupportedOperationException();
        }
    }, firehoseParser);
}

---

From source file:io.druid.firehose.cloudfiles.StaticCloudFilesFirehoseFactory.java

@Override
public Firehose connect(StringInputRowParser stringInputRowParser) throws IOException, ParseException {
    Preconditions.checkNotNull(cloudFilesApi, "null cloudFilesApi");

    final LinkedList<CloudFilesBlob> objectQueue = Lists.newLinkedList(blobs);

    return new FileIteratingFirehose(new Iterator<LineIterator>() {

        @Override//from ww w . java  2  s . co  m
        public boolean hasNext() {
            return !objectQueue.isEmpty();
        }

        @Override
        public LineIterator next() {
            final CloudFilesBlob nextURI = objectQueue.poll();

            final String region = nextURI.getRegion();
            final String container = nextURI.getContainer();
            final String path = nextURI.getPath();

            log.info("Retrieving file from region[%s], container[%s] and path [%s]", region, container, path);
            CloudFilesObjectApiProxy objectApi = new CloudFilesObjectApiProxy(cloudFilesApi, region, container);
            final CloudFilesByteSource byteSource = new CloudFilesByteSource(objectApi, path);

            try {
                final InputStream innerInputStream = byteSource.openStream();
                final InputStream outerInputStream = path.endsWith(".gz")
                        ? CompressionUtils.gzipInputStream(innerInputStream)
                        : innerInputStream;

                return IOUtils.lineIterator(
                        new BufferedReader(new InputStreamReader(outerInputStream, Charsets.UTF_8)));
            } catch (IOException e) {
                log.error(e, "Exception opening container[%s] blob[%s] from region[%s]", container, path,
                        region);

                throw Throwables.propagate(e);
            }
        }

        @Override
        public void remove() {
            throw new UnsupportedOperationException();
        }

    }, stringInputRowParser);
}

---

From source file:mitm.common.postfix.SaslPasswordManager.java

protected static List<SaslPassword> parseContent(String input) {
    List<SaslPassword> passwords = new LinkedList<SaslPassword>();

    if (StringUtils.isEmpty(input)) {
        return passwords;
    }/*from   w  w w. j a v a 2 s.  co m*/

    LineIterator lineIterator = IOUtils.lineIterator(new StringReader(input));

    while (lineIterator.hasNext()) {
        String line = lineIterator.nextLine();

        if (!StringUtils.isBlank(line)) {
            Matcher m = SASL_PASSWORD_PATTERN.matcher(line);

            if (m.matches()) {
                boolean mxLookup = !("[".equals(m.group(1)) && "]".equals(m.group(3)));
                String server = StringUtils.trim(m.group(2));
                Integer port = NumberUtils.toInt(StringUtils.trim(m.group(4)), 0);
                String username = StringUtils.trim(m.group(5));
                String password = StringUtils.trim(m.group(6));

                SaslPassword saslPassword = new SaslPassword();

                saslPassword.setMxLookup(mxLookup);
                saslPassword.setServer(server);
                saslPassword.setPort(port != 0 ? port : null);
                saslPassword.setUsername(username);
                saslPassword.setPassword(password);

                passwords.add(saslPassword);
            }
        }
    }

    return passwords;
}

---

From source file:ch.systemsx.cisd.openbis.plugin.generic.client.web.server.parser.SampleUploadSectionsParser.java

private static List<FileSection> extractSections(IUncheckedMultipartFile multipartFile) {
    List<FileSection> sections = new ArrayList<FileSection>();
    InputStreamReader reader = new InputStreamReader(multipartFile.getInputStream());
    try {//  w  w  w .j av a 2  s  .co  m
        LineIterator it = IOUtils.lineIterator(reader);
        StringBuilder sb = null;
        String sectionName = null;
        while (it.hasNext()) {
            String line = it.nextLine();
            String newSectionName = tryGetSectionName(line);
            if (newSectionName != null) {
                if (sectionName != null && sb != null) {
                    sections.add(new FileSection(sb.toString(), sectionName));
                }
                sectionName = newSectionName;
                sb = new StringBuilder();
            } else if (sectionName == null || sb == null) {
                throw new UserFailureException("Discovered the unnamed section in the file");
            } else {
                if (sb.length() != 0) {
                    sb.append("\n");
                }
                sb.append(line);
            }
            if (it.hasNext() == false) {
                sections.add(new FileSection(sb.toString(), sectionName));
            }
        }
    } finally {
        IOUtils.closeQuietly(reader);
    }
    return sections;
}

---

From source file:com.metamx.druid.indexing.common.index.StaticS3FirehoseFactory.java

@Override
public Firehose connect() throws IOException {
    Preconditions.checkNotNull(s3Client, "null s3Client");

    return new Firehose() {
        LineIterator lineIterator = null;
        final Queue<URI> objectQueue = Lists.newLinkedList(uris);

        // Rolls over our streams and iterators to the next file, if appropriate
        private void maybeNextFile() throws Exception {

            if (lineIterator == null || !lineIterator.hasNext()) {

                // Close old streams, maybe.
                if (lineIterator != null) {
                    lineIterator.close();
                }//from  www . j  av a2s.c om

                // Open new streams, maybe.
                final URI nextURI = objectQueue.poll();
                if (nextURI != null) {

                    final String s3Bucket = nextURI.getAuthority();
                    final S3Object s3Object = new S3Object(
                            nextURI.getPath().startsWith("/") ? nextURI.getPath().substring(1)
                                    : nextURI.getPath());

                    log.info("Reading from bucket[%s] object[%s] (%s)", s3Bucket, s3Object.getKey(), nextURI);

                    int ntry = 0;
                    try {
                        final InputStream innerInputStream = s3Client.getObject(s3Bucket, s3Object.getKey())
                                .getDataInputStream();

                        final InputStream outerInputStream = s3Object.getKey().endsWith(".gz")
                                ? new GZIPInputStream(innerInputStream)
                                : innerInputStream;

                        lineIterator = IOUtils.lineIterator(
                                new BufferedReader(new InputStreamReader(outerInputStream, Charsets.UTF_8)));
                    } catch (IOException e) {
                        log.error(e,
                                "Exception reading from bucket[%s] object[%s] (try %d) (sleeping %d millis)",
                                s3Bucket, s3Object.getKey(), ntry, retryMillis);

                        ntry++;
                        if (ntry <= retryCount) {
                            Thread.sleep(retryMillis);
                        }
                    }

                }
            }

        }

        @Override
        public boolean hasMore() {
            try {
                maybeNextFile();
            } catch (Exception e) {
                throw Throwables.propagate(e);
            }

            return lineIterator != null && lineIterator.hasNext();
        }

        @Override
        public InputRow nextRow() {
            try {
                maybeNextFile();
            } catch (Exception e) {
                throw Throwables.propagate(e);
            }

            if (lineIterator == null) {
                throw new NoSuchElementException();
            }

            return parser.parse(lineIterator.next());
        }

        @Override
        public Runnable commit() {
            // Do nothing.
            return new Runnable() {
                public void run() {
                }
            };
        }

        @Override
        public void close() throws IOException {
            objectQueue.clear();
            if (lineIterator != null) {
                lineIterator.close();
            }
        }
    };
}

---

From source file:com.norconex.importer.handler.tagger.impl.TextStatisticsTagger.java

@Override
protected void tagTextDocument(String reference, Reader input, ImporterMetadata metadata, boolean parsed)
        throws ImporterHandlerException {
    long charCount = 0;
    long wordCharCount = 0;
    long wordCount = 0;
    long sentenceCount = 0;
    long sentenceCharCount = 0;
    long paragraphCount = 0;

    //TODO make this more efficient, by doing all this in one pass.
    LineIterator it = IOUtils.lineIterator(input);
    while (it.hasNext()) {
        String line = it.nextLine().trim();
        if (StringUtils.isBlank(line)) {
            continue;
        }// w  w  w  .  jav  a  2 s.  c om

        // Paragraph
        paragraphCount++;

        // Character
        charCount += line.length();

        // Word
        Matcher matcher = PATTERN_WORD.matcher(line);
        while (matcher.find()) {
            int wordLength = matcher.end() - matcher.start();
            wordCount++;
            wordCharCount += wordLength;
        }

        // Sentence
        BreakIterator boundary = BreakIterator.getSentenceInstance();
        boundary.setText(line);
        int start = boundary.first();
        for (int end = boundary.next(); end != BreakIterator.DONE; start = end, end = boundary.next()) {
            sentenceCharCount += (end - start);
            sentenceCount++;
        }
    }

    String field = StringUtils.EMPTY;
    if (StringUtils.isNotBlank(fieldName)) {
        field = fieldName.trim() + ".";
    }

    //--- Add fields ---
    metadata.addLong("document.stat." + field + "characterCount", charCount);
    metadata.addLong("document.stat." + field + "wordCount", wordCount);
    metadata.addLong("document.stat." + field + "sentenceCount", sentenceCount);
    metadata.addLong("document.stat." + field + "paragraphCount", paragraphCount);
    metadata.addString("document.stat." + field + "averageWordCharacterCount",
            divide(wordCharCount, wordCount));
    metadata.addString("document.stat." + field + "averageSentenceCharacterCount",
            divide(sentenceCharCount, sentenceCount));
    metadata.addString("document.stat." + field + "averageSentenceWordCount", divide(wordCount, sentenceCount));
    metadata.addString("document.stat." + field + "averageParagraphCharacterCount",
            divide(charCount, paragraphCount));
    metadata.addString("document.stat." + field + "averageParagraphSentenceCount",
            divide(sentenceCount, paragraphCount));
    metadata.addString("document.stat." + field + "averageParagraphWordCount",
            divide(wordCount, paragraphCount));

}

---

From source file:com.thalesgroup.hudson.plugins.klocwork.KloSource.java

/**
 * Splits the source code into three blocks: the line to highlight and the
 * source code before and after this line.
 *
 * @param sourceFile the source code of the whole file as rendered HTML string
 *//*w w w  .  j a v a 2s  .  c o m*/
public final void splitSourceFile(final String sourceFile) {
    StringBuilder output = new StringBuilder(sourceFile.length());

    KloFile kloFile = kloWorkspaceFile.getKloFile();
    LineIterator lineIterator = IOUtils.lineIterator(new StringReader(sourceFile));
    int lineNumber = 1;

    //---header
    while (lineNumber < SOURCE_GENERATOR_OFFSET) {
        copyLine(output, lineIterator);
        lineNumber++;
    }
    lineNumber = 1;

    //---iterate before the error line
    while (lineNumber < Integer.parseInt(((String) kloFile.get("line")))) {
        copyLine(output, lineIterator);
        lineNumber++;
    }
    output.append("</code>\n");

    //---Error message
    output.append("</td></tr>\n");
    output.append("<tr><td bgcolor=\"");
    appendRangeColor(output);
    output.append("\">\n");

    output.append("<div tooltip=\"");
    //AM
    //outputEscaped(output, kloFile.getProblemId()+":"+kloFile.getMessage());
    outputEscaped(output, kloFile.get("problemID") + ":" + kloFile.get("message"));
    output.append("\" nodismiss=\"\">\n");
    output.append("<code><b>\n");

    //The current line error
    copyLine(output, lineIterator);
    lineNumber++;

    //End of the code
    output.append("</b></code>\n");
    output.append("</div>\n");
    output.append("</td></tr>\n");

    output.append("<tr><td>\n");
    output.append("<code>\n");
    while (lineIterator.hasNext()) {
        copyLine(output, lineIterator);
    }
    output.append("</code>\n");
    output.append("</td></tr>\n");

    sourceCode = output.toString();
}

---

From source file:com.excilys.ebi.bank.jdbc.SimpleBatchResourceDatabasePopulator.java

/**
 * Execute the given SQL script.//w w  w . ja v a2s . com
 * <p>
 * The script will normally be loaded by classpath. There should be one
 * statement per line. Any {@link #setSeparator(String) statement
 * separators} will be removed.
 * <p>
 * <b>Do not use this method to execute DDL if you expect rollback.</b>
 *
 * @param connection
 *            the JDBC Connection with which to perform JDBC operations
 * @param resource
 *            the resource (potentially associated with a specific encoding)
 *            to load the SQL script from
 * @param continueOnError
 *            whether or not to continue without throwing an exception in
 *            the event of an error
 * @param ignoreFailedDrops
 *            whether of not to continue in the event of specifically an
 *            error on a <code>DROP</code>
 */
private void executeSqlScript(Connection connection, EncodedResource resource, boolean continueOnError,
        boolean ignoreFailedDrops) throws SQLException, IOException {

    if (LOGGER.isInfoEnabled()) {
        LOGGER.info("Executing SQL script from " + resource);
    }

    long startTime = System.currentTimeMillis();
    Iterator<String> statements = IOUtils.lineIterator(resource.getReader());
    int lineNumber = 0;

    boolean initialAutoCommitState = connection.getAutoCommit();

    connection.setAutoCommit(false);
    Statement stmt = connection.createStatement();
    try {
        while (statements.hasNext()) {
            String statement = statements.next();
            lineNumber++;
            try {
                stmt.addBatch(statement);

                if (lineNumber % batchSize == 0) {
                    stmt.executeBatch();
                    connection.commit();
                }
            } catch (SQLException ex) {
                boolean dropStatement = StringUtils.startsWithIgnoreCase(statement.trim(), "drop");
                if (continueOnError || (dropStatement && ignoreFailedDrops)) {
                    if (LOGGER.isDebugEnabled()) {
                        LOGGER.debug("Failed to execute SQL script statement at line " + lineNumber
                                + " of resource " + resource + ": " + statement, ex);
                    }
                } else {
                    Exception nextException = ex.getNextException();
                    throw new ScriptStatementFailedException(statement, lineNumber, resource,
                            nextException != null ? nextException : ex);
                }
            }
        }
    } finally {
        stmt.executeBatch();
        connection.commit();

        connection.setAutoCommit(initialAutoCommitState);

        try {
            stmt.close();
        } catch (Throwable ex) {
            LOGGER.debug("Could not close JDBC Statement", ex);
        }
    }
    long elapsedTime = System.currentTimeMillis() - startTime;
    if (LOGGER.isInfoEnabled()) {
        LOGGER.info("Done executing SQL script from " + resource + " in " + elapsedTime + " ms.");
    }
}

---

From source file:com.thalesgroup.hudson.plugins.cppcheck.CppcheckSource.java

/**
 * Splits the source code into three blocks: the line to highlight and the
 * source code before and after this line.
 *
 * @param sourceFile the source code of the whole file as rendered HTML string
 *///  w w w.j  av  a 2  s  . c  om
private void splitSourceFile(final String sourceFile) {
    StringBuilder output = new StringBuilder(sourceFile.length());

    CppcheckFile cppcheckFile = cppcheckWorkspaceFile.getCppcheckFile();
    LineIterator lineIterator = IOUtils.lineIterator(new StringReader(sourceFile));
    int lineNumber = 1;

    //---header
    while (lineNumber < SOURCE_GENERATOR_OFFSET) {
        copyLine(output, lineIterator);
        lineNumber++;
    }
    lineNumber = 1;

    //---iterate before the error line
    while (lineNumber < cppcheckFile.getLineNumber()) {
        copyLine(output, lineIterator);
        lineNumber++;
    }
    output.append("</code>\n");

    //---Error message
    output.append("</td></tr>\n");
    output.append("<tr><td bgcolor=\"");
    appendRangeColor(output);
    output.append("\">\n");

    output.append("<div tooltip=\"");

    outputEscaped(output, "<h3>");
    outputEscaped(output, cppcheckFile.getCppCheckId());
    output.append(": ");
    outputEscaped(output, cppcheckFile.getMessage());
    outputEscaped(output, "</h3>");

    if (cppcheckFile.getVerbose() != null) {
        outputEscaped(output, "<p style=\"white-space: pre-wrap;\">");
        outputEscaped(output, cppcheckFile.getVerbose());
        outputEscaped(output, "</p>");
    }

    output.append("\" nodismiss=\"\">\n");
    output.append("<code><b>\n");

    //The current line error
    copyLine(output, lineIterator);
    lineNumber++;

    //End of the code
    output.append("</b></code>\n");
    output.append("</div>\n");
    output.append("</td></tr>\n");

    output.append("<tr><td>\n");
    output.append("<code>\n");
    while (lineIterator.hasNext()) {
        copyLine(output, lineIterator);
    }
    output.append("</code>\n");
    output.append("</td></tr>\n");

    sourceCode = output.toString();
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• »