Example usage for org.apache.commons.io IOUtils readLines

Introduction

In this page you can find the example usage for org.apache.commons.io IOUtils readLines.

Prototype

public static List readLines(InputStream input, String encoding) throws IOException 

Source Link

Document

Get the contents of an InputStream as a list of Strings, one entry per line, using the specified character encoding.

Usage

From source file:org.apache.asterix.external.library.KeywordsDetecterFunction.java

@Override
public void initialize(IFunctionHelper functionHelper) throws Exception {
    InputStream in = this.getClass().getClassLoader().getResourceAsStream("KeywordsDetector.txt");
    keywordsList = IOUtils.readLines(in, StandardCharsets.UTF_8);

    result = (JObjects.JString) functionHelper.getResultObject();
}

---

From source file:org.apache.asterix.external.provider.ParserFactoryProvider.java

protected static Map<String, Class> initFactories() throws AsterixException {
    Map<String, Class> factories = new HashMap<>();
    ClassLoader cl = ParserFactoryProvider.class.getClassLoader();
    try {//from w  ww. j  av a  2s  . c om
        Enumeration<URL> urls = cl.getResources(RESOURCE);
        for (URL url : Collections.list(urls)) {
            List<String> classNames;
            try (InputStream is = url.openStream()) {
                classNames = IOUtils.readLines(is, StandardCharsets.UTF_8);
            }
            for (String className : classNames) {
                if (className.startsWith("#")) {
                    continue;
                }
                final Class<?> clazz = Class.forName(className);
                String[] formats = ((IDataParserFactory) clazz.newInstance()).getFormats();
                for (String format : formats) {
                    if (factories.containsKey(format)) {
                        throw new AsterixException("Duplicate format " + format);
                    }
                    factories.put(format, clazz);
                }
            }
        }
    } catch (IOException | ClassNotFoundException | InstantiationException | IllegalAccessException e) {
        throw new AsterixException(e);
    }
    return factories;
}

---

From source file:org.apache.atlas.authorize.simple.FileReaderUtil.java

public static List<String> readFile(InputStream policyStoreStream) throws IOException {
    if (isDebugEnabled) {
        LOG.debug("==> FileReaderUtil readFile()");
    }//from  w  w  w . j  a  v  a 2s .  com
    List<String> list = new ArrayList<>();
    List<String> fileLines = IOUtils.readLines(policyStoreStream, StandardCharsets.UTF_8);
    if (fileLines != null) {
        for (String line : fileLines) {
            if ((!line.startsWith("#")) && Pattern.matches(".+;;.*;;.*;;.+", line))
                list.add(line);
        }
    }

    if (isDebugEnabled) {
        LOG.debug("<== FileReaderUtil readFile()");
        LOG.debug("Policies read :: " + list);
    }

    return list;
}

---

From source file:org.apache.james.transport.mailets.PatternExtractor.java

private List<ReplacingPattern> getPatternsFromStream(InputStream stream, Charset charset)
        throws MailetException, IOException {
    ImmutableList.Builder<ReplacingPattern> patternList = ImmutableList.builder();
    for (String line : IOUtils.readLines(stream, charset)) {
        line = line.trim();//  w ww . j av  a2  s . c om
        if (!isComment(line)) {
            assertPatternSurroundedWithSlashes(line);
            patternList.add(stripSurroundingSlashes(line.substring(1, line.length() - 1)));
        }
    }
    return patternList.build();
}

---

From source file:org.apache.kylin.invertedindex.invertedindex.InvertedIndexLocalTest.java

private List<TableRecord> loadRecordsSorted() throws IOException {
    File file = new File(LOCALMETA_TEST_DATA, "data/flatten_data_for_ii.csv");
    FileInputStream in = new FileInputStream(file);
    List<String> lines = IOUtils.readLines(in, "UTF-8");
    in.close();/*w  ww .  ja v  a  2 s .c  om*/

    List<TableRecord> records = Lists.newArrayList();
    for (String line : lines) {
        String[] fields = line.split(",");
        TableRecord rec = info.createTableRecord();
        for (int col = 0; col < fields.length; col++) {
            rec.setValueString(col, fields[col]);
        }
        records.add(rec);
    }

    Collections.sort(records, new Comparator<TableRecord>() {
        @Override
        public int compare(TableRecord a, TableRecord b) {
            long x = a.getTimestamp() - b.getTimestamp();
            if (x > 0)
                return 1;
            else if (x == 0)
                return 0;
            else
                return -1;
        }
    });

    return records;
}

---

From source file:org.apache.kylin.invertedindex.InvertedIndexLocalTest.java

@Before
public void setUp() throws Exception {
    this.createTestMetadata();
    this.ii = IIManager.getInstance(getTestConfig()).getII("test_kylin_ii_left_join");

    File file = new File(LOCALMETA_TEST_DATA, "data/flatten_data_for_ii.csv");
    FileInputStream in = new FileInputStream(file);
    this.lines = IOUtils.readLines(in, "UTF-8");
    in.close();/*w  ww  . ja  v  a  2  s.co  m*/

    dictionaryMap = buildDictionary(Lists.transform(lines, new Function<String, List<String>>() {
        @Nullable
        @Override
        public List<String> apply(@Nullable String input) {
            return Lists.newArrayList(input.split(","));
        }
    }), ii.getDescriptor());
    this.info = new TableRecordInfo(ii.getDescriptor(), dictionaryMap);
}

---

From source file:org.apache.kylin.rest.util.ControllerSplitter.java

private static void chopOff(File f, String annoPtn) throws IOException {

    System.out.println("Processing " + f);

    FileInputStream is = new FileInputStream(f);
    List<String> lines = IOUtils.readLines(is, "UTF-8");
    is.close();//www .  j  a  va  2s. c  o m
    List<String> outLines = new ArrayList<>(lines.size());

    boolean del = false;
    for (String l : lines) {
        if (l.startsWith("    @") && l.contains(annoPtn))
            del = true;

        if (del)
            System.out.println("x " + l);
        else
            outLines.add(l);

        if (del && l.startsWith("    }"))
            del = false;
    }

    if (!dryRun && outLines.size() < lines.size()) {
        FileOutputStream os = new FileOutputStream(f);
        IOUtils.writeLines(outLines, "\n", os, "UTF-8");
        os.close();
        System.out.println("UPDATED " + f);
    } else {
        System.out.println("skipped");
    }

    System.out.println("============================================================================");
}

---

From source file:org.apache.lens.server.util.ScannedPaths.java

/**
 * Method that computes path of resources matching the input path or path regex pattern.
 * If provided path is a directory it additionally checks for the jar_order or glob_order file
 * that imposes ordering of resources and filters out other resources.
 *
 * Updates finalPaths List with matched paths and returns an iterator for matched paths.
 *///w w w.  j  a  v a2  s .com
private List<String> getMatchedPaths(Path pt, String type) {
    List<String> finalPaths = new ArrayList<>();
    InputStream resourceOrderIStream = null;
    FileSystem fs;

    try {
        fs = pt.getFileSystem(new Configuration());
        if (fs.exists(pt)) {
            if (fs.isFile(pt)) {
                /**
                 * CASE 1 : Direct FILE provided in path
                 **/
                finalPaths.add(pt.toUri().toString());
            } else if (fs.isDirectory(pt)) {
                /**
                 * CASE 2 : DIR provided in path
                 **/
                Path resourceOrderFile;
                FileStatus[] statuses;
                List<String> newMatches;
                List<String> resources;

                resourceOrderFile = new Path(pt, "jar_order");
                /** Add everything in dir if no jar_order or glob_order is present **/
                if (!fs.exists(resourceOrderFile)) {
                    resourceOrderFile = new Path(pt, "glob_order");
                    if (!fs.exists(resourceOrderFile)) {
                        resourceOrderFile = null;
                        /** Get matched resources recursively for all files **/
                        statuses = fs.globStatus(new Path(pt, "*"));
                        if (statuses != null) {
                            for (FileStatus st : statuses) {
                                newMatches = getMatchedPaths(st.getPath(), type);
                                finalPaths.addAll(newMatches);
                            }
                        }
                    }
                }
                if (resourceOrderFile != null) {
                    /** Else get jars as per order specified in jar_order/glob_order **/
                    resourceOrderIStream = fs.open(resourceOrderFile);
                    resources = IOUtils.readLines(resourceOrderIStream, Charset.forName("UTF-8"));
                    for (String resource : resources) {
                        if (StringUtils.isBlank(resource)) {
                            continue;
                        }
                        resource = resource.trim();

                        /** Get matched resources recursively for provided path/pattern **/
                        if (resource.startsWith("/") || resource.contains(":/")) {
                            newMatches = getMatchedPaths(new Path(resource), type);
                        } else {
                            newMatches = getMatchedPaths(new Path(pt, resource), type);
                        }
                        finalPaths.addAll(newMatches);
                    }
                }
            }
        } else {
            /**
             * CASE 3 : REGEX provided in path
             * */
            FileStatus[] statuses = fs.globStatus(Path.getPathWithoutSchemeAndAuthority(pt));
            if (statuses != null) {
                for (FileStatus st : statuses) {
                    List<String> newMatches = getMatchedPaths(st.getPath(), type);
                    finalPaths.addAll(newMatches);
                }
            }
        }
        filterDirsAndJarType(fs, finalPaths);
    } catch (FileNotFoundException fex) {
        log.error("File not found while scanning path. Path: {}, Type: {}", path, type, fex);
    } catch (Exception e) {
        log.error("Exception while initializing PathScanner. Path: {}, Type: {}", path, type, e);
    } finally {
        IOUtils.closeQuietly(resourceOrderIStream);
    }

    return finalPaths;
}

---

From source file:org.apache.metron.common.utils.HDFSUtils.java

/**
 * Reads full HDFS FS file contents into a String. Opens and closes the file system on each call.
 * Never null./* w  ww  .  j  a v  a  2  s . c  om*/
 *
 * @param config Hadoop configuration
 * @param path path to file
 * @return file contents as a String
 * @throws IOException
 */
public static List<String> readFile(Configuration config, String path) throws IOException {
    FileSystem fs = FileSystem.newInstance(config);
    Path hdfsPath = new Path(path);
    FSDataInputStream inputStream = fs.open(hdfsPath);
    return IOUtils.readLines(inputStream, "UTF-8");
}

---

From source file:org.apache.streams.cassandra.test.CassandraPersistIT.java

@Test
public void testCassandraPersist() throws Exception {
    CassandraPersistWriter writer = new CassandraPersistWriter(testConfiguration);

    writer.prepare(null);/*from w w w .ja va2  s  . c om*/

    InputStream testActivityFolderStream = CassandraPersistIT.class.getClassLoader()
            .getResourceAsStream("activities");
    List<String> files = IOUtils.readLines(testActivityFolderStream, StandardCharsets.UTF_8);

    for (String file : files) {
        LOGGER.info("File: " + file);
        InputStream testActivityFileStream = CassandraPersistIT.class.getClassLoader()
                .getResourceAsStream("activities/" + file);
        Activity activity = MAPPER.readValue(testActivityFileStream, Activity.class);
        activity.getAdditionalProperties().remove("$license");
        StreamsDatum datum = new StreamsDatum(activity, activity.getVerb());
        writer.write(datum);

        LOGGER.info("Wrote: " + activity.getVerb());
        count++;
    }

    LOGGER.info("Total Written: {}", count);
    Assert.assertEquals(89, count);

    writer.cleanUp();

    CassandraPersistReader reader = new CassandraPersistReader(testConfiguration);

    reader.prepare(null);

    StreamsResultSet resultSet = reader.readAll();

    LOGGER.info("Total Read: {}", resultSet.size());
    Assert.assertEquals(resultSet.size(), 89);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• »