List of usage examples for org.apache.commons.io IOUtils lineIterator
public static LineIterator lineIterator(InputStream input, String encoding) throws IOException
InputStream, using the character encoding specified (or default encoding if null). From source file:com.cheusov.Jrep.java
private static String[] handleOptions(String[] args) throws ParseException, IOException { if (args.length == 0) { printHelp(options);/* ww w . j a v a2 s.c om*/ System.exit(1); } CommandLineParser parser = new PosixParser(); CommandLine cmd = parser.parse(options, args); inverseMatch = cmd.hasOption("v"); outputFilename = cmd.hasOption("l"); opt_O = cmd.getOptionValue("O"); opt_o = cmd.hasOption("o") || (opt_O != null); wholeContent = cmd.hasOption("8"); opt_L = cmd.hasOption("L"); opt_i = cmd.hasOption("i"); opt_h = cmd.hasOption("h"); opt_H = cmd.hasOption("H"); opt_F = cmd.hasOption("F"); opt_c = cmd.hasOption("c"); opt_n = cmd.hasOption("n"); opt_x = cmd.hasOption("x"); boolean optBool = cmd.hasOption("R"); opt_directories = (optBool || cmd.hasOption("r") ? Directories.RECURSE : Directories.READ); if (!optBool) orExcludeFileFilter.addFileFilter(new SymLinkFileFilter()); opt_s = cmd.hasOption("s"); opt_q = cmd.hasOption("q") || cmd.hasOption("silent"); opt_w = cmd.hasOption("w"); opt_line_buffered = cmd.hasOption("line-buffered"); String[] optArray = cmd.getOptionValues("e"); if (optArray != null && optArray.length != 0) { for (String regexp : optArray) regexps.add(regexp); } optArray = cmd.getOptionValues("include"); if (optArray != null && optArray.length != 0) { for (String globPattern : optArray) orIncludeFileFilter.addFileFilter(new WildcardFileFilter(globPattern)); } else { orIncludeFileFilter.addFileFilter(TrueFileFilter.TRUE); } optArray = cmd.getOptionValues("exclude"); if (optArray != null && optArray.length != 0) { for (String globPattern : optArray) { if (globPattern.startsWith(".") || globPattern.startsWith("/")) orExcludeFileFilter.addFileFilter(new PathFileFilter(globPattern)); else orExcludeFileFilter.addFileFilter(new WildcardFileFilter(globPattern)); } } String optStr = cmd.getOptionValue("exclude-from"); if (optStr != null) { Iterator<String> it = IOUtils.lineIterator(new FileInputStream(optStr), encoding); while (it.hasNext()) { String globPattern = it.next(); if (globPattern.startsWith(".") || globPattern.startsWith("/")) orExcludeFileFilter.addFileFilter(new PathFileFilter(globPattern)); else orExcludeFileFilter.addFileFilter(new WildcardFileFilter(globPattern)); } } optStr = cmd.getOptionValue("f"); if (optStr != null) { Iterator<String> it = IOUtils.lineIterator(new FileInputStream(optStr), encoding); while (it.hasNext()) { regexps.add(it.next()); } } optStr = cmd.getOptionValue("m"); if (optStr != null) opt_m = Integer.valueOf(optStr); optStr = cmd.getOptionValue("label"); if (optStr != null) label = optStr; optStr = cmd.getOptionValue("marker-start"); if (optStr != null) colorEscStart = optStr; optStr = cmd.getOptionValue("marker-end"); if (optStr != null) colorEscEnd = optStr; optStr = cmd.getOptionValue("A"); if (optStr != null) opt_A = Integer.valueOf(optStr); optStr = cmd.getOptionValue("B"); if (optStr != null) opt_B = Integer.valueOf(optStr); optStr = cmd.getOptionValue("C"); if (optStr != null) opt_A = opt_B = Integer.valueOf(optStr); optStr = cmd.getOptionValue("color"); if (optStr == null) optStr = cmd.getOptionValue("colour"); if (optStr == null || optStr.equals("auto")) { if (!isStdoutTTY) colorEscStart = null; } else if (optStr.equals("always")) { } else if (optStr.equals("never")) { colorEscStart = null; } else { throw new IllegalArgumentException("Illegal argument `" + optStr + "` for option --color"); } optStr = cmd.getOptionValue("re-engine"); if (optStr == null) { } else if (optStr.equals("java")) { opt_re_engine = JrepPattern.RE_ENGINE_TYPE.JAVA; } else if (optStr.equals("re2j")) { opt_re_engine = JrepPattern.RE_ENGINE_TYPE.RE2J; } else { throw new IllegalArgumentException("Illegal argument `" + optStr + "` for option --re-engine"); } optStr = cmd.getOptionValue("directories"); if (optStr == null) { } else if (optStr.equals("skip")) { opt_directories = Directories.SKIP; } else if (optStr.equals("read")) { opt_directories = Directories.READ; } else if (optStr.equals("recurse")) { opt_directories = Directories.RECURSE; } else { throw new IllegalArgumentException("Illegal argument `" + optStr + "` for option --directories"); } if (cmd.hasOption("2")) opt_re_engine = JrepPattern.RE_ENGINE_TYPE.RE2J; if (cmd.hasOption("help")) { printHelp(options); System.exit(0); } if (cmd.hasOption("V")) { System.out.println("jrep-" + System.getenv("JREP_VERSION")); System.exit(0); } return cmd.getArgs(); }
From source file:com.datamelt.nifi.processors.ExecuteRuleEngine.java
@Override public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException { // map used to store the attribute name and its value from the content of the flow file final Map<String, String> propertyMap = new HashMap<>(); // get a logger instance final ComponentLog logger = getLogger(); // a header from the content if present final AtomicReference<HeaderRow> header = new AtomicReference<>(); AtomicBoolean error = new AtomicBoolean(); // get the flow file FlowFile flowFile = session.get();/*from w w w .ja va 2 s . c o m*/ if (flowFile == null) { return; } // list of rows from splitting the original flow file content ArrayList<RuleEngineRow> flowFileRows = new ArrayList<RuleEngineRow>(); // list of rows containing the detailed results of the ruleengine ArrayList<RuleEngineRow> flowFileDetails = new ArrayList<RuleEngineRow>(); boolean headerPresent = context.getProperty(ATTRIBUTE_HEADER_PRESENT).getValue().equals("true"); // put the name of the ruleengine zip file in the list of properties propertyMap.put(PROPERTY_RULEENGINE_ZIPFILE_NAME, context.getProperty(ATTRIBUTE_RULEENGINE_ZIPFILE).getValue()); final int batchSize = Integer.parseInt(context.getProperty(BATCH_SIZE_NAME).getValue()); // read flow file into input stream session.read(flowFile, new InputStreamCallback() { public void process(InputStream in) throws IOException { try { // iterator over the lines from the input stream LineIterator iterator = IOUtils.lineIterator(in, "utf-8"); // check if configuration indicates that a header row is present in the flow file content if (headerPresent) { logger.debug("configuration indicates a header row is present in flow file content"); // if there is at least one row of data and the header is not defined yet if (iterator.hasNext() && header.get() == null) { // set the header from the content header.set(new HeaderRow(iterator.nextLine(), separator)); } } // if no header row is present in the flow file content else { logger.debug("configuration indicates no header row is present in flow file content"); // use the header from the field names header.set(headerFromFieldNames); } // loop over all rows of data while (iterator.hasNext()) { // we handle the error per row of data error.set(false); // get a row to process String row = iterator.nextLine(); // check that we have data if (row != null && !row.trim().equals("")) { RowFieldCollection rowFieldCollection = null; try { rowFieldCollection = getRowFieldCollection(row, header.get()); logger.debug("RowFieldCollection header contains: " + rowFieldCollection.getHeader().getNumberOfFields() + " fields"); logger.debug("RowFieldCollection contains: " + rowFieldCollection.getNumberOfFields() + " fields"); // run the ruleengine with the given data from the flow file logger.debug("running business ruleengine..."); // run the business logic/rules against the data ruleEngine.run("flowfile", rowFieldCollection); // add some debugging output that might be useful logger.debug("number of rulegroups: " + ruleEngine.getNumberOfGroups()); logger.debug( "number of rulegroups passed: " + ruleEngine.getNumberOfGroupsPassed()); logger.debug( "number of rulegroups failed: " + ruleEngine.getNumberOfGroupsFailed()); logger.debug( "number of rulegroups skipped: " + ruleEngine.getNumberOfGroupsSkipped()); logger.debug("number of rules: " + ruleEngine.getNumberOfRules()); logger.debug("number of rules passed: " + ruleEngine.getNumberOfRulesPassed()); logger.debug("number of rules failed: " + ruleEngine.getNumberOfRulesFailed()); logger.debug("number of actions: " + ruleEngine.getNumberOfActions()); // add some properties of the ruleengine execution to the map addRuleEngineProperties(propertyMap); } catch (Exception ex) { error.set(true); logger.error(ex.getMessage(), ex); } // if no error occurred we create a save the data for the creation of the flow files if (!error.get()) { // process only if the collection of fields was changed by // a ruleengine action. this means the data was updated so // we will have to re-write/re-create the flow file content. if (rowFieldCollection.isCollectionUpdated()) { // put an indicator that the data was modified by the ruleengine propertyMap.put(PROPERTY_RULEENGINE_CONTENT_MODIFIED, "true"); logger.debug( "data was modified - updating flow file content with ruleengine results"); // the RuleEngineRow instance will contain the row of data and the map of properties // and will later be used when the flow files are created flowFileRows .add(new RuleEngineRow(getResultRow(rowFieldCollection), propertyMap)); } else { // put an indicator that the data was NOT modified by the ruleengine propertyMap.put(PROPERTY_RULEENGINE_CONTENT_MODIFIED, "false"); logger.debug("data was not modified - using original content"); // the RuleEngineRow instance will contain the row of data and the map of properties // and will later be used when the flow files are created flowFileRows.add(new RuleEngineRow(row, propertyMap)); } if (flowFileRows.size() >= batchSize) { // generate flow files from the individual rows List<FlowFile> splitFlowFiles = generateFlowFileSplits(context, session, flowFileRows, header.get(), headerPresent); // transfer all individual rows to success relationship if (splitFlowFiles.size() > 0) { session.transfer(splitFlowFiles, SUCCESS); } } // if the user configured detailed results if (context.getProperty(ATTRIBUTE_OUTPUT_DETAILED_RESULTS).getValue() .equals("true")) { // get the configured output type String outputType = context.getProperty(ATTRIBUTE_OUTPUT_DETAILED_RESULTS_TYPE) .getValue(); logger.debug("configuration set to output detailed results with type [" + outputType + "]"); // we need to create a flow file only, if the ruleengine results are according to the output type settings if (outputType.equals(OUTPUT_TYPE_ALL_GROUPS_ALL_RULES) || (outputType.equals(OUTPUT_TYPE_FAILED_GROUPS_ALL_RULES) && ruleEngine.getNumberOfGroupsFailed() > 0) || (outputType.equals(OUTPUT_TYPE_FAILED_GROUPS_FAILED_RULES) && ruleEngine.getNumberOfGroupsFailed() > 0) || (outputType.equals(OUTPUT_TYPE_FAILED_GROUPS_PASSED_RULES) && ruleEngine.getNumberOfGroupsFailed() > 0) || (outputType.equals(OUTPUT_TYPE_PASSED_GROUPS_ALL_RULES) && ruleEngine.getNumberOfGroupsPassed() > 0) || (outputType.equals(OUTPUT_TYPE_PASSED_GROUPS_FAILED_RULES) && ruleEngine.getNumberOfGroupsPassed() > 0 || (outputType.equals(OUTPUT_TYPE_PASSED_GROUPS_PASSED_RULES) && ruleEngine.getNumberOfGroupsPassed() > 0))) { // create the content for the flow file String content = getFlowFileRuleEngineDetailsContent(header.get(), headerPresent, outputType, row); // add results to the list flowFileDetails.add(new RuleEngineRow(content, propertyMap)); if (flowFileDetails.size() >= batchSize) { List<FlowFile> detailsFlowFiles = generateFlowFilesRuleEngineDetails( context, session, flowFileDetails, header.get(), headerPresent); // transfer all individual rows to detailed relationship if (detailsFlowFiles.size() > 0) { session.transfer(detailsFlowFiles, DETAILED_RESULTS); } } } } // clear the collections of ruleengine results ruleEngine.getRuleExecutionCollection().clear(); } // if we have an error we create a flow file from the current row of data and send it to the failure relationsship else { FlowFile failureFlowFile = generateFailureFlowFile(context, session, row, header.get(), headerPresent); session.transfer(failureFlowFile, FAILURE); } } } LineIterator.closeQuietly(iterator); } catch (Exception ex) { ex.printStackTrace(); logger.error("error running the business ruleengine", ex); } } }); // generate flow files from the individual rows List<FlowFile> splitFlowFiles = generateFlowFileSplits(context, session, flowFileRows, header.get(), headerPresent); // generate flow files from the individual rows List<FlowFile> detailsFlowFiles = generateFlowFilesRuleEngineDetails(context, session, flowFileDetails, header.get(), headerPresent); // transfer the original flow file session.transfer(flowFile, ORIGINAL); // transfer all individual rows to success relationship if (splitFlowFiles.size() > 0) { session.transfer(splitFlowFiles, SUCCESS); } // transfer all individual rows to success relationship if (detailsFlowFiles.size() > 0) { session.transfer(detailsFlowFiles, DETAILED_RESULTS); } }
From source file:com.adobe.acs.tools.csv_asset_importer.impl.CsvAssetImporterServlet.java
/** * Adds a populated terminating field to the ends of CSV entries. * If the last entry in a CSV row is empty, the CSV library has difficulty understanding that is the end of the row. * * @param is the CSV file as an inputstream * @param separator The field separator/*from w w w . ja v a2s. com*/ * @param charset The charset * @return An inputstream that is the same as is, but each line has a populated line termination entry * @throws IOException */ private InputStream terminateLines(final InputStream is, final char separator, final String charset) throws IOException { final ByteArrayOutputStream baos = new ByteArrayOutputStream(); final PrintStream printStream = new PrintStream(baos); final LineIterator lineIterator = IOUtils.lineIterator(is, charset); while (lineIterator.hasNext()) { String line = StringUtils.stripToNull(lineIterator.next()); if (line != null) { line += separator + TERMINATED; printStream.println(line); } } return new ByteArrayInputStream(baos.toByteArray()); }
From source file:com.alexkli.osgi.troubleshoot.impl.TroubleshootServlet.java
private void includeResource(PrintWriter out, String path) { try {/*www .j a v a 2s .c om*/ if (!path.startsWith("/")) { path = "/" + path; } URL url = getClass().getResource(path); if (url == null) { // not found` return; } InputStream ins = url.openConnection().getInputStream(); LineIterator lineIterator = IOUtils.lineIterator(ins, "UTF-8"); boolean startComment = true; while (lineIterator.hasNext()) { String line = lineIterator.nextLine(); if (startComment) { String trimmed = line.trim(); if (!trimmed.isEmpty() && !trimmed.startsWith("/**") && !trimmed.startsWith("*")) { startComment = false; } } if (!startComment) { out.println(line); } } } catch (IOException e) { e.printStackTrace(); } }
From source file:com.example.util.FileUtils.java
/** * Returns an Iterator for the lines in a <code>File</code>. * <p>//from ww w . ja v a2 s . c om * This method opens an <code>InputStream</code> for the file. * When you have finished with the iterator you should close the stream * to free internal resources. This can be done by calling the * {@link LineIterator#close()} or * {@link LineIterator#closeQuietly(LineIterator)} method. * <p> * The recommended usage pattern is: * <pre> * LineIterator it = FileUtils.lineIterator(file, "UTF-8"); * try { * while (it.hasNext()) { * String line = it.nextLine(); * /// do something with line * } * } finally { * LineIterator.closeQuietly(iterator); * } * </pre> * <p> * If an exception occurs during the creation of the iterator, the * underlying stream is closed. * * @param file the file to open for input, must not be {@code null} * @param encoding the encoding to use, {@code null} means platform default * @return an Iterator of the lines in the file, never {@code null} * @throws IOException in case of an I/O error (file closed) * @since 1.2 */ public static LineIterator lineIterator(File file, String encoding) throws IOException { InputStream in = null; try { in = openInputStream(file); return IOUtils.lineIterator(in, encoding); } catch (IOException ex) { IOUtils.closeQuietly(in); throw ex; } catch (RuntimeException ex) { IOUtils.closeQuietly(in); throw ex; } }
From source file:org.apache.camel.component.exec.ExecutableJavaProgram.java
public static void main(String[] args) throws Exception { if (args == null || args.length == 0) { throw new IllegalArgumentException("Empty args are not allowed."); }// ww w . j a v a2 s . c o m if (args[0].equals(PRINT_IN_STDOUT)) { System.out.print(PRINT_IN_STDOUT); System.exit(0); } else if (args[0].equals(PRINT_ARGS_STDOUT)) { for (int i = 0; i < args.length; i++) { String arg = args[i]; System.out.println(i + arg); } System.exit(0); } else if (args[0].equals(PRINT_IN_STDERR)) { System.err.print(PRINT_IN_STDERR); System.exit(1); } else if (args[0].equals(String.valueOf(EXIT_WITH_VALUE_0))) { System.exit(0); } else if (args[0].equals(String.valueOf(EXIT_WITH_VALUE_1))) { System.exit(1); } else if (args[0].equals(THREADS)) { Thread stderrPrinterThread = new Thread(new ErrPrinter()); Thread stdoutPrinterThread = new Thread(new OutPrinter()); stderrPrinterThread.start(); stdoutPrinterThread.start(); stderrPrinterThread.join(); stdoutPrinterThread.join(); } else if (args[0].equals(SLEEP_WITH_TIMEOUT)) { doSleep(); System.exit(0); } else if (READ_INPUT_LINES_AND_PRINT_THEM.equals(args[0])) { LineIterator iterator = IOUtils.lineIterator(System.in, "UTF-8"); while (iterator.hasNext()) { String line = iterator.nextLine(); System.out.println(line); } } else { System.out.println(args[0]); } }
From source file:org.apache.druid.data.input.impl.AbstractTextFilesFirehoseFactory.java
@Override public Firehose connect(StringInputRowParser firehoseParser, File temporaryDirectory) throws IOException { initializeObjectsIfNeeded();/*from w w w. ja va 2s. c om*/ final Iterator<T> iterator = objects.iterator(); return new FileIteratingFirehose(new Iterator<LineIterator>() { @Override public boolean hasNext() { return iterator.hasNext(); } @Override public LineIterator next() { if (!hasNext()) { throw new NoSuchElementException(); } final T object = iterator.next(); try { return IOUtils.lineIterator(wrapObjectStream(object, openObjectStream(object)), StandardCharsets.UTF_8); } catch (Exception e) { LOG.error(e, "Exception reading object[%s]", object); throw Throwables.propagate(e); } } }, firehoseParser); }
From source file:org.apache.druid.query.aggregation.AggregationTestHelper.java
public void createIndex(InputStream inputDataStream, String parserJson, String aggregators, File outDir, long minTimestamp, Granularity gran, int maxRowCount, boolean rollup) throws Exception { try {/*from w ww . j a v a 2 s . c o m*/ StringInputRowParser parser = mapper.readValue(parserJson, StringInputRowParser.class); LineIterator iter = IOUtils.lineIterator(inputDataStream, "UTF-8"); List<AggregatorFactory> aggregatorSpecs = mapper.readValue(aggregators, new TypeReference<List<AggregatorFactory>>() { }); createIndex(iter, parser, aggregatorSpecs.toArray(new AggregatorFactory[0]), outDir, minTimestamp, gran, true, maxRowCount, rollup); } finally { Closeables.close(inputDataStream, true); } }
From source file:org.apache.flume.source.TestNetcatSource.java
/** * Test if an ack is sent for every event in the correct encoding * * @throws InterruptedException/*from www . j a va 2 s .c om*/ * @throws IOException */ @Test public void testAck() throws InterruptedException, IOException { String encoding = "UTF-8"; String ackEvent = "OK"; startSource(encoding, "true", "1", "512"); Socket netcatSocket = new Socket(localhost, selectedPort); LineIterator inputLineIterator = IOUtils.lineIterator(netcatSocket.getInputStream(), encoding); try { // Test on english text snippet for (int i = 0; i < 20; i++) { sendEvent(netcatSocket, english, encoding); Assert.assertArrayEquals("Channel contained our event", english.getBytes(defaultCharset), getFlumeEvent()); Assert.assertEquals("Socket contained the Ack", ackEvent, inputLineIterator.nextLine()); } // Test on french text snippet for (int i = 0; i < 20; i++) { sendEvent(netcatSocket, french, encoding); Assert.assertArrayEquals("Channel contained our event", french.getBytes(defaultCharset), getFlumeEvent()); Assert.assertEquals("Socket contained the Ack", ackEvent, inputLineIterator.nextLine()); } } finally { netcatSocket.close(); stopSource(); } }
From source file:org.apache.flume.source.TestNetcatSource.java
/** * Test that line above MaxLineLength are discarded * * @throws InterruptedException//from w w w. j a va 2 s . c om * @throws IOException */ @Test public void testMaxLineLengthwithAck() throws InterruptedException, IOException { String encoding = "UTF-8"; String ackEvent = "OK"; String ackErrorEvent = "FAILED: Event exceeds the maximum length (10 chars, including newline)"; startSource(encoding, "true", "1", "10"); Socket netcatSocket = new Socket(localhost, selectedPort); LineIterator inputLineIterator = IOUtils.lineIterator(netcatSocket.getInputStream(), encoding); try { sendEvent(netcatSocket, "123456789", encoding); Assert.assertArrayEquals("Channel contained our event", "123456789".getBytes(defaultCharset), getFlumeEvent()); Assert.assertEquals("Socket contained the Ack", ackEvent, inputLineIterator.nextLine()); sendEvent(netcatSocket, english, encoding); Assert.assertEquals("Channel does not contain an event", null, getRawFlumeEvent()); Assert.assertEquals("Socket contained the Error Ack", ackErrorEvent, inputLineIterator.nextLine()); } finally { netcatSocket.close(); stopSource(); } }