List of usage examples for org.apache.commons.io IOUtils lineIterator
public static LineIterator lineIterator(InputStream input, String encoding) throws IOException
InputStream, using the character encoding specified (or default encoding if null). From source file:org.onesec.raven.ivr.impl.ExternalTextToSpeechEngineNodeTest.java
private void printStream(InputStream stream) throws IOException { LineIterator it = IOUtils.lineIterator(stream, "utf-8"); while (it.hasNext()) System.out.println("LINE: " + it.nextLine()); }
From source file:org.openmrs.module.emrmonitor.metric.ConfigurableMetricProducer.java
/** * If multiple lines of output are returned and each is in the format of key=value, then the key will be considered part of the metric, and the value the value * Otherwise, the full contents of output will be the value of a single metric *///from www .ja v a 2 s. co m protected void handleShellScript(Map<String, String> metrics, String namespace, File f) throws IOException { Process process = Runtime.getRuntime().exec(f.getAbsolutePath()); StringBuilder singleValueMetric = new StringBuilder(); Map<String, String> keyValueMetrics = new LinkedHashMap<String, String>(); LineIterator successIterator = null; try { successIterator = IOUtils.lineIterator(process.getInputStream(), "UTF-8"); while (successIterator.hasNext()) { String line = successIterator.nextLine(); String[] split = StringUtils.splitByWholeSeparatorPreserveAllTokens(line, "=", 2); if (split.length == 2) { keyValueMetrics.put(namespace + "." + split[0], split[1]); } else { singleValueMetric.append(line).append(System.getProperty("line.separator")); } } if (singleValueMetric.length() > 0) { metrics.put(namespace, singleValueMetric.toString()); } else { metrics.putAll(keyValueMetrics); } } finally { successIterator.close(); } StringBuilder error = new StringBuilder(); LineIterator errorIterator = null; try { errorIterator = IOUtils.lineIterator(process.getErrorStream(), "UTF-8"); while (errorIterator.hasNext()) { String line = errorIterator.nextLine(); error.append(System.getProperty("line.separator")).append(line); } } finally { errorIterator.close(); } if (error.length() > 0) { throw new RuntimeException( "An error occurred while executing shell script " + f.getName() + ": " + error); } }
From source file:org.openmrs.module.pihmalawi.sql.MysqlRunner.java
/** * Executes a Sql Script/* w ww. j ava 2s . c om*/ */ public static MysqlResult executeSql(String sql, Map<String, Object> parameterValues) { log.info("Executing SQL..."); File toExecute = null; try { // Writing SQL to temporary file for execution toExecute = File.createTempFile("mysqlrunner", ".sql"); StringBuilder sqlToWrite = new StringBuilder(); if (parameterValues != null) { for (String paramName : parameterValues.keySet()) { Object paramValue = parameterValues.get(paramName); sqlToWrite.append("set @").append(paramName); sqlToWrite.append("=").append(getParameterAssignmentString(paramValue)).append(";"); sqlToWrite.append(System.getProperty("line.separator")); } } sqlToWrite.append(sql); FileUtils.writeStringToFile(toExecute, sqlToWrite.toString()); log.debug("Wrote SQL file for execution: " + toExecute.getAbsolutePath()); log.debug("Contents:\n" + sqlToWrite); // Constructing command line elements to execute List<String> commands = new ArrayList<String>(); commands.add("mysql"); commands.add("-u" + Context.getRuntimeProperties().getProperty("connection.username")); commands.add("-p" + Context.getRuntimeProperties().getProperty("connection.password")); commands.add("-esource " + toExecute.getAbsolutePath()); commands.add(DatabaseUpdater.getConnection().getCatalog()); // Database Name log.debug("Constructed command to execute: \n" + OpenmrsUtil.join(commands, " ")); Process process = Runtime.getRuntime().exec(commands.toArray(new String[] {})); MysqlResult result = new MysqlResult(); LineIterator successIterator = null; try { successIterator = IOUtils.lineIterator(process.getInputStream(), "UTF-8"); while (successIterator.hasNext()) { String line = successIterator.nextLine(); String[] elements = StringUtils.splitPreserveAllTokens(line, '\t'); if (result.getColumns().isEmpty()) { result.setColumns(Arrays.asList(elements)); } else { Map<String, String> row = new LinkedHashMap<String, String>(); for (int i = 0; i < result.getColumns().size(); i++) { String value = elements[i].trim(); if ("NULL".equals(value)) { value = null; } row.put(result.getColumns().get(i), value); } result.getData().add(row); } } } finally { successIterator.close(); } LineIterator errorIterator = null; try { errorIterator = IOUtils.lineIterator(process.getErrorStream(), "UTF-8"); while (errorIterator.hasNext()) { String line = errorIterator.nextLine(); if (!line.toLowerCase().startsWith("warning")) { result.getErrors().add(line); } } } finally { errorIterator.close(); } return result; } catch (Exception e) { throw new RuntimeException("An error occurred while executing a SQL file", e); } finally { FileUtils.deleteQuietly(toExecute); } }
From source file:org.opentestsystem.delivery.testreg.upload.TextFileUtils.java
public void processTextFile(final InputStream textFile, final String formatType, final LineMapper lineMapper) throws Exception { final String CHARSET = "UTF-8"; final TextFileProcessor<InputStream, String> textFileProcessor = new TextFileProcessor<InputStream, String>() { @Override/*from w ww . j ava2s .c o m*/ public String process(final InputStream input) throws Exception { LineIterator lineIterator = null; try { lineIterator = IOUtils.lineIterator(textFile, CHARSET); int lineNumber = 1; // First line starts at 1 while (lineIterator.hasNext()) { String line = lineIterator.next(); if (lineNumber == 1) { //worry about BOM chars (/ufeff) line = line.replaceAll("[\uFEFF]", ""); } if (!lineMapper.mapLine(line, lineNumber++, formatType)) { break; } } } catch (RuntimeException | IOException ex) { if (lineIterator != null) { lineIterator.close(); } IOUtils.closeQuietly(textFile); throw ex; } finally { if (lineIterator != null) { lineIterator.close(); } IOUtils.closeQuietly(textFile); } return null; } }; textFileProcessor.process(textFile); }
From source file:org.opoo.press.source.SourceParserImpl.java
@Override public Source parse(SourceEntry sourceEntry) throws NoFrontMatterException { List<String> metaLines = new ArrayList<String>(); List<String> contentLines = new ArrayList<String>(); InputStream stream = null;//from w w w. j av a 2s.c om List<String> currentList = metaLines; try { stream = new FileInputStream(sourceEntry.getFile()); LineIterator iterator = IOUtils.lineIterator(stream, "UTF-8"); if (!iterator.hasNext()) { throw new RuntimeException("File not content: " + sourceEntry.getFile()); } String line = iterator.next(); if (!isFrontMatterStartLine(line, sourceEntry)) { log.debug("Maybe a static file: " + sourceEntry.getFile()); throw new NoFrontMatterException(sourceEntry); } boolean hasFrontMatterEndLine = false; //process headers while (iterator.hasNext()) { line = iterator.next(); if (isFrontMatterEndLine(line)) { hasFrontMatterEndLine = true; currentList = contentLines; continue; } currentList.add(line); } if (!hasFrontMatterEndLine) { log.debug("Maybe a static file: " + sourceEntry.getFile()); throw new NoFrontMatterException(sourceEntry); } } catch (IOException e) { throw new RuntimeException(e); } finally { IOUtils.closeQuietly(stream); } StringWriter metaWriter = new StringWriter(); StringWriter contentWriter = new StringWriter(); try { IOUtils.writeLines(metaLines, null, metaWriter); IOUtils.writeLines(contentLines, null, contentWriter); } catch (IOException e) { throw new RuntimeException(e); } finally { IOUtils.closeQuietly(contentWriter); IOUtils.closeQuietly(metaWriter); } @SuppressWarnings("unchecked") Map<String, Object> map = (Map<String, Object>) yaml.load(metaWriter.toString()); String content = contentWriter.toString(); return new SimpleSource(sourceEntry, map, content); }
From source file:org.sonar.server.computation.batch.BatchReportReaderImpl.java
@Override public Optional<CloseableIterator<String>> readFileSource(int fileRef) { File file = delegate.readFileSource(fileRef); if (file == null) { return Optional.absent(); }// ww w.j a v a 2 s . co m try { return Optional.<CloseableIterator<String>>of(new CloseableLineIterator( IOUtils.lineIterator(FileUtils.openInputStream(file), StandardCharsets.UTF_8))); } catch (IOException e) { throw new IllegalStateException("Fail to traverse file: " + file, e); } }
From source file:org.sonar.server.computation.task.projectanalysis.batch.BatchReportReaderImpl.java
@Override public Optional<CloseableIterator<String>> readFileSource(int fileRef) { ensureInitialized();//from w w w. j ava2s . c o m File file = delegate.readFileSource(fileRef); if (file == null) { return Optional.absent(); } try { return Optional.of(new CloseableLineIterator( IOUtils.lineIterator(FileUtils.openInputStream(file), StandardCharsets.UTF_8))); } catch (IOException e) { throw new IllegalStateException("Fail to traverse file: " + file, e); } }
From source file:org.trend.hgraph.mapreduce.lib.input.CalculateInputSplitMapperTest.java
@Test public void testRun_b4() throws IOException, Exception { // init test table String tableName = "test.vertex-02"; String outputPath = "/run_b4"; createTestTable(tableName, "00030", "00060"); // start test Configuration conf = TEST_UTIL.getConfiguration(); Tool driver = new CalculateInputSplitMapper(conf); int code = driver.run(new String[] { "-b", "4", tableName, outputPath }); Assert.assertEquals(0, code);/*from w w w .j ava 2s.c o m*/ // get test results Path path = new Path(outputPath); FileSystem fs = path.getFileSystem(conf); // FileStatus[] files = fs.listStatus(path); // for (int a = 0; a < files.length; a++) { // System.out.println(files[a].getPath()); // } InputStream is = fs.open(new Path(path, "part-r-00000")); LineIterator it = IOUtils.lineIterator(is, "UTF-8"); System.out.println("print out test results"); while (it.hasNext()) { System.out.println(it.next()); } LineIterator.closeQuietly(it); IOUtils.closeQuietly(is); }
From source file:org.trend.hgraph.mapreduce.lib.input.TableInputFormat.java
@Override public List<InputSplit> getSplits(JobContext context) throws IOException { LinkedList<InputSplit> newSplits = new LinkedList<InputSplit>(); FileSystem fs = null;/*from w w w . j a v a2 s . c o m*/ InputStream is = null; LineIterator it = null; try { List<InputSplit> targets = super.getSplits(context); Map<String, List<TableSplit>> targetMap = createTargetMap(targets); HTable table = getHTable(); byte[] tableName = table.getTableName(); Path path = new Path(inputPath); fs = path.getFileSystem(this.getConf()); is = fs.open(path); it = IOUtils.lineIterator(is, "UTF-8"); String line = null; String[] lineSplits = null; String regionName = null; byte[] startKey = null; byte[] endKey = null; byte[] regionStartKey = null; byte[] regionEndKey = null; boolean nextRegion = false; HRegionLocation regionlocation = null; String location = null; HRegionInfo regionInfo = null; List<TableSplit> splits = null; TableSplit split = null; while (it.hasNext()) { if (!nextRegion) { line = it.nextLine(); } else { nextRegion = false; } if (null == line || "".equals(line)) { String msg = "skip a invalid line"; LOGGER.error(msg); throw new IllegalStateException(msg); } lineSplits = getLineSplits(line); regionName = lineSplits[0]; startKey = Bytes.toBytes(lineSplits[1]); endKey = Bytes.toBytes(lineSplits[2]); LOGGER.info("start to process region:" + regionName); regionlocation = table.getRegionLocation(startKey, false); if (null == regionlocation) { String msg = "can not find location for region:" + regionName + " !!"; LOGGER.error(msg); throw new IllegalStateException(msg); } regionInfo = regionlocation.getRegionInfo(); location = regionlocation.getHostnamePort().split(Addressing.HOSTNAME_PORT_SEPARATOR)[0]; if (!targetMap.containsKey(location)) { String msg = "regionLocation:" + regionlocation + " not found in TableSplits !!"; LOGGER.error(msg); throw new IllegalStateException(msg); } regionStartKey = regionInfo.getStartKey(); regionEndKey = regionInfo.getEndKey(); splits = targetMap.get(location); LOGGER.info("splits.size is " + splits.size()); for (int a = 0; a < splits.size(); a++) { split = splits.get(a); if (Arrays.equals(split.getStartRow(), regionStartKey) && Arrays.equals(split.getEndRow(), regionEndKey)) { splits.remove(a); if (Arrays.equals(HConstants.EMPTY_BYTE_ARRAY, regionStartKey)) { startKey = regionStartKey; } split = new TableSplit(tableName, startKey, endKey, location); newSplits.add(split); if (it.hasNext()) { while (it.hasNext()) { line = it.nextLine(); lineSplits = getLineSplits(line); // keep doing due to it may be in same region if (regionName.equals(lineSplits[0])) { split = new TableSplit(tableName, Bytes.toBytes(lineSplits[1]), Bytes.toBytes(lineSplits[2]), location); newSplits.add(split); } else { checkAndPatchRegionEndKey(newSplits, regionEndKey); nextRegion = true; break; } } } else { checkAndPatchRegionEndKey(newSplits, regionEndKey); } if (splits.size() == 0) { // remove itself if it is empty targetMap.remove(location); } break; } } } // check if still any split not processed yet ? if (targetMap.size() > 0) { String msg = "targetMap.size:" + targetMap.size() + " still bigger than 0," + " which means that there still has TableSplit(s) not processed yet !!"; LOGGER.error(msg); LOGGER.error("" + targetMap); throw new IllegalStateException(msg); } } catch (IOException e) { LOGGER.error("unknow error occues !!", e); throw e; } finally { if (null != it) { LineIterator.closeQuietly(it); } if (null != is) { IOUtils.closeQuietly(is); } } return newSplits; }
From source file:se.alingsas.alfresco.repo.utils.byggreda.ReadMetadataDocument.java
/** * Takes an input stream which should point to a metadata document for * byggreda. Validates and parses the data and returns a set of * ByggRedaDocument// w ww . j a va 2 s . c o m * * @param inputStream * @return */ public static Set<ByggRedaDocument> read(final InputStream inputStream, List<String> globalMessages) { if (inputStream == null) { return null; } Set<ByggRedaDocument> result = new HashSet<ByggRedaDocument>(); LineIterator lineIterator = null; String line = ""; int lineNumber = 1; try { lineIterator = IOUtils.lineIterator(inputStream, "ISO-8859-1"); // Skip first line which is a header line if (lineIterator.hasNext()) { line = lineIterator.nextLine(); if (!line.startsWith("\"Film\";\"") && !line.startsWith("Film;")) { globalMessages.add( "#1: Sidhuvud ej funnet p frsta raden i styrfilen. Frsta raden var: " + line); LOG.error("No header found on the first line in the document. First line was: " + line + ". Aborting..."); return result; } else { LOG.debug("Line #" + lineNumber + ": Skipping header"); } } while (lineIterator.hasNext()) { lineNumber++; line = lineIterator.nextLine(); // if it's an empty line or a comment, skip if (!StringUtils.hasText(line) || line.startsWith("#")) { globalMessages.add("#" + lineNumber + ": Tom rad, eller bortkommenterad rad funnel, skippas"); LOG.info("Line #" + lineNumber + ": Skipping comment or empty line"); continue; } // Validation and error handling ByggRedaDocument document = parseAndValidate(line); document.setLineNumber(lineNumber); if (!document.isReadSuccessfully()) { // An error occured, we need to log this LOG.error("Line #" + document.getLineNumber() + ": " + document.getStatusMsg()); } else { // Document successfully read LOG.debug("Line #" + document.getLineNumber() + ": " + "Successfully read record. , Record number: " + document.getRecordDisplay()); } result.add(document); } } catch (final Exception ex) { globalMessages.add("#" + lineNumber + ": Fel vid inlsning av rad " + lineNumber + " frn styrfil. Radens innehll: " + line + " Systemmeddelande: " + ex.getMessage()); LOG.error("Error on line '" + lineNumber + "'. Line contents: " + line, ex); } finally { IOUtils.closeQuietly(inputStream); LineIterator.closeQuietly(lineIterator); } return result; }