Java tutorial
/** * Tools IO * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package edu.umd.umiacs.clip.tools.io; import java.io.BufferedInputStream; import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.Closeable; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.UncheckedIOException; import java.nio.charset.Charset; import java.nio.charset.CharsetEncoder; import static java.nio.charset.CodingErrorAction.IGNORE; import static java.nio.charset.StandardCharsets.UTF_8; import java.nio.file.Files; import static java.nio.file.Files.newInputStream; import static java.nio.file.Files.newOutputStream; import java.nio.file.OpenOption; import java.nio.file.Path; import static java.nio.file.StandardOpenOption.CREATE_NEW; import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Objects; import java.util.stream.Stream; import java.util.stream.StreamSupport; import java.util.zip.GZIPOutputStream; import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVRecord; /** * * @author Mossaab Bagdouri */ public class AllFiles { public static final int BUFFER_SIZE = 1024 * 1024; public static final boolean REMOVE_OLD_FILE = true; private static List<String> _readAllLines(Path path) throws IOException { try (BufferedReader reader = new BufferedReader( new InputStreamReader(new BufferedInputStream(newInputStream(path), BUFFER_SIZE), UTF_8.newDecoder().onMalformedInput(IGNORE)))) { List<String> result = new ArrayList<>(); String line; while ((line = reader.readLine()) != null) { result.add(line); } return result; } } public static List<String> readAllLines(Path path) { try { return path.toString().endsWith(".gz") ? GZIPFiles.readAllLines(path) : path.toString().endsWith(".bz2") ? BZIP2Files.readAllLines(path) : _readAllLines(path); } catch (IOException e) { throw new UncheckedIOException(e); } } public static List<String> readAllLines(File file) { return readAllLines(file.toPath()); } private static String format(String path) { if (path.startsWith("~" + File.separator)) { path = System.getProperty("user.home") + path.substring(1); } return path; } public static List<String> readAllLines(String path) { File file = new File(format(path)); if (!path.contains("*")) { return readAllLines(file); } List<String> lines = new ArrayList(); Stream.of(file.getParentFile() .listFiles((dir, name) -> name.matches(file.getName().replace(".", "\\.").replace("*", ".+")))) .sorted().map(AllFiles::readAllLines).flatMap(Collection::stream).forEach(lines::add); return lines; } public static List<String> readAllLinesFromResource(String path) { path = format(path); return path.endsWith(".gz") ? GZIPFiles.readAllLinesFromResource(path) : path.endsWith(".bz2") ? BZIP2Files.readAllLinesFromResource(path) : readAllLines(System.class.getResourceAsStream(path)); } public static List<String> readAllLines(InputStream is) { List<String> lines = new ArrayList<>(); try (BufferedReader bis = new BufferedReader( new InputStreamReader(is, UTF_8.newDecoder().onMalformedInput(IGNORE)))) { String line; while ((line = bis.readLine()) != null) { line = line.trim(); if (!line.isEmpty()) { lines.add(line); } } } catch (IOException e) { e.printStackTrace(); } return lines; } public static Stream<String> lines(InputStream is) { BufferedReader br = new BufferedReader( new InputStreamReader(is, UTF_8.newDecoder().onMalformedInput(IGNORE))); return br.lines().onClose(asUncheckedRunnable(br)); } protected static Runnable asUncheckedRunnable(Closeable c) { return () -> { try { c.close(); } catch (IOException e) { throw new UncheckedIOException(e); } }; } public static Stream<File> list(Path dir) { try { return Files.list(dir).map(Path::toFile); } catch (IOException e) { throw new UncheckedIOException(e); } } public static Stream<File> list(File dir) { return list(dir.toPath()); } public static Stream<File> list(String dir) { return list(new File(format(dir))); } public static Stream<String> lines() { BufferedReader br = new BufferedReader(new InputStreamReader( new BufferedInputStream(System.in, BUFFER_SIZE), UTF_8.newDecoder().onMalformedInput(IGNORE))); return br.lines().onClose(asUncheckedRunnable(br)); } public static Stream<CSVRecord> records(CSVFormat format, Path path) { try { String p = path.toString(); if (!p.contains("*")) { return p.endsWith(".gz") ? GZIPFiles.records(format, path) : p.endsWith(".bz2") ? BZIP2Files.records(format, path) : overridenRecords(format, path); } else { File file = path.toFile(); return Stream .of(file.getParentFile().listFiles( (dir, name) -> name.matches(file.getName().replace(".", "\\.").replace("*", ".+")))) .sorted().flatMap(f -> records(format, f)); } } catch (IOException e) { throw new UncheckedIOException(e); } } public static Stream<CSVRecord> records(CSVFormat format, File file) { return records(format, file.toPath()); } public static Stream<CSVRecord> records(CSVFormat format, String path) { return records(format, new File(path)); } public static Stream<String> lines(Path path) { try { String p = path.toString(); if (!p.contains("*")) { return p.endsWith(".gz") ? GZIPFiles.lines(path) : p.endsWith(".bz2") ? BZIP2Files.lines(path) : overridenLines(path); } else { File file = path.toFile(); return Stream .of(file.getParentFile().listFiles( (dir, name) -> name.matches(file.getName().replace(".", "\\.").replace("*", ".+")))) .sorted().flatMap(AllFiles::lines); } } catch (IOException e) { throw new UncheckedIOException(e); } } private static Stream<String> overridenLines(Path path) throws IOException { BufferedReader br = null; try { br = new BufferedReader( new InputStreamReader(new BufferedInputStream(newInputStream(path), BUFFER_SIZE), UTF_8.newDecoder().onMalformedInput(IGNORE))); return br.lines().onClose(asUncheckedRunnable(br)); } catch (IOException e) { try { br.close(); } catch (Exception ex) { try { e.addSuppressed(ex); } catch (Throwable ignore) { } } throw e; } } private static Stream<CSVRecord> overridenRecords(CSVFormat format, Path path) throws IOException { return StreamSupport.stream(format.parse( new BufferedReader(new InputStreamReader(new BufferedInputStream(newInputStream(path), BUFFER_SIZE), UTF_8.newDecoder().onMalformedInput(IGNORE)))) .spliterator(), false); } public static Stream<String> lines(File file) { return lines(file.toPath()); } public static Stream<String> lines(String path) { return lines(new File(format(path))); } public static Path write(Path path, Stream<?> lines, Charset cs, OpenOption... options) { if (options.length == 0) { options = new OpenOption[] { CREATE_NEW }; } Objects.requireNonNull(lines); CharsetEncoder encoder = cs.newEncoder(); try { OutputStream out = newOutputStream(path, options); if (path.toString().endsWith(".gz")) { out = new GZIPOutputStream(out); } else if (path.toString().endsWith(".bz2")) { out = new BZip2CompressorOutputStream(out); } try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(out, encoder), BUFFER_SIZE)) { lines.forEach(line -> { try { writer.append(line.toString()); writer.newLine(); } catch (Exception e) { e.printStackTrace(); } }); } } catch (IOException e) { throw new UncheckedIOException(e); } return path; } public static Path write(Path path, Stream<?> lines, OpenOption... options) { return write(path, lines, UTF_8, options); } public static Path write(String path, Iterable<? extends CharSequence> lines, OpenOption... options) { return write(new File(format(path)), lines, options); } public static Path write(String path, Iterable<? extends CharSequence> lines, boolean removeOldFile, OpenOption... options) { path = format(path); if (removeOldFile) { new File(path).delete(); } return write(new File(path), lines, options); } public static Path write(String path, Stream<?> lines, OpenOption... options) { return write(new File(format(path)), lines, options); } public static Path write(String path, Stream<?> lines, boolean removeOldFile, OpenOption... options) { path = format(path); if (removeOldFile) { new File(path).delete(); } return write(new File(path), lines, options); } public static void write(Stream<?> lines) { lines.forEach(System.out::println); } public static void write(Iterable<?> lines) { lines.forEach(System.out::println); } public static Path write(File file, Iterable<? extends CharSequence> lines, OpenOption... options) { return write(file.toPath(), lines, options); } public static Path write(File file, Stream<?> lines, OpenOption... options) { return write(file.toPath(), lines, options); } public static Path write(Path path, Iterable<? extends CharSequence> lines, OpenOption... options) { if (options.length == 0) { options = new OpenOption[] { CREATE_NEW }; } try { File parent = path.getParent().toFile(); if (!parent.exists()) { parent.mkdirs(); } return path.toString().endsWith(".gz") ? GZIPFiles.write(path, lines, UTF_8, options) : path.toString().endsWith(".bz2") ? BZIP2Files.write(path, lines, UTF_8, options) : Files.write(path, lines, UTF_8, options); } catch (IOException e) { throw new UncheckedIOException(e); } } public static void writeln(String path, String line) { try { OutputStream os = new FileOutputStream(format(path), true); if (path.endsWith(".gz")) { os = new GZIPOutputStream(os); } else if (path.endsWith(".bz2")) { os = new BZip2CompressorOutputStream(os); } try (BufferedWriter out = new BufferedWriter(new OutputStreamWriter(os, "UTF-8"))) { out.write(line + "\n"); } } catch (IOException e) { e.printStackTrace(); } } }