Java tutorial
package org.apache.tika.batch.fs.strawman; /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; import java.util.Arrays; import java.util.Date; import java.util.List; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorCompletionService; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.MarkerFactory; /** * Simple single-threaded class that calls tika-app against every file in a directory. * * This is exceedingly robust. One file per process. * * However, you can use this to compare performance against tika-batch fs code. * * */ public class StrawManTikaAppDriver implements Callable<Integer> { private static final Logger LOG = LoggerFactory.getLogger(StrawManTikaAppDriver.class); private static AtomicInteger threadCount = new AtomicInteger(0); private final int totalThreads; private final int threadNum; private Path inputRoot = null; private Path outputRoot = null; private String[] args = null; public StrawManTikaAppDriver(Path inputRoot, Path outputRoot, int totalThreads, String[] args) { this.inputRoot = inputRoot; this.outputRoot = outputRoot; this.args = args; threadNum = threadCount.getAndIncrement(); this.totalThreads = totalThreads; } private class TikaVisitor extends SimpleFileVisitor<Path> { private int processed = 0; int getProcessed() { return processed; } @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attr) { if (totalThreads > 1) { int hashCode = file.toAbsolutePath().toString().hashCode(); if (Math.abs(hashCode % totalThreads) != threadNum) { return FileVisitResult.CONTINUE; } } assert (file.startsWith(inputRoot)); Path relPath = inputRoot.relativize(file); Path outputFile = Paths.get(outputRoot.toAbsolutePath().toString(), relPath.toString() + ".txt"); try { Files.createDirectories(outputFile.getParent()); } catch (IOException e) { LOG.error(MarkerFactory.getMarker("FATAL"), "parent directory for {} was not made!", outputFile); throw new RuntimeException("couldn't make parent file for " + outputFile); } List<String> commandLine = new ArrayList<>(); for (String arg : args) { commandLine.add(arg); } commandLine.add("-t"); commandLine.add("\"" + outputFile.toAbsolutePath() + "\""); ProcessBuilder builder = new ProcessBuilder(commandLine.toArray(new String[commandLine.size()])); LOG.info("about to process: {}", file.toAbsolutePath()); Process proc = null; RedirectGobbler gobbler = null; Thread gobblerThread = null; try { OutputStream os = Files.newOutputStream(outputFile); proc = builder.start(); gobbler = new RedirectGobbler(proc.getInputStream(), os); gobblerThread = new Thread(gobbler); gobblerThread.start(); } catch (IOException e) { LOG.error(e.getMessage(), e); return FileVisitResult.CONTINUE; } boolean finished = false; long totalTime = 180000;//3 minutes long pulse = 100; for (int i = 0; i < totalTime; i += pulse) { try { Thread.sleep(pulse); } catch (InterruptedException e) { //swallow } try { int exit = proc.exitValue(); finished = true; break; } catch (IllegalThreadStateException e) { //swallow } } if (!finished) { LOG.warn("Had to kill process working on: {}", file.toAbsolutePath()); proc.destroy(); } gobbler.close(); gobblerThread.interrupt(); processed++; return FileVisitResult.CONTINUE; } } @Override public Integer call() throws Exception { long start = new Date().getTime(); TikaVisitor v = new TikaVisitor(); Files.walkFileTree(inputRoot, v); int processed = v.getProcessed(); double elapsedSecs = ((double) new Date().getTime() - (double) start) / (double) 1000; LOG.info("Finished processing {} files in {} seconds.", processed, elapsedSecs); return processed; } private class RedirectGobbler implements Runnable { private OutputStream redirectOs = null; private InputStream redirectIs = null; private RedirectGobbler(InputStream is, OutputStream os) { this.redirectIs = is; this.redirectOs = os; } private void close() { if (redirectOs != null) { try { redirectOs.flush(); } catch (IOException e) { LOG.error("can't flush"); } try { redirectIs.close(); } catch (IOException e) { LOG.error("can't close input in redirect gobbler"); } try { redirectOs.close(); } catch (IOException e) { LOG.error("can't close output in redirect gobbler"); } } } @Override public void run() { try { IOUtils.copy(redirectIs, redirectOs); } catch (IOException e) { LOG.error("IOException while gobbling"); } } } public static String usage() { StringBuilder sb = new StringBuilder(); sb.append("Example usage:\n"); sb.append("java -cp <CP> org.apache.batch.fs.strawman.StrawManTikaAppDriver "); sb.append("<inputDir> <outputDir> <numThreads> "); sb.append("java -jar tika-app-X.Xjar <...commandline arguments for tika-app>\n\n"); return sb.toString(); } public static void main(String[] args) { long start = new Date().getTime(); if (args.length < 6) { System.err.println(StrawManTikaAppDriver.usage()); } Path inputDir = Paths.get(args[0]); Path outputDir = Paths.get(args[1]); int totalThreads = Integer.parseInt(args[2]); List<String> commandLine = new ArrayList<>(); commandLine.addAll(Arrays.asList(args).subList(3, args.length)); totalThreads = (totalThreads < 1) ? 1 : totalThreads; ExecutorService ex = Executors.newFixedThreadPool(totalThreads); ExecutorCompletionService<Integer> completionService = new ExecutorCompletionService<>(ex); for (int i = 0; i < totalThreads; i++) { StrawManTikaAppDriver driver = new StrawManTikaAppDriver(inputDir, outputDir, totalThreads, commandLine.toArray(new String[commandLine.size()])); completionService.submit(driver); } int totalFilesProcessed = 0; for (int i = 0; i < totalThreads; i++) { try { Future<Integer> future = completionService.take(); if (future != null) { totalFilesProcessed += future.get(); } } catch (InterruptedException | ExecutionException e) { LOG.error(e.getMessage(), e); } } double elapsedSeconds = (double) (new Date().getTime() - start) / (double) 1000; LOG.info("Processed {} in {} seconds", totalFilesProcessed, elapsedSeconds); } }