Java tutorial
/* Copyright (c) 2011 Danish Maritime Authority * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 3 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this library. If not, see <http://www.gnu.org/licenses/>. */ package dk.dma.ais.downloader; import dk.dma.ais.packet.AisPacketFilters; import org.apache.tomcat.util.http.fileupload.IOUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Controller; import org.springframework.util.StringUtils; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.ResponseBody; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import javax.servlet.http.HttpServletResponse; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.PrintStream; import java.net.URL; import java.net.URLConnection; import java.nio.channels.Channels; import java.nio.channels.ReadableByteChannel; import java.nio.file.DirectoryStream; import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Collections; import java.util.Date; import java.util.List; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.logging.Level; import java.util.logging.Logger; /** * A Query service.<br> * Used for issuing am AIS store query. The query result will be downloaded * and stored in a client specific folder. * <p> * Also handles streaming, listing and deleting files from the client specific * download folder. */ @Controller @RequestMapping("/downloader/query") @SuppressWarnings("unused") public class QueryService { /** * Defined a worker pool size of 2 to constrain load */ private static final int EXECUTOR_POOL_SIZE = 10; private static final long FILE_EXPIRY_MS = 1000L * 60L * 60L * 4L; // 4 hours private final static Logger log = Logger.getLogger(QueryService.class.getName()); private final static String DOWNLOAD_SUFFIX = ".download"; private Path repoRoot; private ExecutorService processPool; @Value("${ais.view.url:https://ais2.e-navigation.net/aisview/rest/store/query?}") String aisViewUrl; @Value("${repo.root:}") String repoRootPath; @Value("${auth.header:}") String authHeader; /** * Initializes the repository */ @PostConstruct public void init() throws Exception { log.info("******** Using AIS View URL: " + aisViewUrl); // Create the repo root directory if (StringUtils.isEmpty(repoRootPath)) { repoRoot = Paths.get(System.getProperty("user.home")).resolve(".aisdownloader"); } else { repoRoot = Paths.get(repoRootPath); } log.info("******** Using repo root " + repoRoot); if (!Files.exists(getRepoRoot())) { try { Files.createDirectories(getRepoRoot()); } catch (IOException e) { log.log(Level.SEVERE, "Error creating repository dir " + getRepoRoot(), e); } } if (!StringUtils.isEmpty(authHeader)) { log.info("******** Using auth header: " + authHeader); } // Initialize process pool processPool = Executors.newFixedThreadPool(EXECUTOR_POOL_SIZE); log.info("Initialized the QueryService"); } @PreDestroy public void cleanUp() throws Exception { if (processPool != null && !processPool.isShutdown()) { processPool.shutdown(); processPool = null; } log.info("Destroyed the QueryService"); } /** * Returns the repository root * @return the repository root */ public Path getRepoRoot() { return repoRoot; } /** * Creates a URI from the repo file * @param repoFile the repo file * @return the URI for the file */ public String getRepoUri(Path repoFile) { Path filePath = getRepoRoot().relativize(repoFile); return "/rest/repo/file/" + filePath; } /** * Creates a path from the repo file relative to the repo root * @param repoFile the repo file * @return the path for the file */ public String getRepoPath(Path repoFile) { Path filePath = getRepoRoot().relativize(repoFile); return filePath.toString().replace('\\', '/'); } /** * Asynchronously loads the given file * @param url the URL to load * @param path the path to save the file to */ private Future<Path> asyncLoadFile(final String url, final Path path) { Callable<Path> job = () -> { long t0 = System.currentTimeMillis(); // For the resulting file, drop the ".download" suffix String name = path.getFileName().toString(); name = name.substring(0, name.length() - DOWNLOAD_SUFFIX.length()); try { // Set up a few timeouts and fetch the attachment URLConnection con = new URL(url).openConnection(); con.setConnectTimeout(60 * 1000); // 1 minute con.setReadTimeout(60 * 60 * 1000); // 1 hour if (!StringUtils.isEmpty(authHeader)) { con.setRequestProperty("Authorization", authHeader); } try (ReadableByteChannel rbc = Channels.newChannel(con.getInputStream()); FileOutputStream fos = new FileOutputStream(path.toFile())) { fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); } log.info(String.format("Copied %s -> %s in %d ms", url, path, System.currentTimeMillis() - t0)); } catch (Exception e) { log.log(Level.SEVERE, "Failed downloading " + url + ": " + e.getMessage()); // Delete the old file if (Files.exists(path)) { try { Files.delete(path); } catch (IOException e1) { log.finer("Failed deleting old file " + path); } } // Save an error file Path errorFile = path.getParent().resolve(name + ".err.txt"); try (PrintStream err = new PrintStream(new FileOutputStream(errorFile.toFile()))) { e.printStackTrace(err); } catch (IOException ex) { log.finer("Failed generating error file " + errorFile); } return errorFile; } Path resultPath = path.getParent().resolve(name); try { Files.move(path, resultPath); } catch (IOException e) { log.log(Level.SEVERE, "Failed renaming path " + path + ": " + e.getMessage()); } return resultPath; }; log.info("Submitting new job: " + url); return processPool.submit(job); } /** * Execute the given query * @param clientId the client id * @param async whether to execute synchronously or asynchronously * @param params the query parameters * @return the result file */ @RequestMapping(value = "/execute/{clientId}", method = RequestMethod.GET) @ResponseBody public RepoFile executeQuery(@PathVariable("clientId") String clientId, @RequestParam(value = "async", defaultValue = "true") boolean async, @RequestParam("params") String params) throws IOException { String url = aisViewUrl + params; // Create the client ID folder Path dir = repoRoot.resolve(clientId); if (!Files.exists(dir)) { Files.createDirectories(dir); } // Create a new file to hold the result // (could have used Files.createTempFile, but this should be enough to create a unique file) Date now = new Date(); Path file = Files.createTempFile(dir, new SimpleDateFormat("MM-dd HHmmss ").format(now), fileType(url) + DOWNLOAD_SUFFIX); String fileName = file.getFileName().toString(); // Load the file Future<Path> result = asyncLoadFile(url, file); if (!async) { try { Path path = result.get(); // The resulting path may actually by an error file fileName = path.getFileName().toString(); } catch (Exception e) { log.severe("Error executing query: " + params + ", error: " + e); } } // Return a RepoFile for the newly created file RepoFile vo = new RepoFile(); vo.setName(fileName); vo.setPath(clientId + "/" + fileName); vo.setUpdated(now); vo.setSize(0L); return vo; } /** * Returns the file type of the given URL * @param url the url * @return the file type */ private String fileType(String url) { if (url.contains("OUTPUT_TO_KML")) { return ".kml"; } else if (url.contains("OUTPUT_TO_HTML")) { return ".html"; } else if (url.contains("table")) { return ".csv"; } else if (url.contains("json")) { return ".json"; } return ".txt"; } /** * Streams the file specified by the path */ @RequestMapping(value = "/file/{clientId}/{file:.*}", method = RequestMethod.GET) public void streamFile(@PathVariable("clientId") String clientId, @PathVariable("file") String file, HttpServletResponse response) throws IOException { Path path = repoRoot.resolve(clientId).resolve(file); if (Files.notExists(path) || Files.isDirectory(path)) { log.log(Level.WARNING, "Failed streaming file: " + path); response.setStatus(404); return; } response.setContentType(Files.probeContentType(path)); try (InputStream in = Files.newInputStream(path)) { IOUtils.copy(in, response.getOutputStream()); response.flushBuffer(); } } /** * Deletes the file specified by the path */ @RequestMapping(value = "/delete/{clientId}/{file:.*}", method = RequestMethod.GET) @ResponseBody public String deleteFile(@PathVariable("clientId") String clientId, @PathVariable("file") String file, HttpServletResponse response) throws IOException { Path path = repoRoot.resolve(clientId).resolve(file); if (Files.notExists(path) || Files.isDirectory(path)) { log.log(Level.WARNING, "Failed deleting file: " + path); response.setStatus(404); return "404"; } Files.delete(path); log.info("Deleted " + path); return "Deleted " + path; } /** * Deletes all the file of the client folder */ @RequestMapping(value = "/delete-all/{clientId}", method = RequestMethod.GET) @ResponseBody public String deleteFiles(@PathVariable("clientId") String clientId, HttpServletResponse response) throws IOException { int deletedFiles = 0; Path path = repoRoot.resolve(clientId); if (Files.notExists(path) || !Files.isDirectory(path)) { log.log(Level.WARNING, "Failed deleting files in " + path); response.setStatus(404); return "Failed deleting files in " + clientId; } try { Files.walkFileTree(path, new SimpleFileVisitor<Path>() { @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { log.info("Deleting repo file :" + file); Files.delete(file); return FileVisitResult.CONTINUE; } }); } catch (IOException e) { log.log(Level.SEVERE, "Failed cleaning up dir: " + path); return "Failed deleting files in " + clientId; } return "Deleted files in dir " + clientId; } /** * Returns a list of files in the folder specified by the clientId * @return the list of files in the folder specified by the path */ @RequestMapping(value = "/list/{clientId:.*}", method = RequestMethod.GET, produces = "application/json;charset=UTF-8") @ResponseBody public List<RepoFile> listFiles(@PathVariable("clientId") String clientId) throws IOException { List<RepoFile> result = new ArrayList<>(); Path folder = repoRoot.resolve(clientId); if (Files.exists(folder) && Files.isDirectory(folder)) { // Filter out directories and hidden files DirectoryStream.Filter<Path> filter = file -> Files.isRegularFile(file) && !file.getFileName().toString().startsWith("."); try (DirectoryStream<Path> stream = Files.newDirectoryStream(folder, filter)) { stream.forEach(f -> { RepoFile vo = new RepoFile(); vo.setName(f.getFileName().toString()); vo.setPath(clientId + "/" + f.getFileName().toString()); try { vo.setUpdated(new Date(Files.getLastModifiedTime(f).toMillis())); vo.setSize(Files.size(f)); } catch (Exception e) { log.finer("Error reading file attribute for " + f); } vo.setComplete(!f.getFileName().toString().endsWith(DOWNLOAD_SUFFIX)); result.add(vo); }); } } Collections.sort(result); return result; } /** * Validates the AIS filter passed along. The filter must adhere to the * grammar defined by the AisLib: * https://github.com/dma-ais/AisLib * @param filter the filter to validate * @return the the filter is valid or not */ @RequestMapping(value = "/validate-filter", method = RequestMethod.GET) @ResponseBody public boolean validateFilter(@RequestParam("filter") String filter) { // A blank filter is valid if (StringUtils.isEmpty(filter)) { return true; } // Check if the filter can be parsed try { AisPacketFilters.parseExpressionFilter(filter); log.fine("Successfully parsed filter: " + filter); return true; } catch (Exception e) { log.fine("Failed parsing filter: " + filter + ": " + e); return false; } } /***************************************/ /** Repo clean-up methods **/ /***************************************/ /** * called every hour to clean up the repo */ @Scheduled(cron = "12 27 */1 * * *") public void cleanUpRepoFolder() { long now = System.currentTimeMillis(); long expiredTime = now - FILE_EXPIRY_MS; try { Files.walkFileTree(getRepoRoot(), new SimpleFileVisitor<Path>() { @Override public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { if (!dir.equals(getRepoRoot()) && isDirEmpty(dir)) { log.info("Deleting repo directory :" + dir); Files.delete(dir); } return FileVisitResult.CONTINUE; } @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { if (Files.getLastModifiedTime(file).toMillis() < expiredTime) { log.info("Deleting repo file :" + file); Files.delete(file); } return FileVisitResult.CONTINUE; } }); } catch (IOException e) { log.log(Level.SEVERE, "Failed cleaning up repo: " + e.getMessage()); } log.info(String.format("Cleaned up repo in %d ms", System.currentTimeMillis() - now)); } /** * Returns if the directory is empty or not * @param directory the directory to check * @return if the directory is empty or not */ private static boolean isDirEmpty(final Path directory) throws IOException { try (DirectoryStream<Path> dirStream = Files.newDirectoryStream(directory)) { return !dirStream.iterator().hasNext(); } catch (Exception e) { // Should never happen return false; } } }