Java tutorial
/* * Copyright 2016 Danish Maritime Authority. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.niord.core.repo; import org.apache.commons.fileupload.FileItem; import org.apache.commons.fileupload.FileItemFactory; import org.apache.commons.fileupload.FileUploadException; import org.apache.commons.fileupload.disk.DiskFileItemFactory; import org.apache.commons.fileupload.servlet.FileCleanerCleanup; import org.apache.commons.fileupload.servlet.ServletFileUpload; import org.apache.commons.io.FileCleaningTracker; import org.apache.commons.io.FileUtils; import org.apache.commons.io.FilenameUtils; import org.apache.commons.lang.StringUtils; import org.jboss.resteasy.annotations.cache.NoCache; import org.jboss.security.annotation.SecurityDomain; import org.niord.core.settings.annotation.Setting; import org.niord.core.user.Roles; import org.niord.core.util.WebUtils; import org.slf4j.Logger; import javax.annotation.PostConstruct; import javax.annotation.security.PermitAll; import javax.annotation.security.RolesAllowed; import javax.ejb.Lock; import javax.ejb.LockType; import javax.ejb.Schedule; import javax.ejb.Singleton; import javax.inject.Inject; import javax.servlet.ServletContext; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; import javax.ws.rs.POST; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.Context; import javax.ws.rs.core.EntityTag; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Request; import javax.ws.rs.core.Response; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.net.URI; import java.net.URISyntaxException; import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; import java.util.Date; import java.util.List; import java.util.UUID; import static org.niord.core.settings.Setting.Type; /** * A repository service.<br> * Streams files from the repository and facilitates uploading files to the repository. * <p> * The repository is public in as much as everybody can download all files.<br> * * However only "admin" users can upload files to the entire repository.<br> * * Registered users, with the "user" role, can upload files to a sub-root of * the repository, the {@code repoTempRoot}. Files upload to the this part * of the repository will be deleted after 24 hours. * </p> */ @javax.ws.rs.Path("/repo") @Singleton @Lock(LockType.READ) @SecurityDomain("keycloak") @PermitAll @SuppressWarnings("unused") public class RepositoryService { @Context ServletContext servletContext; @Inject @Setting(value = "repoRootPath", defaultValue = "${niord.home}/repo", description = "The root directory of the Niord repository") Path repoRoot; @Inject @Setting(value = "repoCacheTimeout", defaultValue = "5", description = "Cache timeout of repo files in minutes", type = Type.Integer) Integer cacheTimeout; @Inject @Setting(value = "repoFileUploadMaxSize", defaultValue = "1073741824", description = "Max size of uploaded file", type = Type.Long) Long fileUploadMaxSize; @Inject Logger log; @Inject FileTypes fileTypes; @Inject ThumbnailService thumbnailService; /** * Initializes the repository */ @PostConstruct public void init() { // Create the repo root directory if (!Files.exists(getRepoRoot())) { try { Files.createDirectories(getRepoRoot()); } catch (IOException e) { log.error("Error creating repository dir " + getRepoRoot(), e); } } // Create the repo "temp" root directory if (!Files.exists(getTempRepoRoot())) { try { Files.createDirectories(getTempRepoRoot()); } catch (IOException e) { log.error("Error creating repository dir " + getTempRepoRoot(), e); } } } /** * Returns the repository root * @return the repository root */ public Path getRepoRoot() { return repoRoot; } /** * Returns the repository "temp" root * @return the repository "temp" root */ public Path getTempRepoRoot() { return getRepoRoot().resolve("temp"); } /** * Creates a URI from the repo file * @param repoFile the repo file * @return the URI for the file */ public String getRepoUri(Path repoFile) { Path filePath = getRepoRoot().relativize(repoFile); return "/rest/repo/file/" + WebUtils.encodeURI(filePath.toString().replace('\\', '/')); } /** * Creates a path from the repo file relative to the repo root * @param repoFile the repo file * @return the path for the file */ public String getRepoPath(Path repoFile) { Path filePath = getRepoRoot().relativize(repoFile); return filePath.toString().replace('\\', '/'); } /** * Creates two levels of sub-folders within the {@code rootFolder} based on * a MD5 hash of the {@code target}. * If the sub-folder does not exist, it is created. * * @param rootFolder the root folder within the repository root * @param target the target name used for the hash * @param includeTarget whether to create a sub-folder for the target or not * @return the sub-folder associated with the target */ public Path getHashedSubfolder(String rootFolder, String target, boolean includeTarget) throws IOException { byte[] bytes = target.getBytes("utf-8"); // MD5 hash the ID MessageDigest md; try { md = MessageDigest.getInstance("MD5"); } catch (NoSuchAlgorithmException e) { throw new IOException("This should never happen"); } md.update(bytes); bytes = md.digest(); String hash = String.valueOf(Integer.toHexString(bytes[0] & 0xff)); while (hash.length() < 2) { hash = "0" + hash; } Path folder = getRepoRoot(); // Add the root folder if (StringUtils.isNotBlank(rootFolder)) { folder = folder.resolve(rootFolder); } // Add two hashed sub-folder levels folder = folder.resolve(hash.substring(0, 1)).resolve(hash.substring(0, 2)); // Check if we should create a sub-folder for the target as well if (includeTarget) { folder = folder.resolve(target); } // Create the folder if it does not exist if (!Files.exists(folder)) { Files.createDirectories(folder); } return folder; } /** * Streams the file specified by the path * @param path the path * @param request the servlet request * @return the response */ @GET @javax.ws.rs.Path("/file/{file:.+}") public Response streamFile(@PathParam("file") String path, @Context Request request) throws IOException { Path f = repoRoot.resolve(path); if (Files.notExists(f) || Files.isDirectory(f)) { log.warn("Failed streaming file: " + f); return Response.status(HttpServletResponse.SC_NOT_FOUND).entity("File not found: " + path).build(); } // Set expiry to cacheTimeout minutes Date expirationDate = new Date(System.currentTimeMillis() + 1000L * 60L * cacheTimeout); String mt = fileTypes.getContentType(f); // Check for an ETag match EntityTag etag = new EntityTag("" + Files.getLastModifiedTime(f).toMillis() + "_" + Files.size(f), true); Response.ResponseBuilder responseBuilder = request.evaluatePreconditions(etag); if (responseBuilder != null) { // Etag match log.trace("File unchanged. Return code 304"); return responseBuilder.expires(expirationDate).build(); } log.trace("Streaming file: " + f); return Response.ok(f.toFile(), mt).expires(expirationDate).tag(etag).build(); } /** * Deletes the file specified by the path * @param path the path * @return the response */ @DELETE @javax.ws.rs.Path("/file/{file:.+}") @Produces("text/plain") @RolesAllowed(Roles.EDITOR) public Response deleteFile(@PathParam("file") String path) throws IOException { Path f = repoRoot.resolve(path); if (Files.notExists(f) || Files.isDirectory(f)) { log.warn("Failed deleting non-existing file: " + f); return Response.status(HttpServletResponse.SC_NOT_FOUND).entity("File not found: " + path).build(); } Files.delete(f); log.info("Deleted file " + f); return Response.ok("File deleted " + f).build(); } /** * Returns the thumbnail to use for the file specified by the path * @param path the path * @param size the icon size, either 32, 64 or 128 * @return the thumbnail to use for the file specified by the path */ @GET @javax.ws.rs.Path("/thumb/{file:.+}") public Response getThumbnail(@PathParam("file") String path, @QueryParam("size") @DefaultValue("64") int size) throws IOException, URISyntaxException { IconSize iconSize = IconSize.getIconSize(size); Path f = repoRoot.resolve(path); if (Files.notExists(f) || Files.isDirectory(f)) { log.warn("Failed streaming file: " + f); return Response.status(HttpServletResponse.SC_NOT_FOUND).entity("File not found: " + path).build(); } // Check if we can generate a thumbnail for image files String thumbUri; Path thumbFile = thumbnailService.getThumbnail(f, iconSize); if (thumbFile != null) { thumbUri = "../" + getRepoUri(thumbFile); } else { // Fall back to file type icons thumbUri = "../" + fileTypes.getIcon(f, iconSize); } log.trace("Redirecting to thumbnail: " + thumbUri); return Response.temporaryRedirect(new URI(thumbUri)).build(); } /** * Returns a list of files in the folder specified by the path * @param path the path * @return the list of files in the folder specified by the path */ @GET @javax.ws.rs.Path("/list/{folder:.+}") @Produces("application/json;charset=UTF-8") @NoCache public List<RepoFileVo> listFiles(@PathParam("folder") String path) throws IOException { List<RepoFileVo> result = new ArrayList<>(); Path folder = repoRoot.resolve(path); if (Files.exists(folder) && Files.isDirectory(folder)) { // Filter out directories, hidden files, thumbnails and map images DirectoryStream.Filter<Path> filter = file -> Files.isRegularFile(file) && !file.getFileName().toString().startsWith(".") && !file.getFileName().toString().matches(".+_thumb_\\d{1,3}\\.\\w+") && // Thumbnails !file.getFileName().toString().matches("map_\\d{1,3}\\.png"); // Map image try (DirectoryStream<Path> stream = Files.newDirectoryStream(folder, filter)) { stream.forEach(f -> { RepoFileVo vo = new RepoFileVo(); vo.setName(f.getFileName().toString()); vo.setPath(WebUtils.encodeURI(path + "/" + f.getFileName().toString())); vo.setDirectory(Files.isDirectory(f)); try { vo.setUpdated(new Date(Files.getLastModifiedTime(f).toMillis())); vo.setSize(Files.size(f)); } catch (Exception e) { log.trace("Error reading file attribute for " + f); } result.add(vo); }); } } return result; } /** * Handles upload of files * * @param path the folder to upload to * @param request the request */ @POST @javax.ws.rs.Path("/upload/{folder:.+}") @Consumes(MediaType.MULTIPART_FORM_DATA) @Produces("application/json;charset=UTF-8") @RolesAllowed(Roles.EDITOR) public List<String> uploadFile(@PathParam("folder") String path, @Context HttpServletRequest request) throws FileUploadException, IOException { Path folder = repoRoot.resolve(path); if (Files.exists(folder) && !Files.isDirectory(folder)) { log.warn("Failed streaming file to folder: " + folder); throw new WebApplicationException("Invalid upload folder: " + path, 403); } else if (Files.notExists(folder)) { try { Files.createDirectories(folder); } catch (IOException e) { log.error("Error creating repository folder " + folder, e); throw new WebApplicationException("Invalid upload folder: " + path, 403); } } List<String> result = new ArrayList<>(); List<FileItem> items = parseFileUploadRequest(request); for (FileItem item : items) { if (!item.isFormField()) { // Argh - IE includes the path in the item.getName()! String fileName = Paths.get(item.getName()).getFileName().toString(); File destFile = getUniqueFile(folder, fileName).toFile(); log.info("File " + fileName + " is uploaded to " + destFile); try (BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(destFile))) { InputStream in = new BufferedInputStream(item.getInputStream()); byte[] buffer = new byte[1024]; int len = in.read(buffer); while (len != -1) { out.write(buffer, 0, len); len = in.read(buffer); } out.flush(); } // Return the repo-relative path as a result result.add(Paths.get(path, destFile.getName()).toString()); } } return result; } /** * Returns a unique file name in the given folder. * If the given file name is not unique, a new is constructed * by adding a number to the file name * @param folder the folder * @param name the file name * @return the new unique file */ private Path getUniqueFile(Path folder, String name) { Path file = folder.resolve(name); if (Files.exists(file)) { for (int x = 2; true; x++) { String fileName = FilenameUtils.removeExtension(name) + " " + x + "." + FilenameUtils.getExtension(name); file = folder.resolve(fileName); if (!Files.exists(file)) { break; } } } return file; } /** * Returns a new unique "temp" directory. Please note, the directory has not yet been created. * * @return a new unique "temp" directory */ @GET @javax.ws.rs.Path("/new-temp-dir") @Produces("application/json;charset=UTF-8") public RepoFileVo getNewTempDir() { // Construct a unique directory name String name = UUID.randomUUID().toString();// test "f823b7d5-9559-4a76-b3a3-6d32f2bf55f2"; RepoFileVo dir = new RepoFileVo(); dir.setName(name); dir.setPath(WebUtils.encodeURI("temp/" + name)); dir.setDirectory(true); return dir; } /** * Handles upload of files to the "temp" repo root. * Only the "user" role is required to upload to the "temp" root * * @param path the folder to upload to * @param request the request */ @POST @javax.ws.rs.Path("/upload-temp/{folder:.+}") @Consumes(MediaType.MULTIPART_FORM_DATA) @Produces("application/json;charset=UTF-8") @RolesAllowed(Roles.EDITOR) public List<String> uploadTempFile(@PathParam("folder") String path, @Context HttpServletRequest request) throws FileUploadException, IOException { // Check that the specified folder is indeed under the "temp" root validateTempRepoPath(path); return uploadFile(path, request); } /** * Resolves the relative repository path as a temporary repository path * and returns the full path to it. * @param path the path to resolve and validate as a temporary repository path * @return the full path */ public Path validateTempRepoPath(String path) { // Validate that the path is a temporary repository folder path Path folder = getRepoRoot().resolve(path); if (!folder.toAbsolutePath().startsWith(getTempRepoRoot().toAbsolutePath())) { log.warn("Failed streaming file to temp root folder: " + folder); throw new WebApplicationException("Invalid upload folder: " + path, 403); } return folder; } /** * Creates a new DiskFileItemFactory. See: * http://commons.apache.org/proper/commons-fileupload/using.html * @return the new DiskFileItemFactory */ private DiskFileItemFactory newDiskFileItemFactory(ServletContext servletContext) { File repository = (File) servletContext.getAttribute("javax.servlet.context.tempdir"); FileCleaningTracker fileCleaningTracker = FileCleanerCleanup.getFileCleaningTracker(servletContext); DiskFileItemFactory factory = new DiskFileItemFactory(DiskFileItemFactory.DEFAULT_SIZE_THRESHOLD, repository); factory.setFileCleaningTracker(fileCleaningTracker); return factory; } /** * Parses the file upload request and returns the file items * @param request the request * @return the file items */ public List<FileItem> parseFileUploadRequest(HttpServletRequest request) throws FileUploadException { FileItemFactory factory = newDiskFileItemFactory(servletContext); ServletFileUpload upload = new ServletFileUpload(factory); upload.setFileSizeMax(fileUploadMaxSize); upload.setSizeMax(fileUploadMaxSize); return upload.parseRequest(request); } /** * Every hour, check the repo "temp" root, and delete old files and folders */ @Schedule(persistent = false, second = "50", minute = "22", hour = "*") public void cleanUpTempRoot() { Calendar cal = Calendar.getInstance(); cal.add(Calendar.DATE, -1); File[] files = getTempRepoRoot().toFile().listFiles(); if (files != null && files.length > 0) { Arrays.asList(files).forEach(f -> checkDeletePath(f, cal.getTime())); } } /** * Recursively delete one day old files and folders * @param file the current root file or folder * @param date the expiry date */ private void checkDeletePath(File file, Date date) { if (FileUtils.isFileOlder(file, date)) { log.debug("Deleting expired temp file or folder: " + file); FileUtils.deleteQuietly(file); } else if (file.isDirectory()) { File[] files = file.listFiles(); if (files != null && files.length > 0) { Arrays.asList(files).forEach(f -> checkDeletePath(f, date)); } } } /** * Moves the repository from the repoPath the the newRepoPath. * If the directory specified by the repoPath does not exists, false is returned. * @param repoPath the repository path * @param newRepoPath the new repository path */ public boolean moveRepoFolder(String repoPath, String newRepoPath) throws IOException { Path from = getRepoRoot().resolve(repoPath); Path to = getRepoRoot().resolve(newRepoPath); if (Files.exists(from)) { FileUtils.copyDirectory(from.toFile(), to.toFile()); return true; } return false; } /** * Creates a temporary repository folder for the given repository-backed value object * @param vo the message * @param copyToTemp whether to copy all resources to the associated temporary directory or not */ public void createTempEditRepoFolder(IRepoBackedVo vo, boolean copyToTemp) throws IOException { String editRepoPath = getNewTempDir().getPath(); vo.setEditRepoPath(editRepoPath); if (copyToTemp) { // For existing messages, copy the existing message repo to the new repository if (StringUtils.isNotBlank(vo.getRepoPath())) { Path srcPath = getRepoRoot().resolve(vo.getRepoPath()); Path dstPath = getRepoRoot().resolve(editRepoPath); if (Files.exists(srcPath)) { log.info("Copy folder " + srcPath + " to temporary folder " + dstPath); FileUtils.copyDirectory(srcPath.toFile(), dstPath.toFile(), true); } } // Point any embedded links and images to the temporary repository folder vo.rewriteRepoPath(vo.getRepoPath(), vo.getEditRepoPath()); } } /** * Copy new files from the temporary edit-repo path to the actual repo folder associated with the value object * @param vo the value object to update */ public void updateRepoFolderFromTempEditFolder(IRepoBackedVo vo) throws IOException { if (vo != null && StringUtils.isNotBlank(vo.getRepoPath()) && StringUtils.isNotBlank(vo.getEditRepoPath())) { Path srcPath = getRepoRoot().resolve(vo.getEditRepoPath()); Path dstPath = getRepoRoot().resolve(vo.getRepoPath()); String revision = String.valueOf(vo.getRevision()); if (Files.exists(srcPath)) { // Case 1: If this is a new publication, copy the entire directory if (!Files.exists(dstPath)) { log.info("Syncing folder " + srcPath + " with " + dstPath); FileUtils.copyDirectory(srcPath.toFile(), dstPath.toFile(), true); // Case 2: Copy the latest revision sub-folder back to the source folder } else if (Files.exists(srcPath.resolve(revision))) { log.info("Syncing revision " + revision + " of folder " + srcPath + " with folder " + dstPath); FileUtils.copyDirectory(srcPath.resolve(revision).toFile(), dstPath.resolve(revision).toFile(), true); } else { log.info("No new revision files to sync"); } } } } }