Java tutorial
/* * (C) Copyright 2006-2012 Nuxeo SA (http://nuxeo.com/) and others. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * * Contributors: * Florent Guillaume, Mathieu Guillaume, jcarsique */ package org.nuxeo.ecm.core.blob.binary; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.RandomAccessFile; import java.util.Map; import java.util.regex.Pattern; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuxeo.common.Environment; import org.nuxeo.ecm.core.api.NuxeoException; import org.nuxeo.runtime.api.Framework; import org.nuxeo.runtime.trackers.files.FileEventTracker; /** * A simple filesystem-based binary manager. It stores the binaries according to their digest (hash), which means that * no transactional behavior needs to be implemented. * <p> * A garbage collection is needed to purge unused binaries. * <p> * The format of the <em>binaries</em> directory is: * <ul> * <li><em>data/</em> hierarchy with the actual binaries in subdirectories,</li> * <li><em>tmp/</em> temporary storage during creation,</li> * <li><em>config.xml</em> a file containing the configuration used.</li> * </ul> * * @author Florent Guillaume * @since 5.6 */ public class LocalBinaryManager extends AbstractBinaryManager { private static final Log log = LogFactory.getLog(LocalBinaryManager.class); public static final Pattern WINDOWS_ABSOLUTE_PATH = Pattern.compile("[a-zA-Z]:[/\\\\].*"); public static final String DEFAULT_PATH = "binaries"; public static final String DATA = "data"; public static final String TMP = "tmp"; public static final String CONFIG_FILE = "config.xml"; protected File storageDir; protected File tmpDir; @Override public void initialize(String blobProviderId, Map<String, String> properties) throws IOException { super.initialize(blobProviderId, properties); String path = properties.get(BinaryManager.PROP_PATH); if (StringUtils.isBlank(path)) { path = DEFAULT_PATH; } path = Framework.expandVars(path); path = path.trim(); File base; if (path.startsWith("/") || path.startsWith("\\") || path.contains("://") || path.contains(":\\") || WINDOWS_ABSOLUTE_PATH.matcher(path).matches()) { // absolute base = new File(path); } else { // relative File home = Environment.getDefault().getData(); base = new File(home, path); // Backward compliance with versions before 5.4 (NXP-5370) File oldBase = new File(Framework.getRuntime().getHome().getPath(), path); if (oldBase.exists()) { log.warn("Old binaries path used (NXP-5370). Please move " + oldBase + " to " + base); base = oldBase; } } // be sure FileTracker won't steal our files ! FileEventTracker.registerProtectedPath(base.getAbsolutePath()); log.info("Registering binary manager '" + blobProviderId + "' using " + (this.getClass().equals(LocalBinaryManager.class) ? "" : (this.getClass().getSimpleName() + " and ")) + "binary store: " + base); storageDir = new File(base, DATA); tmpDir = new File(base, TMP); storageDir.mkdirs(); tmpDir.mkdirs(); descriptor = getDescriptor(new File(base, CONFIG_FILE)); createGarbageCollector(); } @Override public void close() { if (tmpDir != null) { try { FileUtils.cleanDirectory(tmpDir); } catch (IOException e) { throw new NuxeoException(e); } } } public File getStorageDir() { return storageDir; } @Override protected Binary getBinary(InputStream in) throws IOException { String digest = storeAndDigest(in); File file = getFileForDigest(digest, false); /* * Now we can build the Binary. */ return new Binary(file, digest, blobProviderId); } @Override public Binary getBinary(String digest) { File file = getFileForDigest(digest, false); if (file == null) { // invalid digest return null; } if (!file.exists()) { log.warn("cannot fetch content at " + file.getPath() + " (file does not exist), check your configuration"); return null; } return new Binary(file, digest, blobProviderId); } /** * Gets a file representing the storage for a given digest. * * @param digest the digest * @param createDir {@code true} if the directory containing the file itself must be created * @return the file for this digest */ public File getFileForDigest(String digest, boolean createDir) { int depth = descriptor.depth; if (digest.length() < 2 * depth) { return null; } StringBuilder buf = new StringBuilder(3 * depth - 1); for (int i = 0; i < depth; i++) { if (i != 0) { buf.append(File.separatorChar); } buf.append(digest.substring(2 * i, 2 * i + 2)); } File dir = new File(storageDir, buf.toString()); if (createDir) { dir.mkdirs(); } return new File(dir, digest); } protected String storeAndDigest(InputStream in) throws IOException { File tmp = File.createTempFile("create_", ".tmp", tmpDir); OutputStream out = new FileOutputStream(tmp); /* * First, write the input stream to a temporary file, while computing a digest. */ try { String digest; try { digest = storeAndDigest(in, out); } finally { in.close(); out.close(); } /* * Move the tmp file to its destination. */ File file = getFileForDigest(digest, true); atomicMove(tmp, file); return digest; } finally { tmp.delete(); } } /** * Does an atomic move of the tmp (or source) file to the final file. * <p> * Tries to work well with NFS mounts and different filesystems. */ protected void atomicMove(File source, File dest) throws IOException { if (dest.exists()) { // The file with the proper digest is already there so don't do // anything. This is to avoid "Stale NFS File Handle" problems // which would occur if we tried to overwrite it anyway. // Note that this doesn't try to protect from the case where // two identical files are uploaded at the same time. // Update date for the GC. dest.setLastModified(source.lastModified()); return; } if (!source.renameTo(dest)) { // Failed to rename, probably a different filesystem. // Do *NOT* use Apache Commons IO's FileUtils.moveFile() // because it rewrites the destination file so is not atomic. // Do a copy through a tmp file on the same filesystem then // atomic rename. File tmp = File.createTempFile(dest.getName(), ".tmp", dest.getParentFile()); try { try (InputStream in = new FileInputStream(source); // OutputStream out = new FileOutputStream(tmp)) { IOUtils.copy(in, out); } // then do the atomic rename tmp.renameTo(dest); } finally { tmp.delete(); } // finally remove the original source source.delete(); } if (!dest.exists()) { throw new IOException("Could not create file: " + dest); } } protected void createGarbageCollector() { garbageCollector = new DefaultBinaryGarbageCollector(this); } public static class DefaultBinaryGarbageCollector implements BinaryGarbageCollector { /** * Windows FAT filesystems have a time resolution of 2s. Other common filesystems have 1s. */ public static int TIME_RESOLUTION = 2000; protected final LocalBinaryManager binaryManager; protected volatile long startTime; protected BinaryManagerStatus status; public DefaultBinaryGarbageCollector(LocalBinaryManager binaryManager) { this.binaryManager = binaryManager; } @Override public String getId() { return binaryManager.getStorageDir().toURI().toString(); } @Override public BinaryManagerStatus getStatus() { return status; } @Override public boolean isInProgress() { // volatile as this is designed to be called from another thread return startTime != 0; } @Override public void start() { if (startTime != 0) { throw new RuntimeException("Alread started"); } startTime = System.currentTimeMillis(); status = new BinaryManagerStatus(); } @Override public void mark(String digest) { File file = binaryManager.getFileForDigest(digest, false); if (!file.exists()) { log.error("Unknown file digest: " + digest); return; } touch(file); } @Override public void stop(boolean delete) { if (startTime == 0) { throw new RuntimeException("Not started"); } deleteOld(binaryManager.getStorageDir(), startTime - TIME_RESOLUTION, 0, delete); status.gcDuration = System.currentTimeMillis() - startTime; startTime = 0; } protected void deleteOld(File file, long minTime, int depth, boolean delete) { if (file.isDirectory()) { for (File f : file.listFiles()) { deleteOld(f, minTime, depth + 1, delete); } if (depth > 0 && file.list().length == 0) { // empty directory file.delete(); } } else if (file.isFile() && file.canWrite()) { long lastModified = file.lastModified(); long length = file.length(); if (lastModified == 0) { log.error("Cannot read last modified for file: " + file); } else if (lastModified < minTime) { status.sizeBinariesGC += length; status.numBinariesGC++; if (delete && !file.delete()) { log.warn("Cannot gc file: " + file); } } else { status.sizeBinaries += length; status.numBinaries++; } } } } /** * Sets the last modification date to now on a file * * @param file the file */ public static void touch(File file) { long time = System.currentTimeMillis(); if (file.setLastModified(time)) { // ok return; } if (!file.canWrite()) { // cannot write -> stop won't be able to delete anyway return; } try { // Windows: the file may be open for reading // workaround found by Thomas Mueller, see JCR-2872 RandomAccessFile r = new RandomAccessFile(file, "rw"); try { r.setLength(r.length()); } finally { r.close(); } } catch (IOException e) { log.error("Cannot set last modified for file: " + file, e); } } }