Java tutorial
/** * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU Lesser General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. * * @author Arne Kepp / The Open Planning Project 2009 * */ package org.geowebcache.storage.blobstore.file; import static org.geowebcache.storage.blobstore.file.FilePathUtils.filteredGridSetId; import static org.geowebcache.storage.blobstore.file.FilePathUtils.filteredLayerName; import static org.geowebcache.storage.blobstore.file.FilePathUtils.findZoomLevel; import java.io.File; import java.io.FileFilter; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; import java.io.UnsupportedEncodingException; import java.net.URLDecoder; import java.net.URLEncoder; import java.nio.channels.FileChannel; import java.util.Properties; import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.geowebcache.config.ConfigurationException; import org.geowebcache.io.FileResource; import org.geowebcache.io.Resource; import org.geowebcache.mime.MimeException; import org.geowebcache.mime.MimeType; import org.geowebcache.storage.BlobStore; import org.geowebcache.storage.BlobStoreListener; import org.geowebcache.storage.BlobStoreListenerList; import org.geowebcache.storage.DefaultStorageFinder; import org.geowebcache.storage.StorageException; import org.geowebcache.storage.StorageObject.Status; import org.geowebcache.storage.TileObject; import org.geowebcache.storage.TileRange; import org.geowebcache.util.FileUtils; import org.springframework.scheduling.concurrent.CustomizableThreadFactory; import com.google.common.base.Preconditions; /** * See BlobStore interface description for details * */ public class FileBlobStore implements BlobStore { private static Log log = LogFactory.getLog(org.geowebcache.storage.blobstore.file.FileBlobStore.class); static final int DEFAULT_DISK_BLOCK_SIZE = 4096; public static final int BUFFER_SIZE = 32768; private final File stagingArea; private final String path; private int diskBlockSize = DEFAULT_DISK_BLOCK_SIZE; private final BlobStoreListenerList listeners = new BlobStoreListenerList(); private FilePathGenerator pathGenerator; private File tmp; private ExecutorService deleteExecutorService; public FileBlobStore(DefaultStorageFinder defStoreFinder) throws StorageException, ConfigurationException { this(defStoreFinder.getDefaultPath()); } public FileBlobStore(String rootPath) throws StorageException { this.path = rootPath; pathGenerator = new FilePathGenerator(this.path); // prepare the root File fh = new File(path); fh.mkdirs(); if (!fh.exists() || !fh.isDirectory() || !fh.canWrite()) { throw new StorageException(path + " is not writable directory."); } // and the temporary directory tmp = new File(path, "tmp"); tmp.mkdirs(); if (!tmp.exists() || !tmp.isDirectory() || !tmp.canWrite()) { throw new StorageException(tmp.getPath() + " is not writable directory."); } stagingArea = new File(path, "_gwc_in_progress_deletes_"); createDeleteExecutorService(); issuePendingDeletes(); } private void issuePendingDeletes() { if (!stagingArea.exists()) { return; } if (!stagingArea.isDirectory() || !stagingArea.canWrite()) { throw new IllegalStateException( "Staging area is not writable or is not a directory: " + stagingArea.getAbsolutePath()); } File[] pendings = stagingArea.listFiles(); for (File directory : pendings) { if (directory.isDirectory()) { deletePending(directory); } } } private void deletePending(final File pendingDeleteDirectory) { deleteExecutorService.submit(new DefferredDirectoryDeleteTask(pendingDeleteDirectory)); } private void createDeleteExecutorService() { CustomizableThreadFactory tf; tf = new CustomizableThreadFactory("GWC FileStore delete directory thread-"); tf.setDaemon(true); tf.setThreadPriority(Thread.MIN_PRIORITY); deleteExecutorService = Executors.newFixedThreadPool(1); } /** * Destroy method for Spring */ public void destroy() { deleteExecutorService.shutdownNow(); } private static class DefferredDirectoryDeleteTask implements Runnable { private final File directory; public DefferredDirectoryDeleteTask(final File directory) { this.directory = directory; } public void run() { try { deleteDirectory(directory); } catch (IOException e) { log.warn("Exception occurred while deleting '" + directory.getAbsolutePath() + "'", e); } catch (InterruptedException e) { log.info("FileStore delete background service interrupted while deleting '" + directory.getAbsolutePath() + "'. Process will be resumed at next start up"); } } private void deleteDirectory(File directory) throws IOException, InterruptedException { if (!directory.exists()) { return; } if (Thread.interrupted()) { throw new InterruptedException(); } File[] files = directory.listFiles(); for (int i = 0; i < files.length; i++) { if (Thread.interrupted()) { throw new InterruptedException(); } File file = files[i]; if (file.isDirectory()) { deleteDirectory(file); } else { if (!file.delete()) { throw new IOException("Unable to delete " + file.getAbsolutePath()); } } } if (!directory.delete()) { String message = "Unable to delete directory " + directory + "."; throw new IOException(message); } } } /** * @see org.geowebcache.storage.BlobStore#delete(java.lang.String) */ public boolean delete(final String layerName) throws StorageException { final File source = getLayerPath(layerName); final String target = filteredLayerName(layerName); boolean ret = stageDelete(source, target); this.listeners.sendLayerDeleted(layerName); return ret; } private boolean stageDelete(final File source, final String targetName) throws StorageException { if (!source.exists() || !source.canWrite()) { log.info(source + " does not exist or is not writable"); return false; } if (!stagingArea.exists() && !stagingArea.mkdirs()) { throw new StorageException( "Can't create staging directory for deletes: " + stagingArea.getAbsolutePath()); } File tmpFolder = new File(stagingArea, targetName); int tries = 0; while (tmpFolder.exists()) { ++tries; String dirName = filteredLayerName(targetName + "." + tries); tmpFolder = new File(stagingArea, dirName); } boolean renamed = FileUtils.renameFile(source, tmpFolder); if (!renamed) { throw new IllegalStateException("Can't rename " + source.getAbsolutePath() + " to " + tmpFolder.getAbsolutePath() + " for deletion"); } deletePending(tmpFolder); return true; } /** * @throws StorageException * @see org.geowebcache.storage.BlobStore#deleteByGridsetId(java.lang.String, java.lang.String) */ public boolean deleteByGridsetId(final String layerName, final String gridSetId) throws StorageException { final File layerPath = getLayerPath(layerName); if (!layerPath.exists() || !layerPath.canWrite()) { log.info(layerPath + " does not exist or is not writable"); return false; } final String filteredGridSetId = filteredGridSetId(gridSetId); File[] gridSubsetCaches = layerPath.listFiles(new FileFilter() { public boolean accept(File pathname) { if (!pathname.isDirectory()) { return false; } String dirName = pathname.getName(); return dirName.startsWith(filteredGridSetId); } }); for (File gridSubsetCache : gridSubsetCaches) { String target = filteredLayerName(layerName) + "_" + gridSubsetCache.getName(); stageDelete(gridSubsetCache, target); } listeners.sendGridSubsetDeleted(layerName, gridSetId); return true; } /** * Renames the layer directory for layer {@code oldLayerName} to {@code newLayerName} * * @return true if the directory for the layer was renamed, or the original directory didn't * exist in first place. {@code false} if the original directory exists but can't be * renamed to the target directory * @throws StorageException if the target directory already exists * @see org.geowebcache.storage.BlobStore#rename */ public boolean rename(final String oldLayerName, final String newLayerName) throws StorageException { final File oldLayerPath = getLayerPath(oldLayerName); final File newLayerPath = getLayerPath(newLayerName); if (newLayerPath.exists()) { throw new StorageException("Can't rename layer directory " + oldLayerPath + " to " + newLayerPath + ". Target directory already exists"); } if (!oldLayerPath.exists()) { this.listeners.sendLayerRenamed(oldLayerName, newLayerName); return true; } if (!oldLayerPath.canWrite()) { log.info(oldLayerPath + " is not writable"); return false; } boolean renamed = FileUtils.renameFile(oldLayerPath, newLayerPath); if (renamed) { this.listeners.sendLayerRenamed(oldLayerName, newLayerName); } else { throw new StorageException("Couldn't rename layer directory " + oldLayerPath + " to " + newLayerPath); } return renamed; } private File getLayerPath(String layerName) { String prefix = path + File.separator + filteredLayerName(layerName); File layerPath = new File(prefix); return layerPath; } /** * Delete a particular tile */ public boolean delete(TileObject stObj) throws StorageException { File fh = getFileHandleTile(stObj, false); boolean ret = false; // we call fh.length() here to check wthether the file exists and its length in a single // operation cause lots of calls to exists() may raise the file system cache usage to the // ceiling. File.length() returns 0 if the file does not exist anyway final long length = fh.length(); final boolean exists = length > 0; if (exists) { if (!fh.delete()) { throw new StorageException("Unable to delete " + fh.getAbsolutePath()); } stObj.setBlobSize((int) padSize(length)); listeners.sendTileDeleted(stObj); ret = true; } else { log.trace("delete unexistant file " + fh.toString()); } // Look at the parent directory to prune it if empty File parentDir = fh.getParentFile(); // Try deleting the directory (will not do it if the directory contains files) parentDir.delete(); return ret; } /** * Delete tiles within a range. */ public boolean delete(TileRange trObj) throws StorageException { int count = 0; String prefix = path + File.separator + filteredLayerName(trObj.getLayerName()); final File layerPath = new File(prefix); // If it wasn't there to be deleted, if (!layerPath.exists()) { return true; } // We either want to delete it, or stuff within it if (!layerPath.isDirectory() || !layerPath.canWrite()) { throw new StorageException(prefix + " does is not a directory or is not writable."); } final FilePathFilter tileFinder = new FilePathFilter(trObj); final String layerName = trObj.getLayerName(); final String gridSetId = trObj.getGridSetId(); final String blobFormat = trObj.getMimeType().getFormat(); final String parametersId = trObj.getParametersId(); File[] srsZoomDirs = layerPath.listFiles(tileFinder); final String gridsetPrefix = filteredGridSetId(gridSetId); for (File srsZoomParamId : srsZoomDirs) { int zoomLevel = findZoomLevel(gridsetPrefix, srsZoomParamId.getName()); File[] intermediates = srsZoomParamId.listFiles(tileFinder); for (File imd : intermediates) { File[] tiles = imd.listFiles(tileFinder); long length; for (File tile : tiles) { length = tile.length(); boolean deleted = tile.delete(); if (deleted) { String[] coords = tile.getName().split("\\.")[0].split("_"); long x = Long.parseLong(coords[0]); long y = Long.parseLong(coords[1]); listeners.sendTileDeleted(layerName, gridSetId, blobFormat, parametersId, x, y, zoomLevel, padSize(length)); count++; } } // Try deleting the directory (will be done only if the directory is empty) if (imd.delete()) { // listeners.sendDirectoryDeleted(layerName); } } // Try deleting the zoom directory (will be done only if the directory is empty) if (srsZoomParamId.delete()) { count++; // listeners.sendDirectoryDeleted(layerName); } } log.info("Truncated " + count + " tiles"); return true; } /** * Set the blob property of a TileObject. * * @param stObj the tile to load. Its setBlob() method will be called. * @return true if successful, false otherwise */ public boolean get(TileObject stObj) throws StorageException { File fh = getFileHandleTile(stObj, false); if (!fh.exists()) { stObj.setStatus(Status.MISS); return false; } else { Resource resource = readFile(fh); stObj.setBlob(resource); stObj.setCreated(resource.getLastModified()); stObj.setBlobSize((int) resource.getSize()); return true; } } /** * Store a tile. */ public void put(TileObject stObj) throws StorageException { final File fh = getFileHandleTile(stObj, true); final long oldSize = fh.length(); final boolean existed = oldSize > 0; writeFile(fh, stObj, existed); // mark the last modification as the tile creation time if set, otherwise // we'll leave it to the writing time if (stObj.getCreated() > 0) { try { fh.setLastModified(stObj.getCreated()); } catch (Exception e) { log.debug("Failed to set the last modified time to match the tile request time", e); } } /* * This is important because listeners may be tracking tile existence */ stObj.setBlobSize((int) padSize(stObj.getBlobSize())); if (existed) { listeners.sendTileUpdated(stObj, padSize(oldSize)); } else { listeners.sendTileStored(stObj); } } private File getFileHandleTile(TileObject stObj, boolean create) throws StorageException { final MimeType mimeType; try { mimeType = MimeType.createFromFormat(stObj.getBlobFormat()); } catch (MimeException me) { log.error(me.getMessage()); throw new RuntimeException(me); } final File tilePath = pathGenerator.tilePath(stObj, mimeType); if (create) { File parent = tilePath.getParentFile(); mkdirs(parent, stObj); } return tilePath; } private Resource readFile(File fh) throws StorageException { if (!fh.exists()) { return null; } return new FileResource(fh); } private void writeFile(File target, TileObject stObj, boolean existed) throws StorageException { // first write to temp file tmp.mkdirs(); File temp = new File(tmp, UUID.randomUUID().toString()); try { // Open the output stream and read the blob into the tile FileOutputStream fos = null; FileChannel channel = null; try { fos = new FileOutputStream(temp); channel = fos.getChannel(); try { stObj.getBlob().transferTo(channel); } catch (IOException ioe) { throw new StorageException(ioe.getMessage() + " for " + target.getAbsolutePath()); } finally { try { if (channel != null) { channel.close(); } } catch (IOException ioe) { throw new StorageException(ioe.getMessage() + " for " + target.getAbsolutePath()); } } } catch (FileNotFoundException ioe) { throw new StorageException(ioe.getMessage() + " for " + target.getAbsolutePath()); } finally { IOUtils.closeQuietly(fos); } // rename to final position. This will fail if another GWC also wrote this // file, in such case we'll just eliminate this one if (FileUtils.renameFile(temp, target)) { temp = null; } else if (existed) { // if we are trying to overwrite and old tile, on windows that might fail... delete // and rename instead if (target.delete() && FileUtils.renameFile(temp, target)) { temp = null; } } } finally { if (temp != null) { log.warn("Tile " + target.getPath() + " was already written by another thread/process"); temp.delete(); } } } public void clear() throws StorageException { throw new StorageException("Not implemented yet!"); } /** * Add an event listener */ public void addListener(BlobStoreListener listener) { listeners.addListener(listener); } /** * Remove an event listener */ public boolean removeListener(BlobStoreListener listener) { return listeners.removeListener(listener); } /** * This method will recursively create the missing directories and call the listeners * directoryCreated method for each created directory. * * @param path * @return */ private boolean mkdirs(File path, TileObject stObj) { /* If the terminal directory already exists, answer false */ if (path.exists()) { return false; } /* If the receiver can be created, answer true */ if (path.mkdir()) { // listeners.sendDirectoryCreated(stObj); return true; } String parentDir = path.getParent(); /* If there is no parent and we were not created, answer false */ if (parentDir == null) { return false; } /* Otherwise, try to create a parent directory and then this directory */ mkdirs(new File(parentDir), stObj); if (path.mkdir()) { // listeners.sendDirectoryCreated(stObj); return true; } return false; } /** * @see org.geowebcache.storage.BlobStore#getLayerMetadata(java.lang.String, java.lang.String) */ public String getLayerMetadata(final String layerName, final String key) { Properties metadata = getLayerMetadata(layerName); String value = metadata.getProperty(key); if (value != null) { try { value = URLDecoder.decode(value, "UTF-8"); } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); } } return value; } /** * @see org.geowebcache.storage.BlobStore#putLayerMetadata(java.lang.String, java.lang.String, * java.lang.String) */ public void putLayerMetadata(final String layerName, final String key, final String value) { Properties metadata = getLayerMetadata(layerName); if (null == value) { metadata.remove(key); } else { try { metadata.setProperty(key, URLEncoder.encode(value, "UTF-8")); } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); } } final File metadataFile = getMetadataFile(layerName); final String lockObj = metadataFile.getAbsolutePath().intern(); synchronized (lockObj) { OutputStream out; try { if (!metadataFile.getParentFile().exists()) { metadataFile.getParentFile().mkdirs(); } out = new FileOutputStream(metadataFile); } catch (FileNotFoundException e) { throw new RuntimeException(e); } try { String comments = "auto generated file, do not edit by hand"; metadata.store(out, comments); } catch (Exception e) { throw new RuntimeException(e); } finally { try { out.close(); } catch (IOException e) { log.warn(e.getMessage(), e); } } } } private Properties getLayerMetadata(final String layerName) { final File metadataFile = getMetadataFile(layerName); Properties properties = new Properties(); final String lockObj = metadataFile.getAbsolutePath().intern(); synchronized (lockObj) { if (metadataFile.exists()) { FileInputStream in; try { in = new FileInputStream(metadataFile); } catch (FileNotFoundException e) { throw new RuntimeException(e); } try { properties.load(in); } catch (Exception e) { throw new RuntimeException(e); } finally { try { in.close(); } catch (IOException e) { log.warn(e.getMessage(), e); } } } } return properties; } private File getMetadataFile(final String layerName) { File layerPath = getLayerPath(layerName); File metadataFile = new File(layerPath, "metadata.properties"); return metadataFile; } @Override public boolean layerExists(String layerName) { return getLayerPath(layerName).exists(); } /** * Specify the file system block size, used to pad out tile lenghts to whole blocks when * reporting {@link BlobStoreListener#tileDeleted tileDeleted}, * {@link BlobStoreListener#tileStored tileStored}, or {@link BlobStoreListener#tileUpdated * tileUpdated} events. * * @param fileSystemBlockSize the size of a filesystem block; must be a positive integer, * usually a power of 2 greater or equal to 512. */ public void setBlockSize(int fileSystemBlockSize) { Preconditions.checkArgument(fileSystemBlockSize > 0); this.diskBlockSize = fileSystemBlockSize; } /** * Pads the size of a tile to whole filesystem blocks * * @param fileSize the size of the tile file as reported by {@link File#length()} * @return {@code fileSize} padded to whole blocks as per {@link #diskBlockSize} */ private long padSize(long fileSize) { final int blockSize = this.diskBlockSize; long actuallyUsedStorage = blockSize * (int) Math.ceil((double) fileSize / blockSize); return actuallyUsedStorage; } }