Java tutorial
/** * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU Lesser General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. * * @author Gabriel Roldan, Boundless Spatial Inc, Copyright 2015 */ package org.geowebcache.s3; import static com.google.common.base.Preconditions.checkNotNull; import static java.util.Objects.isNull; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.channels.Channels; import java.nio.channels.WritableByteChannel; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Properties; import java.util.Set; import java.util.stream.Collectors; import javax.annotation.Nullable; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.geowebcache.GeoWebCacheException; import org.geowebcache.filter.parameters.ParametersUtils; import org.geowebcache.io.ByteArrayResource; import org.geowebcache.io.Resource; import org.geowebcache.layer.TileLayerDispatcher; import org.geowebcache.locks.LockProvider; import org.geowebcache.mime.MimeException; import org.geowebcache.mime.MimeType; import org.geowebcache.storage.BlobStore; import org.geowebcache.storage.BlobStoreListener; import org.geowebcache.storage.BlobStoreListenerList; import org.geowebcache.storage.StorageException; import org.geowebcache.storage.TileObject; import org.geowebcache.storage.TileRange; import org.geowebcache.storage.TileRangeIterator; import com.amazonaws.AmazonClientException; import com.amazonaws.AmazonServiceException; import com.amazonaws.services.s3.AmazonS3Client; import com.amazonaws.services.s3.model.AccessControlList; import com.amazonaws.services.s3.model.CannedAccessControlList; import com.amazonaws.services.s3.model.DeleteObjectsRequest; import com.amazonaws.services.s3.model.DeleteObjectsRequest.KeyVersion; import com.amazonaws.services.s3.model.Grant; import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.PutObjectRequest; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectInputStream; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.google.common.base.Function; import com.google.common.base.Throwables; import com.google.common.collect.AbstractIterator; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; import com.google.common.io.ByteStreams; public class S3BlobStore implements BlobStore { static Log log = LogFactory.getLog(S3BlobStore.class); private final BlobStoreListenerList listeners = new BlobStoreListenerList(); private AmazonS3Client conn; private final TMSKeyBuilder keyBuilder; private String bucketName; private volatile boolean shutDown; private final S3Ops s3Ops; public S3BlobStore(S3BlobStoreConfig config, TileLayerDispatcher layers, LockProvider lockProvider) throws StorageException { checkNotNull(config); checkNotNull(layers); checkNotNull(config.getAwsAccessKey(), "Access key not provided"); checkNotNull(config.getAwsSecretKey(), "Secret key not provided"); this.bucketName = config.getBucket(); String prefix = config.getPrefix() == null ? "" : config.getPrefix(); this.keyBuilder = new TMSKeyBuilder(prefix, layers); conn = config.buildClient(); try { log.debug("Checking access rights to bucket " + bucketName); AccessControlList bucketAcl = this.conn.getBucketAcl(bucketName); List<Grant> grants = bucketAcl.getGrantsAsList(); log.debug("Bucket " + bucketName + " permissions: " + grants); } catch (AmazonServiceException se) { throw new StorageException("Server error listing buckets: " + se.getMessage(), se); } catch (AmazonClientException ce) { throw new StorageException("Unable to connect to AWS S3", ce); } this.s3Ops = new S3Ops(conn, bucketName, keyBuilder, lockProvider); } @Override public void destroy() { this.shutDown = true; AmazonS3Client conn = this.conn; this.conn = null; if (conn != null) { s3Ops.shutDown(); conn.shutdown(); } } @Override public void addListener(BlobStoreListener listener) { listeners.addListener(listener); } @Override public boolean removeListener(BlobStoreListener listener) { return listeners.removeListener(listener); } @Override public void put(TileObject obj) throws StorageException { final Resource blob = obj.getBlob(); checkNotNull(blob); checkNotNull(obj.getBlobFormat()); final String key = keyBuilder.forTile(obj); ObjectMetadata objectMetadata = new ObjectMetadata(); objectMetadata.setContentLength(blob.getSize()); String blobFormat = obj.getBlobFormat(); String mimeType; try { mimeType = MimeType.createFromFormat(blobFormat).getMimeType(); } catch (MimeException me) { throw Throwables.propagate(me); } objectMetadata.setContentType(mimeType); // don't bother for the extra call if there are no listeners final boolean existed; ObjectMetadata oldObj; if (listeners.isEmpty()) { existed = false; oldObj = null; } else { oldObj = s3Ops.getObjectMetadata(key); existed = oldObj != null; } final ByteArrayInputStream input = toByteArray(blob); PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, key, input, objectMetadata) .withCannedAcl(CannedAccessControlList.PublicRead); log.trace(log.isTraceEnabled() ? ("Storing " + key) : ""); s3Ops.putObject(putObjectRequest); putParametersMetadata(obj.getLayerName(), obj.getParametersId(), obj.getParameters()); /* * This is important because listeners may be tracking tile existence */ if (!listeners.isEmpty()) { if (existed) { long oldSize = oldObj.getContentLength(); listeners.sendTileUpdated(obj, oldSize); } else { listeners.sendTileStored(obj); } } } private ByteArrayInputStream toByteArray(final Resource blob) throws StorageException { final byte[] bytes; if (blob instanceof ByteArrayResource) { bytes = ((ByteArrayResource) blob).getContents(); } else { ByteArrayOutputStream out = new ByteArrayOutputStream((int) blob.getSize()); WritableByteChannel channel = Channels.newChannel(out); try { blob.transferTo(channel); } catch (IOException e) { throw new StorageException("Error copying blob contents", e); } bytes = out.toByteArray(); } ByteArrayInputStream input = new ByteArrayInputStream(bytes); return input; } @Override public boolean get(TileObject obj) throws StorageException { final String key = keyBuilder.forTile(obj); final S3Object object = s3Ops.getObject(key); if (object == null) { return false; } try (S3ObjectInputStream in = object.getObjectContent()) { byte[] bytes = ByteStreams.toByteArray(in); obj.setBlobSize(bytes.length); obj.setBlob(new ByteArrayResource(bytes)); obj.setCreated(object.getObjectMetadata().getLastModified().getTime()); } catch (IOException e) { throw new StorageException("Error getting " + key, e); } return true; } private class TileToKey implements Function<long[], KeyVersion> { private final String coordsPrefix; private final String extension; public TileToKey(String coordsPrefix, MimeType mimeType) { this.coordsPrefix = coordsPrefix; this.extension = mimeType.getInternalName(); } @Override public KeyVersion apply(long[] loc) { long z = loc[2]; long x = loc[0]; long y = loc[1]; StringBuilder sb = new StringBuilder(coordsPrefix); sb.append(z).append('/').append(x).append('/').append(y).append('.').append(extension); return new KeyVersion(sb.toString()); } } @Override public boolean delete(final TileRange tileRange) throws StorageException { final String coordsPrefix = keyBuilder.coordinatesPrefix(tileRange); if (!s3Ops.prefixExists(coordsPrefix)) { return false; } final Iterator<long[]> tileLocations = new AbstractIterator<long[]>() { // TileRange iterator with 1x1 meta tiling factor private TileRangeIterator trIter = new TileRangeIterator(tileRange, new int[] { 1, 1 }); @Override protected long[] computeNext() { long[] gridLoc = trIter.nextMetaGridLocation(new long[3]); return gridLoc == null ? endOfData() : gridLoc; } }; if (listeners.isEmpty()) { // if there are no listeners, don't bother requesting every tile // metadata to notify the listeners Iterator<List<long[]>> partition = Iterators.partition(tileLocations, 1000); final TileToKey tileToKey = new TileToKey(coordsPrefix, tileRange.getMimeType()); while (partition.hasNext() && !shutDown) { List<long[]> locations = partition.next(); List<KeyVersion> keys = Lists.transform(locations, tileToKey); DeleteObjectsRequest req = new DeleteObjectsRequest(bucketName); req.setQuiet(true); req.setKeys(keys); conn.deleteObjects(req); } } else { long[] xyz; String layerName = tileRange.getLayerName(); String gridSetId = tileRange.getGridSetId(); String format = tileRange.getMimeType().getFormat(); Map<String, String> parameters = tileRange.getParameters(); while (tileLocations.hasNext()) { xyz = tileLocations.next(); TileObject tile = TileObject.createQueryTileObject(layerName, xyz, gridSetId, format, parameters); tile.setParametersId(tileRange.getParametersId()); delete(tile); } } return true; } @Override public boolean delete(String layerName) throws StorageException { checkNotNull(layerName, "layerName"); final String metadataKey = keyBuilder.layerMetadata(layerName); final String layerPrefix = keyBuilder.forLayer(layerName); s3Ops.deleteObject(metadataKey); boolean layerExists; try { layerExists = s3Ops.scheduleAsyncDelete(layerPrefix); } catch (GeoWebCacheException e) { throw Throwables.propagate(e); } if (layerExists) { listeners.sendLayerDeleted(layerName); } return layerExists; } @Override public boolean deleteByGridsetId(final String layerName, final String gridSetId) throws StorageException { checkNotNull(layerName, "layerName"); checkNotNull(gridSetId, "gridSetId"); final String gridsetPrefix = keyBuilder.forGridset(layerName, gridSetId); boolean prefixExists; try { prefixExists = s3Ops.scheduleAsyncDelete(gridsetPrefix); } catch (GeoWebCacheException e) { throw Throwables.propagate(e); } if (prefixExists) { listeners.sendGridSubsetDeleted(layerName, gridSetId); } return prefixExists; } @Override public boolean delete(TileObject obj) throws StorageException { final String key = keyBuilder.forTile(obj); // don't bother for the extra call if there are no listeners if (listeners.isEmpty()) { return s3Ops.deleteObject(key); } ObjectMetadata oldObj = s3Ops.getObjectMetadata(key); if (oldObj == null) { return false; } s3Ops.deleteObject(key); obj.setBlobSize((int) oldObj.getContentLength()); listeners.sendTileDeleted(obj); return true; } @Override public boolean rename(String oldLayerName, String newLayerName) throws StorageException { log.debug("No need to rename layers, S3BlobStore uses layer id as key root"); if (s3Ops.prefixExists(oldLayerName)) { listeners.sendLayerRenamed(oldLayerName, newLayerName); } return true; } @Override public void clear() throws StorageException { throw new UnsupportedOperationException("clear() should not be called"); } @Nullable @Override public String getLayerMetadata(String layerName, String key) { Properties properties = getLayerMetadata(layerName); String value = properties.getProperty(key); return value; } @Override public void putLayerMetadata(String layerName, String key, String value) { Properties properties = getLayerMetadata(layerName); properties.setProperty(key, value); String resourceKey = keyBuilder.layerMetadata(layerName); try { s3Ops.putProperties(resourceKey, properties); } catch (StorageException e) { Throwables.propagate(e); } } private Properties getLayerMetadata(String layerName) { String key = keyBuilder.layerMetadata(layerName); return s3Ops.getProperties(key); } private void putParametersMetadata(String layerName, String parametersId, Map<String, String> parameters) { assert (isNull(parametersId) == isNull(parameters)); if (isNull(parametersId)) { return; } Properties properties = new Properties(); parameters.forEach(properties::setProperty); String resourceKey = keyBuilder.parametersMetadata(layerName, parametersId); try { s3Ops.putProperties(resourceKey, properties); } catch (StorageException e) { Throwables.propagate(e); } } @Override public boolean layerExists(String layerName) { final String coordsPrefix = keyBuilder.forLayer(layerName); boolean layerExists = s3Ops.prefixExists(coordsPrefix); return layerExists; } @Override public boolean deleteByParametersId(String layerName, String parametersId) throws StorageException { checkNotNull(layerName, "layerName"); checkNotNull(parametersId, "parametersId"); boolean prefixExists = keyBuilder.forParameters(layerName, parametersId).stream().map(prefix -> { try { return s3Ops.scheduleAsyncDelete(prefix); } catch (RuntimeException | GeoWebCacheException e) { throw Throwables.propagate(e); } }).reduce(Boolean::logicalOr) // Don't use Stream.anyMatch as it would short circuit .orElse(false); if (prefixExists) { listeners.sendParametersDeleted(layerName, parametersId); } return prefixExists; } @SuppressWarnings("unchecked") @Override public Set<Map<String, String>> getParameters(String layerName) { return s3Ops.objectStream(keyBuilder.parametersMetadataPrefix(layerName)).map(S3ObjectSummary::getKey) .map(s3Ops::getProperties).map(props -> (Map<String, String>) (Map<?, ?>) props) .collect(Collectors.toSet()); } @SuppressWarnings("unchecked") public Map<String, Optional<Map<String, String>>> getParametersMapping(String layerName) { return s3Ops.objectStream(keyBuilder.parametersMetadataPrefix(layerName)).map(S3ObjectSummary::getKey) .map(s3Ops::getProperties).map(props -> (Map<String, String>) (Map<?, ?>) props) .collect(Collectors.toMap(ParametersUtils::getId, Optional::of)); } }