Java tutorial
/** * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU Lesser General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. * * @author Gabriel Roldan, Boundless Spatial Inc, Copyright 2015 */ package org.geowebcache.s3; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Properties; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ThreadFactory; import java.util.concurrent.atomic.AtomicInteger; import javax.annotation.Nullable; import org.apache.commons.io.IOUtils; import org.geowebcache.GeoWebCacheException; import org.geowebcache.locks.LockProvider; import org.geowebcache.locks.LockProvider.Lock; import org.geowebcache.locks.NoOpLockProvider; import org.geowebcache.storage.StorageException; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3Client; import com.amazonaws.services.s3.iterable.S3Objects; import com.amazonaws.services.s3.model.AmazonS3Exception; import com.amazonaws.services.s3.model.DeleteObjectsRequest; import com.amazonaws.services.s3.model.DeleteObjectsRequest.KeyVersion; import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.PutObjectRequest; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectInputStream; import com.amazonaws.services.s3.model.S3ObjectSummary; import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.stream.StreamSupport; import com.google.common.base.Throwables; import com.google.common.util.concurrent.ThreadFactoryBuilder; class S3Ops { private final AmazonS3Client conn; private final String bucketName; private final TMSKeyBuilder keyBuilder; private final LockProvider locks; private ExecutorService deleteExecutorService; private Map<String, Long> pendingDeletesKeyTime = new ConcurrentHashMap<>(); public S3Ops(AmazonS3Client conn, String bucketName, TMSKeyBuilder keyBuilder, LockProvider locks) throws StorageException { this.conn = conn; this.bucketName = bucketName; this.keyBuilder = keyBuilder; this.locks = locks == null ? new NoOpLockProvider() : locks; this.deleteExecutorService = createDeleteExecutorService(); issuePendingBulkDeletes(); } private ExecutorService createDeleteExecutorService() { ThreadFactory tf = new ThreadFactoryBuilder().setDaemon(true) .setNameFormat("GWC S3BlobStore bulk delete thread-%d. Bucket: " + bucketName) .setPriority(Thread.MIN_PRIORITY).build(); return Executors.newCachedThreadPool(tf); } public void shutDown() { deleteExecutorService.shutdownNow(); } private void issuePendingBulkDeletes() throws StorageException { final String pendingDeletesKey = keyBuilder.pendingDeletes(); Lock lock; try { lock = locks.getLock(pendingDeletesKey); } catch (GeoWebCacheException e) { throw new StorageException("Unable to lock pending deletes", e); } try { Properties deletes = getProperties(pendingDeletesKey); for (Entry<Object, Object> e : deletes.entrySet()) { final String prefix = e.getKey().toString(); final long timestamp = Long.parseLong(e.getValue().toString()); S3BlobStore.log.info(String.format("Restarting pending bulk delete on '%s/%s':%d", bucketName, prefix, timestamp)); asyncDelete(prefix, timestamp); } } finally { try { lock.release(); } catch (GeoWebCacheException e) { throw new StorageException("Unable to unlock pending deletes", e); } } } private void clearPendingBulkDelete(final String prefix, final long timestamp) throws GeoWebCacheException { Long taskTime = pendingDeletesKeyTime.get(prefix); if (taskTime == null) { return; // someone else cleared it up for us. A task that run after this one but // finished before? } if (taskTime.longValue() > timestamp) { return;// someone else issued a bulk delete after this one for the same key prefix } final String pendingDeletesKey = keyBuilder.pendingDeletes(); final Lock lock = locks.getLock(pendingDeletesKey); try { Properties deletes = getProperties(pendingDeletesKey); String storedVal = (String) deletes.remove(prefix); long storedTimestamp = storedVal == null ? Long.MIN_VALUE : Long.parseLong(storedVal); if (timestamp >= storedTimestamp) { putProperties(pendingDeletesKey, deletes); } else { S3BlobStore.log.info( String.format("bulk delete finished but there's a newer one ongoing for bucket '%s/%s'", bucketName, prefix)); } } catch (StorageException e) { Throwables.propagate(e); } finally { lock.release(); } } public boolean scheduleAsyncDelete(final String prefix) throws GeoWebCacheException { final long timestamp = currentTimeSeconds(); String msg = String.format("Issuing bulk delete on '%s/%s' for objects older than %d", bucketName, prefix, timestamp); S3BlobStore.log.info(msg); Lock lock = locks.getLock(prefix); try { boolean taskRuns = asyncDelete(prefix, timestamp); if (taskRuns) { final String pendingDeletesKey = keyBuilder.pendingDeletes(); Properties deletes = getProperties(pendingDeletesKey); deletes.setProperty(prefix, String.valueOf(timestamp)); putProperties(pendingDeletesKey, deletes); } return taskRuns; } catch (StorageException e) { throw Throwables.propagate(e); } finally { lock.release(); } } // S3 truncates timestamps to seconds precision and does not allow to programmatically set // the last modified time private long currentTimeSeconds() { final long timestamp = (long) Math.ceil(System.currentTimeMillis() / 1000D) * 1000L; return timestamp; } private synchronized boolean asyncDelete(final String prefix, final long timestamp) { if (!prefixExists(prefix)) { return false; } Long currentTaskTime = pendingDeletesKeyTime.get(prefix); if (currentTaskTime != null && currentTaskTime.longValue() > timestamp) { return false; } BulkDelete task = new BulkDelete(conn, bucketName, prefix, timestamp); deleteExecutorService.submit(task); pendingDeletesKeyTime.put(prefix, timestamp); return true; } @Nullable public ObjectMetadata getObjectMetadata(String key) throws StorageException { ObjectMetadata obj = null; try { obj = conn.getObjectMetadata(bucketName, key); } catch (AmazonS3Exception e) { if (404 != e.getStatusCode()) {// 404 == not found throw new StorageException("Error checking existence of " + key + ": " + e.getMessage(), e); } } return obj; } public void putObject(PutObjectRequest putObjectRequest) throws StorageException { try { conn.putObject(putObjectRequest); } catch (RuntimeException e) { throw new StorageException("Error storing " + putObjectRequest.getKey(), e); } } @Nullable public S3Object getObject(String key) throws StorageException { final S3Object object; try { object = conn.getObject(bucketName, key); } catch (AmazonS3Exception e) { if (404 == e.getStatusCode()) {// 404 == not found return null; } throw new StorageException("Error fetching " + key + ": " + e.getMessage(), e); } if (isPendingDelete(object)) { return null; } return object; } public boolean deleteObject(final String key) { try { conn.deleteObject(bucketName, key); } catch (AmazonS3Exception e) { return false; } return true; } private boolean isPendingDelete(S3Object object) { if (pendingDeletesKeyTime.isEmpty()) { return false; } final String key = object.getKey(); final long lastModified = object.getObjectMetadata().getLastModified().getTime(); for (Map.Entry<String, Long> e : pendingDeletesKeyTime.entrySet()) { String parentKey = e.getKey(); if (key.startsWith(parentKey)) { long deleteTime = e.getValue().longValue(); return deleteTime >= lastModified; } } return false; } @Nullable public byte[] getBytes(String key) throws StorageException { S3Object object = getObject(key); if (object == null) { return null; } try (S3ObjectInputStream in = object.getObjectContent()) { byte[] bytes = IOUtils.toByteArray(in); return bytes; } catch (IOException e) { throw new StorageException("Error getting " + key, e); } } /** * Simply checks if there are objects starting with {@code prefix} */ public boolean prefixExists(String prefix) { boolean hasNext = S3Objects.withPrefix(conn, bucketName, prefix).withBatchSize(1).iterator().hasNext(); return hasNext; } public Properties getProperties(String key) { Properties properties = new Properties(); byte[] bytes; try { bytes = getBytes(key); } catch (StorageException e) { throw Throwables.propagate(e); } if (bytes != null) { try { properties.load(new InputStreamReader(new ByteArrayInputStream(bytes), StandardCharsets.UTF_8)); } catch (IOException e) { throw Throwables.propagate(e); } } return properties; } public void putProperties(String resourceKey, Properties properties) throws StorageException { ByteArrayOutputStream out = new ByteArrayOutputStream(); try { properties.store(out, ""); } catch (IOException e) { throw Throwables.propagate(e); } byte[] bytes = out.toByteArray(); ObjectMetadata objectMetadata = new ObjectMetadata(); objectMetadata.setContentLength(bytes.length); objectMetadata.setContentType("text/plain"); InputStream in = new ByteArrayInputStream(bytes); PutObjectRequest putReq = new PutObjectRequest(bucketName, resourceKey, in, objectMetadata); putObject(putReq); } public Stream<S3ObjectSummary> objectStream(String prefix) { return StreamSupport.stream(S3Objects.withPrefix(conn, bucketName, prefix).spliterator(), false); } private class BulkDelete implements Callable<Long> { private final String prefix; private final long timestamp; private final AmazonS3 conn; private final String bucketName; public BulkDelete(final AmazonS3 conn, final String bucketName, final String prefix, final long timestamp) { this.conn = conn; this.bucketName = bucketName; this.prefix = prefix; this.timestamp = timestamp; } @Override public Long call() throws Exception { long count = 0L; try { checkInterrupted(); S3BlobStore.log .info(String.format("Running bulk delete on '%s/%s':%d", bucketName, prefix, timestamp)); Predicate<S3ObjectSummary> filter = new TimeStampFilter(timestamp); AtomicInteger n = new AtomicInteger(0); Iterable<List<S3ObjectSummary>> partitions = objectStream(prefix).filter(filter) .collect(Collectors.groupingBy((x) -> n.getAndIncrement() % 1000)).values(); for (List<S3ObjectSummary> partition : partitions) { checkInterrupted(); List<KeyVersion> keys = new ArrayList<>(partition.size()); for (S3ObjectSummary so : partition) { String key = so.getKey(); keys.add(new KeyVersion(key)); } checkInterrupted(); if (!keys.isEmpty()) { DeleteObjectsRequest deleteReq = new DeleteObjectsRequest(bucketName); deleteReq.setQuiet(true); deleteReq.setKeys(keys); checkInterrupted(); conn.deleteObjects(deleteReq); count += keys.size(); } } } catch (InterruptedException | IllegalStateException e) { S3BlobStore.log.info(String.format( "S3 bulk delete aborted for '%s/%s'. Will resume on next startup.", bucketName, prefix)); return null; } catch (Exception e) { S3BlobStore.log.warn( String.format("Unknown error performing bulk S3 delete of '%s/%s'", bucketName, prefix), e); throw e; } S3BlobStore.log.info(String.format("Finished bulk delete on '%s/%s':%d. %d objects deleted", bucketName, prefix, timestamp, count)); S3Ops.this.clearPendingBulkDelete(prefix, timestamp); return count; } private void checkInterrupted() throws InterruptedException { if (Thread.interrupted()) { throw new InterruptedException(); } } } /** * Filters objects that are newer than the given timestamp * */ private static class TimeStampFilter implements Predicate<S3ObjectSummary> { private long timeStamp; public TimeStampFilter(long timeStamp) { this.timeStamp = timeStamp; } @Override public boolean test(S3ObjectSummary summary) { long lastModified = summary.getLastModified().getTime(); boolean applies = timeStamp >= lastModified; return applies; } } }