Java tutorial
/** * * Copyright (C) 2009 Cloud Conscious, LLC. <info@cloudconscious.com> * * ==================================================================== * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. * ==================================================================== */ package org.jclouds.blobstore.integration.internal; import static com.google.common.base.Preconditions.checkNotNull; import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.io.ObjectInput; import java.io.ObjectInputStream; import java.io.ObjectOutput; import java.io.ObjectOutputStream; import java.net.URI; import java.util.Collection; import java.util.List; import java.util.Map; import java.util.SortedSet; import java.util.TreeSet; import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import javax.inject.Inject; import javax.ws.rs.core.HttpHeaders; import org.apache.commons.io.IOUtils; import org.apache.commons.io.output.ByteArrayOutputStream; import org.jclouds.blobstore.AsyncBlobStore; import org.jclouds.blobstore.ContainerNotFoundException; import org.jclouds.blobstore.KeyNotFoundException; import org.jclouds.blobstore.attr.ConsistencyModel; import org.jclouds.blobstore.attr.ConsistencyModels; import org.jclouds.blobstore.domain.Blob; import org.jclouds.blobstore.domain.BlobMetadata; import org.jclouds.blobstore.domain.ListContainerResponse; import org.jclouds.blobstore.domain.ListResponse; import org.jclouds.blobstore.domain.MutableBlobMetadata; import org.jclouds.blobstore.domain.MutableResourceMetadata; import org.jclouds.blobstore.domain.ResourceMetadata; import org.jclouds.blobstore.domain.ResourceType; import org.jclouds.blobstore.domain.internal.ListContainerResponseImpl; import org.jclouds.blobstore.domain.internal.ListResponseImpl; import org.jclouds.blobstore.domain.internal.MutableResourceMetadataImpl; import org.jclouds.blobstore.functions.HttpGetOptionsListToGetOptions; import org.jclouds.blobstore.options.GetOptions; import org.jclouds.blobstore.options.ListContainerOptions; import org.jclouds.http.HttpCommand; import org.jclouds.http.HttpRequest; import org.jclouds.http.HttpResponse; import org.jclouds.http.HttpResponseException; import org.jclouds.http.HttpUtils; import org.jclouds.http.options.HttpRequestOptions; import org.jclouds.util.DateService; import org.jclouds.util.Utils; import org.joda.time.DateTime; import com.google.common.base.Function; import com.google.common.base.Predicate; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.inject.internal.Nullable; /** * Implementation of {@link S3BlobStore} which keeps all data in a local Map object. * * @author Adrian Cole * @author James Murty */ @ConsistencyModel(ConsistencyModels.STRICT) public class StubAsyncBlobStore implements AsyncBlobStore { protected final DateService dateService; private final ConcurrentMap<String, ConcurrentMap<String, Blob>> containerToBlobs; protected final Blob.Factory blobProvider; protected final HttpGetOptionsListToGetOptions httpGetOptionsConverter; @Inject protected StubAsyncBlobStore(ConcurrentMap<String, ConcurrentMap<String, Blob>> containerToBlobs, DateService dateService, Blob.Factory blobProvider, HttpGetOptionsListToGetOptions httpGetOptionsConverter) { this.dateService = checkNotNull(dateService, "dateService"); this.containerToBlobs = checkNotNull(containerToBlobs, "containerToBlobs"); this.blobProvider = checkNotNull(blobProvider, "blobProvider"); this.httpGetOptionsConverter = checkNotNull(httpGetOptionsConverter, "httpGetOptionsConverter"); } /** * @throws java.io.IOException */ public static byte[] toByteArray(Object data) throws IOException { checkNotNull(data, "data must be set before calling generateETag()"); byte[] bytes = null; if (data == null || data instanceof byte[]) { bytes = (byte[]) data; } else if (data instanceof String) { bytes = ((String) data).getBytes(); } else if (data instanceof File || data instanceof InputStream) { InputStream io = (data instanceof InputStream) ? (InputStream) data : new FileInputStream((File) data); bytes = IOUtils.toByteArray(io); IOUtils.closeQuietly(io); } else { throw new UnsupportedOperationException("Content not supported " + data.getClass()); } return bytes; } public Future<Blob> getBlob(final String bucketName, final String key) { return new FutureBase<Blob>() { public Blob get() throws InterruptedException, ExecutionException { if (!getContainerToBlobs().containsKey(bucketName)) throw new ContainerNotFoundException(bucketName); Map<String, Blob> realContents = getContainerToBlobs().get(bucketName); if (!realContents.containsKey(key)) throw new KeyNotFoundException(bucketName, key); Blob object = realContents.get(key); Blob returnVal = blobProvider.create(copy(object.getMetadata())); returnVal.setData(new ByteArrayInputStream((byte[]) object.getData())); return returnVal; } }; } public Future<? extends ListContainerResponse<? extends ResourceMetadata>> list(final String name, ListContainerOptions... optionsList) { final ListContainerOptions options = (optionsList.length == 0) ? new ListContainerOptions() : optionsList[0]; return new FutureBase<ListContainerResponse<ResourceMetadata>>() { public ListContainerResponse<ResourceMetadata> get() throws InterruptedException, ExecutionException { final Map<String, Blob> realContents = getContainerToBlobs().get(name); if (realContents == null) throw new ContainerNotFoundException(name); SortedSet<ResourceMetadata> contents = Sets.newTreeSet( Iterables.transform(realContents.keySet(), new Function<String, ResourceMetadata>() { public ResourceMetadata apply(String key) { return copy(realContents.get(key).getMetadata()); } })); if (options.getMarker() != null) { final String finalMarker = options.getMarker(); ResourceMetadata lastMarkerMetadata = Iterables.find(contents, new Predicate<ResourceMetadata>() { public boolean apply(ResourceMetadata metadata) { return metadata.getName().equals(finalMarker); } }); contents = contents.tailSet(lastMarkerMetadata); contents.remove(lastMarkerMetadata); } final String prefix = options.getPath(); if (prefix != null) { contents = Sets.newTreeSet(Iterables.filter(contents, new Predicate<ResourceMetadata>() { public boolean apply(ResourceMetadata o) { return (o != null && o.getName().startsWith(prefix)); } })); } int maxResults = contents.size(); boolean truncated = false; String marker = null; if (options.getMaxResults() != null && contents.size() > 0) { SortedSet<ResourceMetadata> contentsSlice = firstSliceOfSize(contents, options.getMaxResults().intValue()); maxResults = options.getMaxResults(); if (!contentsSlice.contains(contents.last())) { // Partial listing truncated = true; marker = contentsSlice.last().getName(); } else { marker = null; } contents = contentsSlice; } final String delimiter = options.isRecursive() ? null : "/"; if (delimiter != null) { SortedSet<String> commonPrefixes = null; Iterable<String> iterable = Iterables.transform(contents, new CommonPrefixes(prefix != null ? prefix : null, delimiter)); commonPrefixes = iterable != null ? Sets.newTreeSet(iterable) : new TreeSet<String>(); commonPrefixes.remove(CommonPrefixes.NO_PREFIX); contents = Sets.newTreeSet(Iterables.filter(contents, new DelimiterFilter(prefix != null ? prefix : null, delimiter))); Iterables.<ResourceMetadata>addAll(contents, Iterables.transform(commonPrefixes, new Function<String, ResourceMetadata>() { public ResourceMetadata apply(String o) { MutableResourceMetadata md = new MutableResourceMetadataImpl(); md.setType(ResourceType.RELATIVE_PATH); md.setName(o); return md; } })); } return new ListContainerResponseImpl<ResourceMetadata>(contents, prefix, marker, maxResults, truncated); } }; } public MutableBlobMetadata copy(MutableBlobMetadata in) { ByteArrayOutputStream bout = new ByteArrayOutputStream(); ObjectOutput os; try { os = new ObjectOutputStream(bout); os.writeObject(in); ObjectInput is = new ObjectInputStream(new ByteArrayInputStream(bout.toByteArray())); MutableBlobMetadata metadata = (MutableBlobMetadata) is.readObject(); convertUserMetadataKeysToLowercase(metadata); return metadata; } catch (Exception e) { throw new RuntimeException(e); } } private void convertUserMetadataKeysToLowercase(MutableBlobMetadata metadata) { Map<String, String> lowerCaseUserMetadata = Maps.newHashMap(); for (Entry<String, String> entry : metadata.getUserMetadata().entrySet()) { lowerCaseUserMetadata.put(entry.getKey().toLowerCase(), entry.getValue()); } metadata.setUserMetadata(lowerCaseUserMetadata); } public MutableBlobMetadata copy(MutableBlobMetadata in, String newKey) { MutableBlobMetadata newMd = copy(in); newMd.setName(newKey); return newMd; } public BlobMetadata metadata(final String container, final String key) { if (!getContainerToBlobs().containsKey(container)) throw new ContainerNotFoundException(container); Map<String, Blob> realContents = getContainerToBlobs().get(container); if (!realContents.containsKey(key)) throw new KeyNotFoundException(container, key); return copy(realContents.get(key).getMetadata()); } public Future<Void> removeBlob(final String container, final String key) { return new FutureBase<Void>() { public Void get() throws InterruptedException, ExecutionException { if (getContainerToBlobs().containsKey(container)) { getContainerToBlobs().get(container).remove(key); } return null; } }; } public Future<Void> deleteContainer(final String container) { return new FutureBase<Void>() { public Void get() throws InterruptedException, ExecutionException { if (getContainerToBlobs().containsKey(container)) { getContainerToBlobs().remove(container); } return null; } }; } public Future<Boolean> deleteContainerImpl(final String container) { return new FutureBase<Boolean>() { public Boolean get() throws InterruptedException, ExecutionException { if (getContainerToBlobs().containsKey(container)) { if (getContainerToBlobs().get(container).size() == 0) getContainerToBlobs().remove(container); else return false; } return true; } }; } public Future<Boolean> exists(final String container) { return new FutureBase<Boolean>() { public Boolean get() throws InterruptedException, ExecutionException { return getContainerToBlobs().containsKey(container); } }; } public static abstract class FutureBase<V> implements Future<V> { public boolean cancel(boolean b) { return false; } public boolean isCancelled() { return false; } public boolean isDone() { return true; } public V get(long l, TimeUnit timeUnit) throws InterruptedException, ExecutionException, TimeoutException { return get(); } } public Future<? extends ListResponse<? extends ResourceMetadata>> list() { return new FutureBase<ListResponse<? extends ResourceMetadata>>() { public ListResponse<ResourceMetadata> get() throws InterruptedException, ExecutionException { return new ListResponseImpl<ResourceMetadata>(Iterables.transform(getContainerToBlobs().keySet(), new Function<String, ResourceMetadata>() { public ResourceMetadata apply(String name) { MutableResourceMetadata cmd = create(); cmd.setName(name); cmd.setType(ResourceType.CONTAINER); return cmd; } }), null, null, false); } }; } protected MutableResourceMetadata create() { return new MutableResourceMetadataImpl(); } public Future<Boolean> createContainer(final String name) { return new FutureBase<Boolean>() { public Boolean get() throws InterruptedException, ExecutionException { if (!getContainerToBlobs().containsKey(name)) { getContainerToBlobs().put(name, new ConcurrentHashMap<String, Blob>()); } return getContainerToBlobs().containsKey(name); } }; } public String getFirstQueryOrNull(String string, @Nullable HttpRequestOptions options) { if (options == null) return null; Collection<String> values = options.buildQueryParameters().get(string); return (values != null && values.size() >= 1) ? values.iterator().next() : null; } protected class DelimiterFilter implements Predicate<ResourceMetadata> { private final String prefix; private final String delimiter; public DelimiterFilter(String prefix, String delimiter) { this.prefix = prefix; this.delimiter = delimiter; } public boolean apply(ResourceMetadata metadata) { if (prefix == null) return metadata.getName().indexOf(delimiter) == -1; if (metadata.getName().startsWith(prefix)) return metadata.getName().replaceFirst(prefix, "").indexOf(delimiter) == -1; return false; } } protected class CommonPrefixes implements Function<ResourceMetadata, String> { private final String prefix; private final String delimiter; public static final String NO_PREFIX = "NO_PREFIX"; public CommonPrefixes(String prefix, String delimiter) { this.prefix = prefix; this.delimiter = delimiter; } public String apply(ResourceMetadata metadata) { String working = metadata.getName(); if (prefix != null) { if (working.startsWith(prefix)) { working = working.replaceFirst(prefix, ""); } } if (working.contains(delimiter)) { return working.substring(0, working.indexOf(delimiter)); } return NO_PREFIX; } } public static <T extends Comparable<?>> SortedSet<T> firstSliceOfSize(Iterable<T> elements, int size) { List<List<T>> slices = Lists.partition(Lists.newArrayList(elements), size); return Sets.newTreeSet(slices.get(0)); } public void throwResponseException(int code) throws ExecutionException { HttpResponse response = null; response = new HttpResponse(); // TODO: Get real object URL? response.setStatusCode(code); throw new ExecutionException(new HttpResponseException(new HttpCommand() { public int getRedirectCount() { return 0; } public int incrementRedirectCount() { return 0; } public boolean isReplayable() { return false; } public HttpRequest setHostAndPort(String host, int port) { return null; } public HttpRequest setMethod(String method) { return null; } public Exception getException() { return null; } public int getFailureCount() { return 0; } public HttpRequest getRequest() { return new HttpRequest("GET", URI.create("http://stub")); } public int incrementFailureCount() { return 0; } public void setException(Exception exception) { } }, response)); } public Future<String> putBlob(final String bucketName, final Blob object) { Map<String, Blob> container = getContainerToBlobs().get(bucketName); if (container == null) { new RuntimeException("bucketName not found: " + bucketName); } try { MutableBlobMetadata newMd = copy(object.getMetadata()); newMd.setLastModified(new DateTime()); byte[] data = toByteArray(object.getData()); final byte[] md5 = HttpUtils.md5(data); final String eTag = HttpUtils.toHexString(md5); newMd.setETag(eTag); newMd.setContentMD5(md5); newMd.setContentType(object.getMetadata().getContentType()); Blob blob = blobProvider.create(newMd); blob.setData(data); container.put(blob.getMetadata().getName(), blob); // Set HTTP headers to match metadata blob.getAllHeaders().put(HttpHeaders.LAST_MODIFIED, dateService.rfc822DateFormat(newMd.getLastModified())); blob.getAllHeaders().put(HttpHeaders.ETAG, eTag); blob.getAllHeaders().put(HttpHeaders.CONTENT_TYPE, newMd.getContentType()); blob.getAllHeaders().put(HttpHeaders.CONTENT_LENGTH, newMd.getSize() + ""); for (Entry<String, String> userMD : newMd.getUserMetadata().entrySet()) { blob.getAllHeaders().put(userMD.getKey(), userMD.getValue()); } return new FutureBase<String>() { public String get() throws InterruptedException, ExecutionException { return eTag; } }; } catch (IOException e) { throw new RuntimeException(e); } } public Future<? extends Blob> getBlob(final String bucketName, final String key, GetOptions... optionsList) { final GetOptions options = (optionsList.length == 0) ? new GetOptions() : optionsList[0]; return new FutureBase<Blob>() { public Blob get() throws InterruptedException, ExecutionException { if (!getContainerToBlobs().containsKey(bucketName)) throw new ContainerNotFoundException(bucketName); Map<String, Blob> realContents = getContainerToBlobs().get(bucketName); if (!realContents.containsKey(key)) throw new KeyNotFoundException(bucketName, key); Blob object = realContents.get(key); if (options.getIfMatch() != null) { if (!object.getMetadata().getETag().equals(options.getIfMatch())) throwResponseException(412); } if (options.getIfNoneMatch() != null) { if (object.getMetadata().getETag().equals(options.getIfNoneMatch())) throwResponseException(304); } if (options.getIfModifiedSince() != null) { DateTime modifiedSince = options.getIfModifiedSince(); if (object.getMetadata().getLastModified().isBefore(modifiedSince)) { HttpResponse response = new HttpResponse(); response.setStatusCode(304); throw new ExecutionException(new HttpResponseException(String.format("%1$s is before %2$s", object.getMetadata().getLastModified(), modifiedSince), null, response)); } } if (options.getIfUnmodifiedSince() != null) { DateTime unmodifiedSince = options.getIfUnmodifiedSince(); if (object.getMetadata().getLastModified().isAfter(unmodifiedSince)) { HttpResponse response = new HttpResponse(); response.setStatusCode(412); throw new ExecutionException(new HttpResponseException(String.format("%1$s is after %2$s", object.getMetadata().getLastModified(), unmodifiedSince), null, response)); } } Blob returnVal = copyBlob(object); if (options.getRanges() != null && options.getRanges().size() > 0) { byte[] data = (byte[]) returnVal.getData(); ByteArrayOutputStream out = new ByteArrayOutputStream(); for (String s : options.getRanges()) { if (s.startsWith("-")) { int length = Integer.parseInt(s); out.write(data, data.length - length, length); } else if (s.endsWith("-")) { int offset = Integer.parseInt(s); out.write(data, offset, data.length - offset); } else if (s.contains("-")) { String[] firstLast = s.split("\\-"); int offset = Integer.parseInt(firstLast[0]); int last = Integer.parseInt(firstLast[1]); int length = (last < data.length) ? last + 1 : data.length - offset; out.write(data, offset, length); } else { throw new IllegalArgumentException("first and last were null!"); } } returnVal.setData(out.toByteArray()); returnVal.setContentLength(out.size()); returnVal.getMetadata().setSize(new Long(data.length)); } returnVal.setData(new ByteArrayInputStream((byte[]) returnVal.getData())); return returnVal; } }; } public Future<BlobMetadata> blobMetadata(final String container, final String key) { return new FutureBase<BlobMetadata>() { public BlobMetadata get() throws InterruptedException, ExecutionException { try { return copy(getBlob(container, key).get().getMetadata()); } catch (Exception e) { Utils.<ContainerNotFoundException>rethrowIfRuntimeOrSameType(e); Utils.<KeyNotFoundException>rethrowIfRuntimeOrSameType(e); throw new RuntimeException(e);// TODO } } }; } private Blob copyBlob(Blob object) { Blob returnVal = blobProvider.create(copy(object.getMetadata())); returnVal.setData(object.getData()); return returnVal; } public ConcurrentMap<String, ConcurrentMap<String, Blob>> getContainerToBlobs() { return containerToBlobs; } public Future<Void> clearContainer(final String container) { return new FutureBase<Void>() { public Void get() throws InterruptedException, ExecutionException { getContainerToBlobs().get(container).clear(); return null; } }; } public Blob newBlob() { return blobProvider.create(null); } }