Java tutorial
/* * Copyright 2010 Talis Information Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.talis.storage.s3; import static org.easymock.classextension.EasyMock.createMock; import static org.easymock.classextension.EasyMock.replay; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.net.URI; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.SortedSet; import java.util.TreeSet; import java.util.UUID; import javax.ws.rs.core.MediaType; import org.apache.commons.io.IOUtils; import org.jets3t.service.S3Service; import org.jets3t.service.model.S3Object; import org.junit.Test; import com.google.inject.Provider; import com.talis.storage.AbstractStoreTest; import com.talis.storage.InvalidKeyException; import com.talis.storage.Store; import com.talis.storage.SubmittedItem; public class S3StoreTest extends AbstractStoreTest { protected String bucketname; protected S3Service s3; protected S3ObjectFactory objectFactory; protected ChunkHandler chunkHandler; @Override protected Store getStore() throws Exception { bucketname = UUID.randomUUID().toString(); Provider<S3Service> serviceProvider = initialiseServiceProvider(); final S3Object stubObject = new S3Object("foo"); objectFactory = new S3ObjectFactory(bucketname) { @Override public ExternalizableS3Object newObject(String key, int index, MediaType mediaType, ByteBuffer buffer) throws IOException { ExternalizableS3Object obj = new ExternalizableS3Object(); obj.setKey(key + "/" + index); obj.setContentType(S3Store.TMB_CHUNK_TYPE.toString()); obj.addMetadata(S3Store.ACTUAL_CONTENT_TYPE_HEADER, mediaType.toString()); byte[] bytes = new byte[buffer.position()]; buffer.rewind(); buffer.get(bytes); obj.setDataInputStream(new ByteArrayInputStream(bytes)); return obj; } }; chunkHandler = new StubChunkHandler(); return new S3Store(objectFactory, chunkHandler); } @Test public void s3KeysDerivedFromSubmittedItemURIPlusPrefix() throws Exception { S3Store s3Store = (S3Store) store; String prefix = S3Store.KEY_PREFIX; URI itemUri = URI.create("http://example.com/1/2/3/4"); assertEquals(prefix + "/" + "1/2/3/4", s3Store.mapUriToNativeKey(itemUri)); } @Test(expected = InvalidKeyException.class) public void submittedItemURIMustContainPath() throws Exception { S3Store s3Store = (S3Store) store; String prefix = S3Store.KEY_PREFIX; URI itemUri = URI.create("http://example.com"); s3Store.mapUriToNativeKey(itemUri); } @Test(expected = InvalidKeyException.class) public void submittedItemURIMustContainNonEmptyPath() throws Exception { S3Store s3Store = (S3Store) store; String prefix = S3Store.KEY_PREFIX; URI itemUri = URI.create("http://example.com/"); s3Store.mapUriToNativeKey(itemUri); } @Test public void inputDataLargerThanChunkSize() throws Exception { try { System.setProperty(S3Store.CHUNK_SIZE_PROPERTY, "10"); System.setProperty(S3Store.READ_BUFFER_SIZE_PROPERTY, "1"); S3Store s3store = (S3Store) getStore(); s3store.write(itemURI, new SubmittedItem(MediaType.TEXT_PLAIN_TYPE, new ByteArrayInputStream(data))); S3Object[] chunks = chunkHandler.listChunks(s3store.mapUriToNativeKey(itemURI)); assertEquals(6, chunks.length); ByteArrayOutputStream buffer = new ByteArrayOutputStream(); for (S3Object chunk : chunks) { IOUtils.copy(chunk.getDataInputStream(), buffer); } assertTrue(Arrays.equals(data, buffer.toByteArray())); } finally { System.clearProperty(S3Store.CHUNK_SIZE_PROPERTY); System.clearProperty(S3Store.READ_BUFFER_SIZE_PROPERTY); } } // chunk size must be >= read buffer // read and write with different stores + chunk sizes private Provider<S3Service> initialiseServiceProvider() throws Exception { // bucket = new S3Bucket(S3Store.BUCKETNAME); s3 = createMock(S3Service.class); // s3.getOrCreateBucket(S3Store.BUCKETNAME); // expectLastCall().andReturn(bucket); replay(s3); return new Provider<S3Service>() { @Override public S3Service get() { return s3; } }; } class StubChunkHandler implements ChunkHandler { private Map<String, S3Object> stubS3; StubChunkHandler() { stubS3 = new HashMap<String, S3Object>(); } @Override public String getBucketname() { return bucketname; } @Override public S3Object writeChunk(S3Object chunk) throws IOException { S3Object stored = clone(chunk); stored.setLastModifiedDate(new Date(System.currentTimeMillis())); stored.setETag("this-is-a-stub-etag"); stubS3.put(chunk.getKey(), stored); return stored; } @Override public S3Object[] listChunks(String key) throws IOException { SortedSet<String> keys = new TreeSet<String>(stubS3.keySet()); List<S3Object> chunks = new ArrayList<S3Object>(); for (String storedKey : keys) { if (storedKey.startsWith(key)) { chunks.add(clone(stubS3.get(storedKey))); } } return chunks.toArray(new S3Object[0]); } @Override public S3Object getChunk(String key) throws IOException { if (stubS3.containsKey(key)) { return clone(stubS3.get(key)); } else { throw new IOException("Not found"); } } @Override public void deleteChunk(String key) throws IOException { stubS3.remove(key); } @Override public InputStream reconstructChunks(S3Object[] chunks) throws IOException { try { ByteArrayOutputStream buffer = new ByteArrayOutputStream(); for (S3Object chunk : chunks) { S3Object clone = getChunk(chunk.getKey()); buffer.write(getEntityBytes(clone)); } return new ByteArrayInputStream(buffer.toByteArray()); } catch (Exception e) { throw new IOException("Error", e); } } private S3Object clone(S3Object chunk) throws IOException { try { S3Object clone = new S3Object(chunk.getKey()); clone.setContentType(S3Store.TMB_CHUNK_TYPE.toString()); clone.setDataInputStream(new ByteArrayInputStream(getEntityBytes(chunk))); clone.addAllMetadata(chunk.getMetadataMap()); clone.setETag(chunk.getETag()); if (null != chunk.getLastModifiedDate()) { clone.setLastModifiedDate(chunk.getLastModifiedDate()); } return clone; } catch (Exception e) { throw new IOException("Error", e); } } private byte[] getEntityBytes(S3Object chunk) throws IOException { try { ByteArrayOutputStream out = new ByteArrayOutputStream(); IOUtils.copy(chunk.getDataInputStream(), out); chunk.getDataInputStream().reset(); return out.toByteArray(); } catch (Exception e) { throw new IOException("Error", e); } } } }