Java tutorial
/* * Copyright 2010 Outerthought bvba * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.lilyproject.repository.impl.test; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.util.Arrays; import java.util.HashSet; import java.util.Random; import java.util.Set; import org.apache.commons.io.IOUtils; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.util.Bytes; import org.junit.Test; import org.lilyproject.repository.api.Blob; import org.lilyproject.repository.api.BlobException; import org.lilyproject.repository.api.BlobManager; import org.lilyproject.repository.api.BlobNotFoundException; import org.lilyproject.repository.api.BlobReference; import org.lilyproject.repository.api.FieldNotFoundException; import org.lilyproject.repository.api.FieldType; import org.lilyproject.repository.api.FieldTypeEntry; import org.lilyproject.repository.api.FieldTypeNotFoundException; import org.lilyproject.repository.api.HierarchyPath; import org.lilyproject.repository.api.InvalidRecordException; import org.lilyproject.repository.api.QName; import org.lilyproject.repository.api.Record; import org.lilyproject.repository.api.RecordId; import org.lilyproject.repository.api.RecordNotFoundException; import org.lilyproject.repository.api.RecordType; import org.lilyproject.repository.api.Repository; import org.lilyproject.repository.api.RepositoryException; import org.lilyproject.repository.api.Scope; import org.lilyproject.repository.api.TypeManager; import org.lilyproject.repository.api.ValueType; import org.lilyproject.repository.impl.BlobIncubatorMonitor; import org.lilyproject.repository.impl.BlobStoreAccessRegistry; import org.lilyproject.repository.impl.id.IdGeneratorImpl; import org.lilyproject.repotestfw.RepositorySetup; import org.lilyproject.util.hbase.LilyHBaseSchema; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; public abstract class AbstractBlobStoreTest { private static String namespace = "test"; protected static final RepositorySetup repoSetup = new RepositorySetup(); static { repoSetup.setBlobLimits(50, 1024); } protected static Repository repository; protected static TypeManager typeManager; protected static Random random = new Random(); protected static BlobStoreAccessRegistry testBlobStoreAccessRegistry; protected static BlobManager blobManager; @Test public void testCreate() throws Exception { QName fieldName = new QName(namespace, "testCreate"); FieldType fieldType = typeManager.newFieldType(typeManager.getValueType("BLOB"), fieldName, Scope.NON_VERSIONED); fieldType = typeManager.createFieldType(fieldType); RecordType recordType = typeManager.newRecordType(new QName(namespace, "testCreateRT")); FieldTypeEntry fieldTypeEntry = typeManager.newFieldTypeEntry(fieldType.getId(), true); recordType.addFieldTypeEntry(fieldTypeEntry); recordType = typeManager.createRecordType(recordType); byte[] bytes = Bytes.toBytes("someBytes"); Blob blob = writeBlob(bytes, "aMediaType", "testCreate"); Record record = repository.newRecord(); record.setRecordType(recordType.getName()); record.setField(fieldName, blob); record = repository.create(record); byte[] readBytes = readBlob(record.getId(), fieldName); assertTrue(Arrays.equals(bytes, readBytes)); // Test the getInputStream with giving the record instead of the recordId InputStream inputStream = repository.getInputStream(record, fieldName); try { byte[] readBytes2 = IOUtils.toByteArray(inputStream); assertTrue(Arrays.equals(bytes, readBytes2)); } finally { IOUtils.closeQuietly(inputStream); } } @Test public void testThreeSizes() throws Exception { QName fieldName1 = new QName(namespace, "testThreeSizes1"); QName fieldName2 = new QName(namespace, "testThreeSizes2"); QName fieldName3 = new QName(namespace, "testThreeSizes3"); FieldType fieldType1 = typeManager.newFieldType(typeManager.getValueType("BLOB"), fieldName1, Scope.NON_VERSIONED); fieldType1 = typeManager.createFieldType(fieldType1); FieldType fieldType2 = typeManager.newFieldType(typeManager.getValueType("BLOB"), fieldName2, Scope.NON_VERSIONED); fieldType2 = typeManager.createFieldType(fieldType2); FieldType fieldType3 = typeManager.newFieldType(typeManager.getValueType("BLOB"), fieldName3, Scope.NON_VERSIONED); fieldType3 = typeManager.createFieldType(fieldType3); RecordType recordType = typeManager.newRecordType(new QName(namespace, "testThreeSizes")); recordType.addFieldTypeEntry(typeManager.newFieldTypeEntry(fieldType1.getId(), true)); recordType.addFieldTypeEntry(typeManager.newFieldTypeEntry(fieldType2.getId(), true)); recordType.addFieldTypeEntry(typeManager.newFieldTypeEntry(fieldType3.getId(), true)); recordType = typeManager.createRecordType(recordType); byte[] small = new byte[10]; random.nextBytes(small); byte[] medium = new byte[100]; random.nextBytes(medium); byte[] large = new byte[2048]; random.nextBytes(large); Blob smallBlob = writeBlob(small, "mime/small", "small"); Blob mediumBlob = writeBlob(medium, "mime/medium", "medium"); Blob largeBlob = writeBlob(large, "mime/large", "large"); Record record = repository.newRecord(); record.setRecordType(recordType.getName()); record.setField(fieldName1, smallBlob); record.setField(fieldName2, mediumBlob); record.setField(fieldName3, largeBlob); record = repository.create(record); byte[] readBytes = readBlob(record.getId(), fieldName1); assertTrue(Arrays.equals(small, readBytes)); readBytes = readBlob(record.getId(), fieldName2); assertTrue(Arrays.equals(medium, readBytes)); readBytes = readBlob(record.getId(), fieldName3); assertTrue(Arrays.equals(large, readBytes)); } /** * Test case to reproduce the 'Row key is invalid' problem reported here: * https://groups.google.com/forum/#!topic/lily-discuss/XiRxOxJTv70/discussion * * @throws Exception */ @Test public void testForceInline() throws Exception { QName fieldName = new QName(namespace, "testForceInline"); FieldType fieldType = typeManager.newFieldType(typeManager.getValueType("BLOB"), fieldName, Scope.NON_VERSIONED); fieldType = typeManager.createFieldType(fieldType); RecordType recordType = typeManager.newRecordType(new QName(namespace, "testForceInlineRT")); FieldTypeEntry fieldTypeEntry = typeManager.newFieldTypeEntry(fieldType.getId(), true); recordType.addFieldTypeEntry(fieldTypeEntry); recordType = typeManager.createRecordType(recordType); int size = 4096; Random rg = new Random(); byte[] bytes = new byte[size]; rg.nextBytes(bytes); // create BLOB object Blob blob = new Blob("application/pdf", 0L, "Document"); // create a stream to write the BLOB OutputStream bos = repository.getOutputStream(blob); // write the data bos.write(bytes); bos.close(); blob.setSize(5L); // create a new record ID RecordId rid = repository.getIdGenerator().newRecordId(); // create a new record Record record = repository.newRecord(rid); record.setRecordType(new QName(namespace, "testForceInlineRT")); // set the blob record.setField(fieldName, blob); // create the record record = repository.create(record); byte[] readBytes = readBlob(record.getId(), fieldName); assertTrue(Arrays.equals(bytes, readBytes)); // Test the getInputStream with giving the record instead of the recordId InputStream inputStream = repository.getInputStream(record, fieldName); try { byte[] readBytes2 = IOUtils.toByteArray(inputStream); assertTrue(Arrays.equals(bytes, readBytes2)); } finally { IOUtils.closeQuietly(inputStream); } } @Test public void testCreateTwoRecordsWithSameBlob() throws Exception { QName fieldName = new QName(namespace, "ablob2"); FieldType fieldType = typeManager.newFieldType(typeManager.getValueType("BLOB"), fieldName, Scope.VERSIONED); fieldType = typeManager.createFieldType(fieldType); RecordType recordType = typeManager .newRecordType(new QName(namespace, "testCreateTwoRecordsWithSameBlobRT")); FieldTypeEntry fieldTypeEntry = typeManager.newFieldTypeEntry(fieldType.getId(), true); recordType.addFieldTypeEntry(fieldTypeEntry); recordType = typeManager.createRecordType(recordType); byte[] bytes = Bytes.toBytes("someBytes"); Blob blob = writeBlob(bytes, "aMediaType", "testCreate"); Record record = repository.newRecord(); record.setRecordType(recordType.getName(), null); record.setField(fieldName, blob); record = repository.create(record); Record record2 = repository.newRecord(); record2.setRecordType(recordType.getName(), null); record2.setField(fieldName, blob); record2 = repository.create(record2); // For an inline record this succeeds byte[] bytesLarge = new byte[3000]; random.nextBytes(bytesLarge); Blob largeBlob = writeBlob(bytesLarge, "largeBlob", "testCreate"); Record record3 = repository.newRecord(); record3.setRecordType(recordType.getName(), null); record3.setField(fieldName, largeBlob); record3 = repository.create(record3); Record record4 = repository.newRecord(); record4.setRecordType(recordType.getName(), null); record4.setField(fieldName, largeBlob); try { record4 = repository.create(record4); fail("Using the same blob in two records should not succeed"); } catch (InvalidRecordException expected) { } } @Test public void testUpdateNonVersionedBlobHDFS() throws Exception { testUpdateNonVersionedBlob(3000, true); } @Test public void testUpdateNonVersionedBlobHBase() throws Exception { testUpdateNonVersionedBlob(150, true); } @Test public void testUpdateNonVersionedBlobInline() throws Exception { testUpdateNonVersionedBlob(50, false); } private void testUpdateNonVersionedBlob(int size, boolean expectDelete) throws Exception { QName fieldName = new QName(namespace, "testUpdateNonVersionedBlob" + size); FieldType fieldType = typeManager.newFieldType(typeManager.getValueType("BLOB"), fieldName, Scope.NON_VERSIONED); fieldType = typeManager.createFieldType(fieldType); RecordType recordType = typeManager .newRecordType(new QName(namespace, "testUpdateNonVersionedBlobRT" + size)); FieldTypeEntry fieldTypeEntry = typeManager.newFieldTypeEntry(fieldType.getId(), true); recordType.addFieldTypeEntry(fieldTypeEntry); recordType = typeManager.createRecordType(recordType); byte[] bytes = new byte[size]; random.nextBytes(bytes); Blob blob = writeBlob(bytes, "aMediaType", "testUpdateNonVersionedBlob"); byte[] bytes2 = new byte[size]; random.nextBytes(bytes2); Blob blob2 = writeBlob(bytes2, "aMediaType", "testUpdateNonVersionedBlob2"); Record record = repository.newRecord(); record.setRecordType(recordType.getName(), null); record.setField(fieldName, blob); record = repository.create(record); Record record2 = repository.newRecord(record.getId()); record2.setRecordType(recordType.getName(), null); record2.setField(fieldName, blob2); record = repository.update(record2); // Reading should return blob2 byte[] readBytes = readBlob(record.getId(), record.getVersion(), fieldName); assertTrue(Arrays.equals(bytes2, readBytes)); assertBlobDelete(expectDelete, blob); } @Test public void testDeleteNonVersionedBlobHDFS() throws Exception { testDeleteNonVersionedBlob(3000, true); } @Test public void testDeleteNonVersionedBlobHBase() throws Exception { testDeleteNonVersionedBlob(150, true); } @Test public void testDeleteNonVersionedBlobInline() throws Exception { testDeleteNonVersionedBlob(50, false); } private void testDeleteNonVersionedBlob(int size, boolean expectDelete) throws Exception { QName fieldName = new QName(namespace, "testDeleteNonVersionedBlob" + size); FieldType fieldType = typeManager.newFieldType(typeManager.getValueType("BLOB"), fieldName, Scope.NON_VERSIONED); fieldType = typeManager.createFieldType(fieldType); RecordType recordType = typeManager .newRecordType(new QName(namespace, "testDeleteNonVersionedBlobRT" + size)); FieldTypeEntry fieldTypeEntry = typeManager.newFieldTypeEntry(fieldType.getId(), false); recordType.addFieldTypeEntry(fieldTypeEntry); recordType = typeManager.createRecordType(recordType); byte[] bytes = new byte[size]; random.nextBytes(bytes); Blob blob = writeBlob(bytes, "aMediaType", "testDeleteNonVersionedBlob"); Record record = repository.newRecord(); record.setRecordType(recordType.getName(), null); record.setField(fieldName, blob); record = repository.create(record); Record record2 = repository.newRecord(record.getId()); record2.setRecordType(recordType.getName(), null); record2.addFieldsToDelete(Arrays.asList(fieldName)); record = repository.update(record2); assertBlobDelete(expectDelete, blob); } @Test public void testUpdateMutableBlobHDFS() throws Exception { testUpdateMutableBlob(3000, true); } @Test public void testUpdateMutableBlobHBase() throws Exception { testUpdateMutableBlob(150, true); } @Test public void testUpdateMutableBlobInline() throws Exception { testUpdateMutableBlob(50, false); } private void testUpdateMutableBlob(int size, boolean expectDelete) throws Exception { QName fieldName = new QName(namespace, "testUpdateMutableBlob" + size); FieldType fieldType = typeManager.newFieldType(typeManager.getValueType("BLOB"), fieldName, Scope.VERSIONED_MUTABLE); fieldType = typeManager.createFieldType(fieldType); RecordType recordType = typeManager.newRecordType(new QName(namespace, "testUpdateMutableBlobRT" + size)); FieldTypeEntry fieldTypeEntry = typeManager.newFieldTypeEntry(fieldType.getId(), true); recordType.addFieldTypeEntry(fieldTypeEntry); recordType = typeManager.createRecordType(recordType); byte[] bytes = new byte[size]; random.nextBytes(bytes); Blob blob = writeBlob(bytes, "aMediaType", "testUpdateMutableBlob"); byte[] bytes2 = new byte[size]; random.nextBytes(bytes2); Blob blob2 = writeBlob(bytes2, "aMediaType", "testUpdateMutableBlob2"); Record record = repository.newRecord(); record.setRecordType(recordType.getName(), null); record.setField(fieldName, blob); record = repository.create(record); Record record2 = repository.newRecord(record.getId()); record2.setRecordType(recordType.getName(), null); record2.setField(fieldName, blob2); record2.setVersion(record.getVersion()); record = repository.update(record2, true, false); // Blob2 should still exist byte[] readBytes = readBlob(record.getId(), record.getVersion(), fieldName); assertTrue(Arrays.equals(bytes2, readBytes)); assertBlobDelete(expectDelete, blob); } @Test public void testDeleteMutableBlobHDFS() throws Exception { testDeleteMutableBlob(3000, true); } @Test public void testDeleteMutableBlobHBase() throws Exception { testDeleteMutableBlob(150, true); } @Test public void testDeleteMutableBlobInline() throws Exception { testDeleteMutableBlob(50, false); } private void testDeleteMutableBlob(int size, boolean expectDelete) throws Exception { QName fieldName = new QName(namespace, "testDeleteMutableBlob" + size); FieldType fieldType = typeManager.newFieldType(typeManager.getValueType("BLOB"), fieldName, Scope.VERSIONED_MUTABLE); fieldType = typeManager.createFieldType(fieldType); RecordType recordType = typeManager.newRecordType(new QName(namespace, "testDeleteMutableBlobRT" + size)); FieldTypeEntry fieldTypeEntry = typeManager.newFieldTypeEntry(fieldType.getId(), false); recordType.addFieldTypeEntry(fieldTypeEntry); recordType = typeManager.createRecordType(recordType); byte[] bytes = new byte[size]; random.nextBytes(bytes); Blob blob = writeBlob(bytes, "aMediaType", "testDeleteMutableBlob"); byte[] bytes2 = new byte[size]; random.nextBytes(bytes2); Blob blob2 = writeBlob(bytes2, "aMediaType", "testDeleteMutableBlob2"); Record record = repository.newRecord(); record.setRecordType(recordType.getName(), null); record.setField(fieldName, blob); record = repository.create(record); Record record2 = repository.newRecord(record.getId()); record2.setRecordType(recordType.getName(), null); record2.setField(fieldName, blob2); repository.update(record2, false, false); // Blob1 should still exist byte[] readBytes = readBlob(record.getId(), record.getVersion(), fieldName); assertTrue(Arrays.equals(bytes, readBytes)); // Blob2 should still exist readBytes = readBlob(record2.getId(), record2.getVersion(), fieldName); assertTrue(Arrays.equals(bytes2, readBytes)); Record record3 = repository.newRecord(record.getId()); record3.setRecordType(recordType.getName(), null); record3.addFieldsToDelete(Arrays.asList(fieldName)); record3.setVersion(record.getVersion()); repository.update(record3, true, false); // Blob2 should still exist readBytes = readBlob(record2.getId(), record2.getVersion(), fieldName); assertTrue(Arrays.equals(bytes2, readBytes)); assertBlobDelete(expectDelete, blob); } @Test public void testUpdateMutableMultivalueBlobHDFS() throws Exception { testUpdateMutableMultivalueBlob(3000, true); } @Test public void testUpdateMutableMultivalueBlobHBase() throws Exception { testUpdateMutableMultivalueBlob(150, true); } @Test public void testUpdateMutableMultivalueBlobInline() throws Exception { testUpdateMutableMultivalueBlob(50, false); } private void testUpdateMutableMultivalueBlob(int size, boolean expectDelete) throws Exception { QName fieldName = new QName(namespace, "testUpdateMutableMultivalueBlob" + size); FieldType fieldType = typeManager.newFieldType(typeManager.getValueType("LIST<BLOB>"), fieldName, Scope.VERSIONED_MUTABLE); fieldType = typeManager.createFieldType(fieldType); RecordType recordType = typeManager .newRecordType(new QName(namespace, "testUpdateMutableMultivalueBlobRT" + size)); FieldTypeEntry fieldTypeEntry = typeManager.newFieldTypeEntry(fieldType.getId(), true); recordType.addFieldTypeEntry(fieldTypeEntry); recordType = typeManager.createRecordType(recordType); byte[] bytes = new byte[size]; random.nextBytes(bytes); Blob blob = writeBlob(bytes, "aMediaType", "testUpdateMutableMultivalueBlob"); byte[] bytes2 = new byte[size]; random.nextBytes(bytes2); Blob blob2 = writeBlob(bytes2, "aMediaType", "testUpdateMutableMultivalueBlob2"); byte[] bytes3 = new byte[size]; random.nextBytes(bytes3); Blob blob3 = writeBlob(bytes3, "aMediaType", "testUpdateMutableMultivalueBlob3"); byte[] bytes4 = new byte[size]; random.nextBytes(bytes4); Blob blob4 = writeBlob(bytes4, "aMediaType", "testUpdateMutableMultivalueBlob4"); Record record = repository.newRecord(); record.setRecordType(recordType.getName(), null); record.setField(fieldName, Arrays.asList(blob, blob2)); record = repository.create(record); Record record2 = repository.newRecord(record.getId()); record2.setRecordType(recordType.getName(), null); record2.setField(fieldName, Arrays.asList(blob2, blob3)); record2 = repository.update(record2, false, false); // Mutable update of first version Record record3 = repository.newRecord(record.getId()); record3.setVersion(record.getVersion()); record3.setRecordType(recordType.getName(), null); record3.setField(fieldName, Arrays.asList(blob4)); record3 = repository.update(record3, true, false); //Blob2 byte[] readBytes = readBlob(record2.getId(), record2.getVersion(), fieldName, 0); assertTrue(Arrays.equals(bytes2, readBytes)); //Blob3 readBytes = readBlob(record2.getId(), record2.getVersion(), fieldName, 1); assertTrue(Arrays.equals(bytes3, readBytes)); //Blob4 in version 1 readBytes = readBlob(record.getId(), record.getVersion(), fieldName, 0); assertTrue(Arrays.equals(bytes4, readBytes)); assertBlobDelete(expectDelete, blob); try { readBlob(record.getId(), record.getVersion(), fieldName); fail("BlobNotFoundException expected since index should not be null"); } catch (BlobNotFoundException expected) { } try { readBlob(record.getId(), record.getVersion(), fieldName, 1); fail("BlobNotFoundException expected since index is out of bounds"); } catch (BlobNotFoundException expected) { } } @Test public void testUpdateMutableHierarchyBlobHDFS() throws Exception { testUpdateMutableHierarchyBlob(3000, true); } @Test public void testUpdateMutableHierarchyBlobHBase() throws Exception { testUpdateMutableHierarchyBlob(150, true); } @Test public void testUpdateMutableHierarchyBlobInline() throws Exception { testUpdateMutableHierarchyBlob(50, false); } private void testUpdateMutableHierarchyBlob(int size, boolean expectDelete) throws Exception { QName fieldName = new QName(namespace, "testUpdateMutableHierarchyBlob" + size); FieldType fieldType = typeManager.newFieldType(typeManager.getValueType("PATH<BLOB>"), fieldName, Scope.VERSIONED_MUTABLE); fieldType = typeManager.createFieldType(fieldType); RecordType recordType = typeManager .newRecordType(new QName(namespace, "testUpdateMutableHierarchyBlobRT" + size)); FieldTypeEntry fieldTypeEntry = typeManager.newFieldTypeEntry(fieldType.getId(), true); recordType.addFieldTypeEntry(fieldTypeEntry); recordType = typeManager.createRecordType(recordType); byte[] bytes = new byte[size]; random.nextBytes(bytes); Blob blob = writeBlob(bytes, "aMediaType", "testUpdateMutableHierarchyBlob"); byte[] bytes2 = new byte[size]; random.nextBytes(bytes2); Blob blob2 = writeBlob(bytes2, "aMediaType", "testUpdateMutableHierarchyBlob2"); byte[] bytes3 = new byte[size]; random.nextBytes(bytes3); Blob blob3 = writeBlob(bytes3, "aMediaType", "testUpdateMutableHierarchyBlob3"); byte[] bytes4 = new byte[size]; random.nextBytes(bytes4); Blob blob4 = writeBlob(bytes4, "aMediaType", "testUpdateMutableHierarchyBlob4"); Record record = repository.newRecord(); record.setRecordType(recordType.getName(), null); record.setField(fieldName, new HierarchyPath(blob, blob2)); record = repository.create(record); Record record2 = repository.newRecord(record.getId()); record2.setRecordType(recordType.getName(), null); record2.setField(fieldName, new HierarchyPath(blob2, blob3, blob4)); record2 = repository.update(record2, false, false); // Mutable update of first version Record record3 = repository.newRecord(record.getId()); record3.setVersion(record.getVersion()); record3.setRecordType(recordType.getName(), null); record3.setField(fieldName, new HierarchyPath(blob4, blob4)); record3 = repository.update(record3, true, false); // Blob2 byte[] readBytes = readBlob(record2.getId(), record2.getVersion(), fieldName, 0); assertTrue(Arrays.equals(bytes2, readBytes)); // Blob3 readBytes = readBlob(record2.getId(), record2.getVersion(), fieldName, 1); assertTrue(Arrays.equals(bytes3, readBytes)); // Blob4 in version1 readBytes = readBlob(record.getId(), record.getVersion(), fieldName, 1); assertTrue(Arrays.equals(bytes4, readBytes)); assertBlobDelete(expectDelete, blob); try { readBlob(record.getId(), record.getVersion(), fieldName); fail("BlobNotFoundException expected since index should not be null"); } catch (BlobNotFoundException expected) { } try { readBlob(record.getId(), record.getVersion(), fieldName, 2); fail("BlobNotFoundException expected since index is out of bounds"); } catch (BlobNotFoundException expected) { } } @Test public void testUpdateMutableMultivalueHierarchyBlobHDFS() throws Exception { testUpdateMutableMultivalueHierarchyBlob(3000, true); } @Test public void testUpdateMutableMultivalueHierarchyBlobHBase() throws Exception { testUpdateMutableMultivalueHierarchyBlob(150, true); } @Test public void testUpdateMutableMultivalueHierarchyBlobInline() throws Exception { testUpdateMutableMultivalueHierarchyBlob(50, false); } private void testUpdateMutableMultivalueHierarchyBlob(int size, boolean expectDelete) throws Exception { QName fieldName = new QName(namespace, "testUpdateMutableMultivalueHierarchyBlob" + size); FieldType fieldType = typeManager.newFieldType(typeManager.getValueType("LIST<PATH<BLOB>>"), fieldName, Scope.VERSIONED_MUTABLE); fieldType = typeManager.createFieldType(fieldType); RecordType recordType = typeManager .newRecordType(new QName(namespace, "testUpdateMutableMultivalueHierarchyBlobRT" + size)); FieldTypeEntry fieldTypeEntry = typeManager.newFieldTypeEntry(fieldType.getId(), true); recordType.addFieldTypeEntry(fieldTypeEntry); recordType = typeManager.createRecordType(recordType); byte[] bytes = new byte[size]; random.nextBytes(bytes); Blob blob = writeBlob(bytes, "aMediaType", "testUpdateMutableMultivalueHierarchyBlob"); byte[] bytes2 = new byte[size]; random.nextBytes(bytes); Blob blob2 = writeBlob(bytes2, "aMediaType", "testUpdateMutableMultivalueHierarchyBlob2"); byte[] bytes3 = new byte[size]; random.nextBytes(bytes3); Blob blob3 = writeBlob(bytes3, "aMediaType", "testUpdateMutableMultivalueHierarchyBlob3"); byte[] bytes4 = new byte[size]; random.nextBytes(bytes4); Blob blob4 = writeBlob(bytes4, "aMediaType", "testUpdateMutableMultivalueHierarchyBlob4"); Record record = repository.newRecord(); record.setRecordType(recordType.getName(), null); record.setField(fieldName, Arrays.asList(new HierarchyPath(blob, blob2), new HierarchyPath(blob3))); record = repository.create(record); Record record2 = repository.newRecord(record.getId()); record2.setRecordType(recordType.getName(), null); record2.setField(fieldName, Arrays.asList(new HierarchyPath(blob2), new HierarchyPath(blob3, blob4))); record2 = repository.update(record2, false, false); // Mutable update of first version Record record3 = repository.newRecord(record.getId()); record3.setVersion(record.getVersion()); record3.setRecordType(recordType.getName(), null); record3.setField(fieldName, Arrays.asList(new HierarchyPath(blob3, blob4), new HierarchyPath(blob4))); record3 = repository.update(record3, true, false); // Blob2 byte[] readBytes = readBlob(record2.getId(), record2.getVersion(), fieldName, 0, 0); assertTrue(Arrays.equals(bytes2, readBytes)); // Blob3 readBytes = readBlob(record2.getId(), record2.getVersion(), fieldName, 1, 0); assertTrue(Arrays.equals(bytes3, readBytes)); // Blob4 in version1 readBytes = readBlob(record.getId(), record.getVersion(), fieldName, 0, 1); assertTrue(Arrays.equals(bytes4, readBytes)); assertBlobDelete(expectDelete, blob); try { readBlob(record.getId(), record.getVersion(), fieldName); fail("BlobNotFoundException expected since index should not be null"); } catch (BlobNotFoundException expected) { } try { readBlob(record.getId(), record.getVersion(), fieldName, 0); fail("BlobNotFoundException expected since index should not be null"); } catch (BlobNotFoundException expected) { } try { readBlob(record.getId(), record.getVersion(), fieldName, 2, 0); fail("BlobNotFoundException expected since index is out of bounds"); } catch (BlobNotFoundException expected) { } try { readBlob(record.getId(), record.getVersion(), fieldName, 1, 1); fail("BlobNotFoundException expected since index is out of bounds"); } catch (BlobNotFoundException expected) { } } @Test public void testDelete() throws Exception { QName fieldName = new QName(namespace, "testDelete"); FieldType fieldType = typeManager.newFieldType(typeManager.getValueType("BLOB"), fieldName, Scope.NON_VERSIONED); fieldType = typeManager.createFieldType(fieldType); RecordType recordType = typeManager.newRecordType(new QName(namespace, "testDeleteRT")); FieldTypeEntry fieldTypeEntry = typeManager.newFieldTypeEntry(fieldType.getId(), true); recordType.addFieldTypeEntry(fieldTypeEntry); recordType = typeManager.createRecordType(recordType); byte[] bytes = new byte[3000]; random.nextBytes(bytes); Blob blob = writeBlob(bytes, "aMediaType", "testCreate"); Record record = repository.newRecord(); record.setRecordType(recordType.getName()); record.setField(fieldName, blob); record = repository.create(record); repository.delete(record.getId()); assertBlobDelete(true, blob); } @Test public void testDeleteMultivalueHierarchyBlobSmall() throws Exception { testDeleteMultivalueHierarchyBlob(50, false); // An inputstream for the inline blob is created on the blobKey directly } @Test public void testDeleteMultivalueHierarchyBlobMedium() throws Exception { testDeleteMultivalueHierarchyBlob(150, true); } @Test public void testDeleteMultivalueHierarchyBlobLarge() throws Exception { testDeleteMultivalueHierarchyBlob(3000, true); } private void testDeleteMultivalueHierarchyBlob(int size, boolean expectDelete) throws Exception { QName fieldName = new QName(namespace, "testDeleteMultivalueHierarchyBlob" + size); FieldType fieldType = typeManager.newFieldType(typeManager.getValueType("LIST<PATH<BLOB>>"), fieldName, Scope.VERSIONED_MUTABLE); fieldType = typeManager.createFieldType(fieldType); RecordType recordType = typeManager .newRecordType(new QName(namespace, "testDeleteMultivalueHierarchyBlobRT" + size)); FieldTypeEntry fieldTypeEntry = typeManager.newFieldTypeEntry(fieldType.getId(), true); recordType.addFieldTypeEntry(fieldTypeEntry); recordType = typeManager.createRecordType(recordType); byte[] bytes = new byte[size]; random.nextBytes(bytes); Blob blob = writeBlob(bytes, "aMediaType", "testUpdateMutableMultivalueHierarchyBlob"); byte[] bytes2 = new byte[size]; random.nextBytes(bytes); Blob blob2 = writeBlob(bytes2, "aMediaType", "testUpdateMutableMultivalueHierarchyBlob2"); byte[] bytes3 = new byte[size]; random.nextBytes(bytes3); Blob blob3 = writeBlob(bytes3, "aMediaType", "testUpdateMutableMultivalueHierarchyBlob3"); byte[] bytes4 = new byte[size]; random.nextBytes(bytes4); Blob blob4 = writeBlob(bytes4, "aMediaType", "testUpdateMutableMultivalueHierarchyBlob4"); Record record = repository.newRecord(); record.setRecordType(recordType.getName(), null); record.setField(fieldName, Arrays.asList(new HierarchyPath(blob, blob2), new HierarchyPath(blob3))); record = repository.create(record); Record record2 = repository.newRecord(record.getId()); record2.setRecordType(recordType.getName(), null); record2.setField(fieldName, Arrays.asList(new HierarchyPath(blob2), new HierarchyPath(blob3, blob4))); record2 = repository.update(record2, false, false); repository.delete(record.getId()); assertBlobDelete(expectDelete, blob); assertBlobDelete(expectDelete, blob2); assertBlobDelete(expectDelete, blob3); assertBlobDelete(expectDelete, blob4); } @Test public void testBlobIncubatorMonitorUnusedBlob() throws Exception { QName fieldName = new QName(namespace, "testBlobIncubatorMonitorUnusedBlob"); FieldType fieldType = typeManager.newFieldType(typeManager.getValueType("BLOB"), fieldName, Scope.NON_VERSIONED); fieldType = typeManager.createFieldType(fieldType); RecordType recordType = typeManager .newRecordType(new QName(namespace, "testBlobIncubatorMonitorUnusedBlobRT")); FieldTypeEntry fieldTypeEntry = typeManager.newFieldTypeEntry(fieldType.getId(), true); recordType.addFieldTypeEntry(fieldTypeEntry); recordType = typeManager.createRecordType(recordType); // Incubate blob but never use it byte[] bytes = new byte[3000]; random.nextBytes(bytes); Blob blob = writeBlob(bytes, "aMediaType", "testCreate"); // Give time for the blob to expire Thread.sleep(60); BlobIncubatorMonitor monitor = new BlobIncubatorMonitor(repoSetup.getZk(), repoSetup.getHbaseTableFactory(), repoSetup.getTableManager(), blobManager, typeManager, 50, 0, 0); monitor.runMonitorOnce(); assertBlobDelete(true, blob); } @Test public void testBlobIncubatorMonitorFailureAfterReservation() throws Exception { QName fieldName = new QName(namespace, "testBlobIncubatorMonitorFailureAfterReservation"); FieldType fieldType = typeManager.newFieldType(typeManager.getValueType("BLOB"), fieldName, Scope.NON_VERSIONED); fieldType = typeManager.createFieldType(fieldType); RecordType recordType = typeManager .newRecordType(new QName(namespace, "testBlobIncubatorMonitorFailureAfterReservationRT")); FieldTypeEntry fieldTypeEntry = typeManager.newFieldTypeEntry(fieldType.getId(), true); recordType.addFieldTypeEntry(fieldTypeEntry); recordType = typeManager.createRecordType(recordType); // This is the failure scenario where creating the record fails after reserving the blob byte[] bytes = new byte[3000]; random.nextBytes(bytes); Blob blob = writeBlob(bytes, "aMediaType", "testCreate"); IdGeneratorImpl idGeneratorImpl = new IdGeneratorImpl(); RecordId recordId = idGeneratorImpl.newRecordId(); BlobReference blobReference = new BlobReference(blob, recordId, fieldType); Set<BlobReference> blobs = new HashSet<BlobReference>(); blobs.add(blobReference); blobManager.reserveBlobs(blobs); // Give time for the blob to expire Thread.sleep(60); BlobIncubatorMonitor monitor = new BlobIncubatorMonitor(repoSetup.getZk(), repoSetup.getHbaseTableFactory(), repoSetup.getTableManager(), blobManager, typeManager, 50, 0, 0); monitor.runMonitorOnce(); assertBlobDelete(true, blob); } @Test public void testBlobIncubatorMonitorFailureBeforeRemovingReservation() throws Exception { QName fieldName = new QName(namespace, "testBlobIncubatorMonitorFailureBeforeRemovingReservation"); FieldType fieldType = typeManager.newFieldType(typeManager.getValueType("BLOB"), fieldName, Scope.NON_VERSIONED); fieldType = typeManager.createFieldType(fieldType); RecordType recordType = typeManager .newRecordType(new QName(namespace, "testBlobIncubatorMonitorFailureBeforeRemovingReservation")); FieldTypeEntry fieldTypeEntry = typeManager.newFieldTypeEntry(fieldType.getId(), true); recordType.addFieldTypeEntry(fieldTypeEntry); recordType = typeManager.createRecordType(recordType); // This is the failure scenario where creating the record fails after reserving the blob byte[] bytes = new byte[3000]; random.nextBytes(bytes); Blob blob = writeBlob(bytes, "aMediaType", "testCreate"); IdGeneratorImpl idGeneratorImpl = new IdGeneratorImpl(); RecordId recordId = idGeneratorImpl.newRecordId(); BlobReference blobReference = new BlobReference(blob, recordId, fieldType); Set<BlobReference> blobs = new HashSet<BlobReference>(); blobs.add(blobReference); repository.newRecord(); Record record = repository.newRecord(); record.setRecordType(recordType.getName()); record.setField(fieldName, blob); record = repository.create(record); // Faking failure HTableInterface blobIncubatorTable = LilyHBaseSchema.getBlobIncubatorTable(repoSetup.getHbaseTableFactory(), true); Put put = new Put(blob.getValue()); put.add(LilyHBaseSchema.BlobIncubatorCf.REF.bytes, LilyHBaseSchema.BlobIncubatorColumn.RECORD.bytes, record.getId().toBytes()); put.add(LilyHBaseSchema.BlobIncubatorCf.REF.bytes, LilyHBaseSchema.BlobIncubatorColumn.FIELD.bytes, fieldType.getId().getBytes()); blobIncubatorTable.put(put); // Give time for the blob to expire Thread.sleep(60); BlobIncubatorMonitor monitor = new BlobIncubatorMonitor(repoSetup.getZk(), repoSetup.getHbaseTableFactory(), repoSetup.getTableManager(), blobManager, typeManager, 50, 0, 0); monitor.runMonitorOnce(); assertBlobDelete(false, blob); Get get = new Get(blob.getValue()); Result result = blobIncubatorTable.get(get); assertTrue(result == null || result.isEmpty()); } private void assertBlobDelete(boolean expectDelete, Blob blob) throws BlobNotFoundException, BlobException { if (expectDelete) { try { testBlobStoreAccessRegistry.getBlobAccess(blob).getInputStream(); fail("The blob " + blob + " should have been deleted."); } catch (BlobException expected) { } } else { testBlobStoreAccessRegistry.getBlobAccess(blob).getInputStream(); } } @Test public void testBadEncoding() throws Exception { Blob blob = new Blob("aMediaType", (long) 10, "aName"); blob.setValue(new byte[0]); try { testBlobStoreAccessRegistry.getBlobAccess(blob).getInputStream(); fail(); } catch (BlobException expected) { } } @Test public void testInvalidReadRequests() throws Exception { ValueType stringType = typeManager.getValueType("STRING"); ValueType blobType = typeManager.getValueType("BLOB"); FieldType nonBlobField = typeManager.newFieldType(stringType, new QName(namespace, "NonBlobField"), Scope.VERSIONED); nonBlobField = typeManager.createFieldType(nonBlobField); FieldType absentField = typeManager.newFieldType(blobType, new QName(namespace, "AbsentField"), Scope.VERSIONED); absentField = typeManager.createFieldType(absentField); RecordType rt = typeManager.newRecordType(new QName(namespace, "NoBlobsRT")); rt.addFieldTypeEntry(nonBlobField.getId(), false); rt = typeManager.createRecordType(rt); Record record = repository.newRecord(); record.setRecordType(rt.getName()); record.setField(nonBlobField.getName(), "This is not a blob"); record = repository.create(record); try { repository.getInputStream(record.getId(), record.getVersion(), nonBlobField.getName(), null, null); fail("Expected exception"); } catch (BlobException e) { // ok } try { repository.getInputStream(record.getId(), record.getVersion(), absentField.getName(), null, null); fail("Expected exception"); } catch (FieldNotFoundException e) { // ok } try { repository.getInputStream(record.getId(), record.getVersion(), new QName(namespace, "nonExistingFieldType"), null, null); fail("Expected exception"); } catch (FieldTypeNotFoundException e) { // ok } try { repository.getInputStream(record.getId(), record.getVersion(), null, null, null); fail("Expected exception"); } catch (IllegalArgumentException e) { // ok } try { repository.getInputStream(repoSetup.getIdGenerator().fromString("USER.nonexistingrecord"), null, absentField.getName()); fail("Expected exception"); } catch (RecordNotFoundException e) { // ok } } private Blob writeBlob(byte[] bytes, String mediaType, String name) throws RepositoryException, InterruptedException, IOException { return writeBlob(bytes, mediaType, name, bytes.length); } /** * @param length The blob site to be used when constructing the blob * (this can be used to control how the blob will be stored) */ private Blob writeBlob(byte[] bytes, String mediaType, String name, long length) throws RepositoryException, InterruptedException, IOException { Blob blob = new Blob(mediaType, length, name); OutputStream outputStream = repository.getOutputStream(blob); outputStream.write(bytes); outputStream.close(); return blob; } private byte[] readBlob(RecordId recordId, QName fieldName) throws RepositoryException, InterruptedException, IOException { return readBlob(recordId, null, fieldName); } private byte[] readBlob(RecordId recordId, Long version, QName fieldName, int... indexes) throws RepositoryException, InterruptedException, IOException { InputStream inputStream = repository.getInputStream(recordId, version, fieldName, indexes); try { return IOUtils.toByteArray(inputStream); } finally { IOUtils.closeQuietly(inputStream); } } }