Java tutorial
/* * Copyright 2010 Outerthought bvba * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.lilyproject.repository.impl; import java.io.IOException; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.NavigableMap; import java.util.Set; import com.ngdata.lily.security.hbase.client.HBaseAuthzUtil; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.filter.ColumnPrefixFilter; import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.filter.FilterList; import org.apache.hadoop.hbase.filter.PrefixFilter; import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; import org.apache.hadoop.hbase.filter.WritableByteArrayComparable; import org.apache.hadoop.hbase.util.Bytes; import org.lilyproject.bytes.api.DataOutput; import org.lilyproject.bytes.impl.DataOutputImpl; import org.lilyproject.repository.api.Blob; import org.lilyproject.repository.api.BlobException; import org.lilyproject.repository.api.BlobManager; import org.lilyproject.repository.api.BlobReference; import org.lilyproject.repository.api.ConcurrentRecordUpdateException; import org.lilyproject.repository.api.FieldType; import org.lilyproject.repository.api.FieldTypeEntry; import org.lilyproject.repository.api.FieldTypeNotFoundException; import org.lilyproject.repository.api.FieldTypes; import org.lilyproject.repository.api.IdGenerator; import org.lilyproject.repository.api.IdentityRecordStack; import org.lilyproject.repository.api.InvalidRecordException; import org.lilyproject.repository.api.Metadata; import org.lilyproject.repository.api.MetadataBuilder; import org.lilyproject.repository.api.MutationCondition; import org.lilyproject.repository.api.QName; import org.lilyproject.repository.api.Record; import org.lilyproject.repository.api.RecordBuilder; import org.lilyproject.repository.api.RecordException; import org.lilyproject.repository.api.RecordExistsException; import org.lilyproject.repository.api.RecordFactory; import org.lilyproject.repository.api.RecordId; import org.lilyproject.repository.api.RecordNotFoundException; import org.lilyproject.repository.api.RecordType; import org.lilyproject.repository.api.RepositoryException; import org.lilyproject.repository.api.TableManager; import org.lilyproject.repository.api.ResponseStatus; import org.lilyproject.repository.api.SchemaId; import org.lilyproject.repository.api.Scope; import org.lilyproject.repository.api.TypeException; import org.lilyproject.repository.api.ValueType; import org.lilyproject.repository.api.VersionNotFoundException; import org.lilyproject.repository.impl.RepositoryMetrics.Action; import org.lilyproject.repository.impl.hbase.ContainsValueComparator; import org.lilyproject.repository.impl.id.SchemaIdImpl; import org.lilyproject.repository.impl.valuetype.BlobValueType; import org.lilyproject.repository.spi.AuthorizationContextHolder; import org.lilyproject.repository.spi.RecordUpdateHook; import org.lilyproject.util.ArgumentValidator; import org.lilyproject.util.Pair; import org.lilyproject.util.hbase.LilyHBaseSchema.RecordCf; import org.lilyproject.util.hbase.LilyHBaseSchema.RecordColumn; import org.lilyproject.util.io.Closer; import org.lilyproject.util.repo.RecordEvent; import org.lilyproject.util.repo.RecordEvent.Type; import static org.lilyproject.repository.impl.RecordDecoder.RECORD_TYPE_ID_QUALIFIERS; import static org.lilyproject.repository.impl.RecordDecoder.RECORD_TYPE_VERSION_QUALIFIERS; /** * Repository implementation. */ public class HBaseRepository extends BaseRepository { private List<RecordUpdateHook> updateHooks = Collections.emptyList(); private final Log log = LogFactory.getLog(getClass()); private static final Object METADATA_ONLY_UPDATE = new Object(); public HBaseRepository(RepoTableKey ttk, AbstractRepositoryManager repositoryManager, HTableInterface recordTable, HTableInterface nonAuthRecordTable, BlobManager blobManager, TableManager tableManager, RecordFactory recordFactory) throws IOException, InterruptedException { super(ttk, repositoryManager, blobManager, recordTable, nonAuthRecordTable, new RepositoryMetrics("hbaserepository"), tableManager, recordFactory); } @Override public void close() throws IOException { } /** * Sets the record update hooks. */ public void setRecordUpdateHooks(List<RecordUpdateHook> recordUpdateHooks) { this.updateHooks = recordUpdateHooks == null ? Collections.<RecordUpdateHook>emptyList() : recordUpdateHooks; } @Override public IdGenerator getIdGenerator() { return idGenerator; } @Override public Record createOrUpdate(Record record) throws RepositoryException, InterruptedException { return createOrUpdate(record, true); } @Override public Record createOrUpdate(Record record, boolean useLatestRecordType) throws RepositoryException, InterruptedException { if (record.getId() == null) { // While we could generate an ID ourselves in this case, this would defeat partly the purpose of // createOrUpdate, which is that clients would be able to retry the operation (in case of IO exceptions) // without having to worry that more than one record might be created. throw new RecordException("Record ID is mandatory when using create-or-update."); } byte[] rowId = record.getId().toBytes(); Get get = new Get(rowId); get.addColumn(RecordCf.DATA.bytes, RecordColumn.DELETED.bytes); int attempts; for (attempts = 0; attempts < 3; attempts++) { Result result; try { result = recordTable.get(get); } catch (IOException e) { throw new RecordException("Error reading record row for record id " + record.getId(), e); } byte[] deleted = recdec.getLatest(result, RecordCf.DATA.bytes, RecordColumn.DELETED.bytes); if ((deleted == null) || (Bytes.toBoolean(deleted))) { // do the create try { return create(record); } catch (RecordExistsException e) { // someone created the record since we checked, we will try again } } else { // do the update try { record = update(record, false, useLatestRecordType); return record; } catch (RecordNotFoundException e) { // some deleted the record since we checked, we will try again } } } throw new RecordException("Create-or-update failed after " + attempts + " attempts, toggling between create and update mode."); } @Override public Record create(Record record) throws RepositoryException { long before = System.currentTimeMillis(); try { checkCreatePreconditions(record); RecordId recordId = record.getId(); if (recordId == null) { recordId = idGenerator.newRecordId(); } byte[] rowId = recordId.toBytes(); try { FieldTypes fieldTypes = typeManager.getFieldTypesSnapshot(); long version = 1L; byte[] oldOccBytes = null; long newOcc = 1L; // If the record existed it would have been deleted. // The version numbering continues from where it has been deleted. Get get = new Get(rowId); get.addColumn(RecordCf.DATA.bytes, RecordColumn.DELETED.bytes); get.addColumn(RecordCf.DATA.bytes, RecordColumn.VERSION.bytes); get.addColumn(RecordCf.DATA.bytes, RecordColumn.OCC.bytes); Result result = recordTable.get(get); if (!result.isEmpty()) { // If the record existed it should have been deleted byte[] recordDeleted = result.getValue(RecordCf.DATA.bytes, RecordColumn.DELETED.bytes); if (recordDeleted != null && !Bytes.toBoolean(recordDeleted)) { throw new RecordExistsException(recordId); } oldOccBytes = result.getValue(RecordCf.DATA.bytes, RecordColumn.OCC.bytes); newOcc = Bytes.toLong(nextOcc(oldOccBytes)); byte[] oldVersion = result.getValue(RecordCf.DATA.bytes, RecordColumn.VERSION.bytes); if (oldVersion != null) { version = Bytes.toLong(oldVersion) + 1; // Make sure any old data gets cleared and old blobs are deleted // This is to cover the failure scenario where a record was deleted, but a failure // occurred before executing the clearData // If this was already done, this is a no-op // Note: since the removal of the row locking, this part could run concurrent with other // threads trying to re-create a record or with a delete still being in progress. This // should be no problem since the clearData will only remove the versions at the old // timestamps, and leave the non-versioned fields untouched. clearData(recordId, null, Bytes.toLong(oldVersion)); } } RecordEvent recordEvent = new RecordEvent(); recordEvent.setType(Type.CREATE); recordEvent.setTableName(getTableName()); if (record.hasAttributes()) { recordEvent.getAttributes().putAll(record.getAttributes()); } Record newRecord = record.cloneRecord(); newRecord.setId(recordId); for (RecordUpdateHook hook : updateHooks) { hook.beforeCreate(newRecord, this, fieldTypes, recordEvent); } Set<BlobReference> referencedBlobs = new HashSet<BlobReference>(); Set<BlobReference> unReferencedBlobs = new HashSet<BlobReference>(); Put put = buildPut(newRecord, version, fieldTypes, recordEvent, referencedBlobs, unReferencedBlobs, newOcc); // Make sure the record type changed flag stays false for a newly // created record recordEvent.setRecordTypeChanged(false); Long newVersion = newRecord.getVersion(); if (newVersion != null) { recordEvent.setVersionCreated(newVersion); } // Reserve blobs so no other records can use them reserveBlobs(null, referencedBlobs); put.add(RecordCf.DATA.bytes, RecordColumn.PAYLOAD.bytes, recordEvent.toJsonBytes()); boolean success = recordTable.checkAndPut(put.getRow(), RecordCf.DATA.bytes, RecordColumn.OCC.bytes, oldOccBytes, put); if (!success) { throw new RecordExistsException(recordId); } // Remove the used blobs from the blobIncubator blobManager.handleBlobReferences(recordId, referencedBlobs, unReferencedBlobs); newRecord.setResponseStatus(ResponseStatus.CREATED); removeUnidirectionalState(newRecord); return newRecord; } catch (IOException e) { throw new RecordException( "Exception occurred while creating record '" + recordId + "' in HBase table", e); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new RecordException( "Exception occurred while creating record '" + recordId + "' in HBase table", e); } catch (BlobException e) { throw new RecordException("Exception occurred while creating record '" + recordId + "'", e); } } finally { metrics.report(Action.CREATE, System.currentTimeMillis() - before); } } /** * Build a Put for inserting a new (blank) record into a Lily repository table. */ public Put buildPut(Record newRecord, long version, FieldTypes fieldTypes, RecordEvent recordEvent, Set<BlobReference> referencedBlobs, Set<BlobReference> unReferencedBlobs, long occ) throws RecordException, InterruptedException, RepositoryException { Record dummyOriginalRecord = newRecord(); Put put = new Put(newRecord.getId().toBytes()); put.add(RecordCf.DATA.bytes, RecordColumn.DELETED.bytes, 1L, Bytes.toBytes(false)); calculateRecordChanges(newRecord, dummyOriginalRecord, version, put, recordEvent, referencedBlobs, unReferencedBlobs, false, fieldTypes); put.add(RecordCf.DATA.bytes, RecordColumn.OCC.bytes, 1L, Bytes.toBytes(occ)); return put; } /** * Removes state from the record which was present on submit, but shouldn't be present in the record * returned to the client. */ private void removeUnidirectionalState(Record record) { record.getFieldsToDelete().clear(); // Clearing the fieldsToDelete of the metadata's is a bit more complex since those objects are immutable Map<QName, Metadata> changedMetadata = null; for (Map.Entry<QName, Metadata> metadataEntry : record.getMetadataMap().entrySet()) { if (metadataEntry.getValue().getFieldsToDelete().size() > 0) { MetadataBuilder builder = new MetadataBuilder(); for (Map.Entry<String, Object> entry : metadataEntry.getValue().getMap().entrySet()) { builder.object(entry.getKey(), entry.getValue()); } if (changedMetadata == null) { changedMetadata = new HashMap<QName, Metadata>(); } changedMetadata.put(metadataEntry.getKey(), builder.build()); } } if (changedMetadata != null) { record.getMetadataMap().putAll(changedMetadata); } } private void checkCreatePreconditions(Record record) throws InvalidRecordException { ArgumentValidator.notNull(record, "record"); if (record.getRecordTypeName() == null) { throw new InvalidRecordException("The recordType cannot be null for a record to be created.", record.getId()); } if (record.getFields().isEmpty()) { throw new InvalidRecordException("Creating an empty record is not allowed", record.getId()); } } @Override public Record update(Record record) throws RepositoryException, InterruptedException { return update(record, false, true); } @Override public Record update(Record record, List<MutationCondition> conditions) throws RepositoryException, InterruptedException { return update(record, false, true, conditions); } @Override public Record update(Record record, boolean updateVersion, boolean useLatestRecordType) throws RepositoryException, InterruptedException { return update(record, updateVersion, useLatestRecordType, null); } @Override public Record update(Record record, boolean updateVersion, boolean useLatestRecordType, List<MutationCondition> conditions) throws RepositoryException, InterruptedException { long before = System.currentTimeMillis(); RecordId recordId = record.getId(); try { if (recordId == null) { throw new InvalidRecordException("The recordId cannot be null for a record to be updated.", record.getId()); } FieldTypes fieldTypes = typeManager.getFieldTypesSnapshot(); // Check if the update is an update of mutable fields if (updateVersion) { try { return updateMutableFields(record, useLatestRecordType, conditions, fieldTypes); } catch (BlobException e) { throw new RecordException("Exception occurred while updating record '" + record.getId() + "'", e); } } else { return updateRecord(record, useLatestRecordType, conditions, fieldTypes); } } finally { metrics.report(Action.UPDATE, System.currentTimeMillis() - before); } } private Record updateRecord(Record record, boolean useLatestRecordType, List<MutationCondition> conditions, FieldTypes fieldTypes) throws RepositoryException { RecordId recordId = record.getId(); try { Pair<Record, byte[]> recordAndOcc = readWithOcc(record.getId(), null, null, fieldTypes); Record originalRecord = new UnmodifiableRecord(recordAndOcc.getV1()); byte[] oldOccBytes = recordAndOcc.getV2(); RecordEvent recordEvent = new RecordEvent(); recordEvent.setType(Type.UPDATE); recordEvent.setTableName(getTableName()); if (record.hasAttributes()) { recordEvent.getAttributes().putAll(record.getAttributes()); } for (RecordUpdateHook hook : updateHooks) { hook.beforeUpdate(record, originalRecord, this, fieldTypes, recordEvent); } Record newRecord = record.cloneRecord(); Put put = new Put(newRecord.getId().toBytes()); Set<BlobReference> referencedBlobs = new HashSet<BlobReference>(); Set<BlobReference> unReferencedBlobs = new HashSet<BlobReference>(); long newVersion = originalRecord.getVersion() == null ? 1 : originalRecord.getVersion() + 1; // Check the mutation conditions. // It is important that we do this before checking if the record needs updating at all: otherwise, // another client might already have performed the update we intended to do, which is problematic // in cases like incrementing a counter (the counter should be updated twice, not once). Record conditionsResponse = MutationConditionVerifier.checkConditions(originalRecord, conditions, this, record); if (conditionsResponse != null) { return conditionsResponse; } if (calculateRecordChanges(newRecord, originalRecord, newVersion, put, recordEvent, referencedBlobs, unReferencedBlobs, useLatestRecordType, fieldTypes)) { // Reserve blobs so no other records can use them reserveBlobs(record.getId(), referencedBlobs); put.add(RecordCf.DATA.bytes, RecordColumn.PAYLOAD.bytes, recordEvent.toJsonBytes()); put.add(RecordCf.DATA.bytes, RecordColumn.OCC.bytes, 1L, nextOcc(oldOccBytes)); boolean occSuccess = recordTable.checkAndPut(put.getRow(), RecordCf.DATA.bytes, RecordColumn.OCC.bytes, oldOccBytes, put); if (!occSuccess) { throw new ConcurrentRecordUpdateException(recordId); } // Remove the used blobs from the blobIncubator and delete unreferenced blobs from the blobstore blobManager.handleBlobReferences(recordId, referencedBlobs, unReferencedBlobs); newRecord.setResponseStatus(ResponseStatus.UPDATED); } else { newRecord.setResponseStatus(ResponseStatus.UP_TO_DATE); } removeUnidirectionalState(newRecord); return newRecord; } catch (IOException e) { throw new RecordException("Exception occurred while updating record '" + recordId + "' on HBase table", e); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new RecordException("Exception occurred while updating record '" + recordId + "' on HBase table", e); } catch (BlobException e) { throw new RecordException( "Exception occurred while putting updated record '" + recordId + "' on HBase table", e); } } // Calculates the changes that are to be made on the record-row and puts // this information on the Put object and the RecordEvent private boolean calculateRecordChanges(Record record, Record originalRecord, Long version, Put put, RecordEvent recordEvent, Set<BlobReference> referencedBlobs, Set<BlobReference> unReferencedBlobs, boolean useLatestRecordType, FieldTypes fieldTypes) throws InterruptedException, RepositoryException { final QName newRecordTypeName; final Long newRecordTypeVersion; if (record.getRecordTypeName() == null) { newRecordTypeName = originalRecord.getRecordTypeName(); newRecordTypeVersion = null; } else { newRecordTypeName = record.getRecordTypeName(); newRecordTypeVersion = useLatestRecordType ? null : record.getRecordTypeVersion(); } RecordType recordType = typeManager.getRecordTypeByName(newRecordTypeName, newRecordTypeVersion); // Check which fields have changed EnumSet<Scope> changedScopes = calculateChangedFields(record, originalRecord, recordType, version, put, recordEvent, referencedBlobs, unReferencedBlobs, fieldTypes); // If no versioned fields have changed, keep the original version boolean versionedFieldsHaveChanged = changedScopes.contains(Scope.VERSIONED) || changedScopes.contains(Scope.VERSIONED_MUTABLE); if (!versionedFieldsHaveChanged) { version = originalRecord.getVersion(); } // The provided recordTypeVersion could have been null, so the latest version of the recordType was taken // and we need to know which version that is Long actualRecordTypeVersion = newRecordTypeVersion == null ? recordType.getVersion() : newRecordTypeVersion; boolean recordTypeHasChanged = !newRecordTypeName.equals(originalRecord.getRecordTypeName()) || !actualRecordTypeVersion.equals(originalRecord.getRecordTypeVersion()); boolean fieldsHaveChanged = !changedScopes.isEmpty(); boolean changed = needToApplyChanges(newRecordTypeVersion, useLatestRecordType, fieldsHaveChanged); if (changed) { if ((recordTypeHasChanged && fieldsHaveChanged) || (recordTypeHasChanged && !useLatestRecordType)) { recordEvent.setRecordTypeChanged(true); // If NON_VERSIONED is in the changed scopes, the record type will already have been set as part // of calculateChangedFields, and the result would be double key-values in the Put object if (!changedScopes.contains(Scope.NON_VERSIONED)) { put.add(RecordCf.DATA.bytes, RecordColumn.NON_VERSIONED_RT_ID.bytes, 1L, recordType.getId().getBytes()); put.add(RecordCf.DATA.bytes, RecordColumn.NON_VERSIONED_RT_VERSION.bytes, 1L, Bytes.toBytes(actualRecordTypeVersion)); changedScopes.add(Scope.NON_VERSIONED); // because the record type version changed } } // Always set the record type on the record since the requested // record type could have been given without a version number record.setRecordType(newRecordTypeName, actualRecordTypeVersion); if (version != null) { byte[] versionBytes = Bytes.toBytes(version); put.add(RecordCf.DATA.bytes, RecordColumn.VERSION.bytes, 1L, versionBytes); } validateRecord(record, originalRecord, recordType, fieldTypes); } setRecordTypesAfterUpdate(record, originalRecord, changedScopes); // Always set the version on the record. If no fields were changed this // will give the latest version in the repository record.setVersion(version); if (versionedFieldsHaveChanged) { recordEvent.setVersionCreated(version); } return changed; } private boolean needToApplyChanges(Long newRecordTypeVersion, boolean useLatestRecordType, boolean fieldsHaveChanged) { if (fieldsHaveChanged) { // this is the most obvious case. It also results in record type changes to also be applied. return true; } else if (newRecordTypeVersion == null) { // no field changes and no record type version set, we do nothing. return false; } else if (!useLatestRecordType) { // no field changes and a record type version is set without requesting the "latest record type" to be used. // This should trigger an actual update of the record type. return true; } else return false; } private void setRecordTypesAfterUpdate(Record record, Record originalRecord, Set<Scope> changedScopes) { // The returned record object after an update should always contain complete record type information for // all the scopes for (Scope scope : Scope.values()) { // For any unchanged or non-existing scope, we reset the record type information to the one of the // original record, so that the returned record object corresponds to the repository state (= same // as when one would do a fresh read) // // Copy over the original record type of a scope if: // - the scope was unchanged. If it was changed, the record type will already have been filled in // by calculateRecordChanges. // - for the non-versioned scope, only copy it over if none of the scopes changed, because the // record type of the non-versioned scope is always brought up to date in case any scope is changed if (!changedScopes.contains(scope) && (scope != Scope.NON_VERSIONED || changedScopes.isEmpty())) { record.setRecordType(scope, originalRecord.getRecordTypeName(scope), originalRecord.getRecordTypeVersion(scope)); } } } private void validateRecord(Record record, Record originalRecord, RecordType recordType, FieldTypes fieldTypes) throws TypeException, InvalidRecordException, InterruptedException { // Check mandatory fields Collection<FieldTypeEntry> fieldTypeEntries = recordType.getFieldTypeEntries(); List<QName> fieldsToDelete = record.getFieldsToDelete(); for (FieldTypeEntry fieldTypeEntry : fieldTypeEntries) { if (fieldTypeEntry.isMandatory()) { FieldType fieldType = fieldTypes.getFieldType(fieldTypeEntry.getFieldTypeId()); QName fieldName = fieldType.getName(); if (fieldsToDelete.contains(fieldName)) { throw new InvalidRecordException("Field: '" + fieldName + "' is mandatory.", record.getId()); } if (!record.hasField(fieldName) && !originalRecord.hasField(fieldName)) { throw new InvalidRecordException("Field: '" + fieldName + "' is mandatory.", record.getId()); } } } } // Calculates which fields have changed and updates the record types of the scopes that have changed fields private EnumSet<Scope> calculateChangedFields(Record record, Record originalRecord, RecordType recordType, Long version, Put put, RecordEvent recordEvent, Set<BlobReference> referencedBlobs, Set<BlobReference> unReferencedBlobs, FieldTypes fieldTypes) throws InterruptedException, RepositoryException { Map<QName, Object> originalFields = originalRecord.getFields(); EnumSet<Scope> changedScopes = EnumSet.noneOf(Scope.class); Map<QName, Object> fields = getFieldsToUpdate(record); changedScopes.addAll(calculateUpdateFields(record, fields, record.getMetadataMap(), originalFields, originalRecord.getMetadataMap(), null, version, put, recordEvent, referencedBlobs, unReferencedBlobs, false, fieldTypes)); for (BlobReference referencedBlob : referencedBlobs) { referencedBlob.setRecordId(record.getId()); } for (BlobReference unReferencedBlob : unReferencedBlobs) { unReferencedBlob.setRecordId(record.getId()); } // Update record types for (Scope scope : changedScopes) { long versionOfRTField = version; if (Scope.NON_VERSIONED.equals(scope)) { versionOfRTField = 1L; // For non-versioned fields the record type is always stored at version 1. } // Only update the recordTypeNames and versions if they have indeed changed QName originalScopeRecordTypeName = originalRecord.getRecordTypeName(scope); if (originalScopeRecordTypeName == null) { put.add(RecordCf.DATA.bytes, RECORD_TYPE_ID_QUALIFIERS.get(scope), versionOfRTField, recordType.getId().getBytes()); put.add(RecordCf.DATA.bytes, RECORD_TYPE_VERSION_QUALIFIERS.get(scope), versionOfRTField, Bytes.toBytes(recordType.getVersion())); } else { RecordType originalScopeRecordType = typeManager.getRecordTypeByName(originalScopeRecordTypeName, originalRecord.getRecordTypeVersion(scope)); if (!recordType.getId().equals(originalScopeRecordType.getId())) { put.add(RecordCf.DATA.bytes, RECORD_TYPE_ID_QUALIFIERS.get(scope), versionOfRTField, recordType.getId().getBytes()); } if (!recordType.getVersion().equals(originalScopeRecordType.getVersion())) { put.add(RecordCf.DATA.bytes, RECORD_TYPE_VERSION_QUALIFIERS.get(scope), versionOfRTField, Bytes.toBytes(recordType.getVersion())); } } record.setRecordType(scope, recordType.getName(), recordType.getVersion()); } return changedScopes; } // Returns a map (fieldname -> field) of all fields that are indicated by the record to be updated. // The map also includes the fields that need to be deleted. Their name is mapped onto the delete marker. private Map<QName, Object> getFieldsToUpdate(Record record) { // Work with a copy of the map Map<QName, Object> fields = new HashMap<QName, Object>(); fields.putAll(record.getFields()); for (QName qName : record.getFieldsToDelete()) { fields.put(qName, FieldFlags.getDeleteMarker()); } return fields; } // Checks for each field if it is different from its previous value and indeed needs to be updated. private Set<Scope> calculateUpdateFields(Record parentRecord, Map<QName, Object> fields, Map<QName, Metadata> fieldMetadata, Map<QName, Object> originalFields, Map<QName, Metadata> originalFieldMetadata, Map<QName, Object> originalNextFields, Long version, Put put, RecordEvent recordEvent, Set<BlobReference> referencedBlobs, Set<BlobReference> unReferencedBlobs, boolean mutableUpdate, FieldTypes fieldTypes) throws InterruptedException, RepositoryException { Set<Scope> changedScopes = EnumSet.noneOf(Scope.class); // In the below algorithm, the following facts are good to know about metadata: // - there can only be field metadata when there is a field value // - metadata is updated in the same way as fields: only updated values need to be specified, and deletes // are explicit (Metadata.deletedFields). // - it is possible/supported that only metadata changes, and that the field value stayed the same. Thus it // can be that a Record object contains metadata for a field but no field value (because that stays // the same). For versioned fields, this causes a new version. // Map containing the actual new metadata that needs to be applied: thus the merged view of the old // and new metadata. In case the metadata has not changed, there will be not an entry in here, and // the metadata from the old field needs to be copied. Map<QName, Metadata> newMetadata = new HashMap<QName, Metadata>(); Iterator<Entry<QName, Metadata>> fieldMetadataIt = fieldMetadata.entrySet().iterator(); while (fieldMetadataIt.hasNext()) { Entry<QName, Metadata> entry = fieldMetadataIt.next(); // If it's not a deleted field if (!isDeleteMarker(fields.get(entry.getKey()))) { // If the metadata has changed if (entry.getValue().updates(originalFieldMetadata.get(entry.getKey()))) { boolean needMetadata; // And the field itself didn't change if (!fields.containsKey(entry.getKey())) { // And if the field existed before (you can't add metadata without having a field value) if (originalFields.containsKey(entry.getKey())) { // Then add the field in the fields map so that it will be treated in the loop that // handles updated field values below fields.put(entry.getKey(), METADATA_ONLY_UPDATE); needMetadata = true; } else { // No new or old field value: can't have metadata for a field without a value needMetadata = false; // Remove this invalid metadata from the record object (the idea being that the record // object returned to the user should correspond to persisted state). fieldMetadataIt.remove(); } } else { // Both field & metadata changed needMetadata = true; } if (needMetadata) { // Now that we've done all the checks to determine we need the metadata, calculate it newMetadata.put(entry.getKey(), mergeMetadata(entry.getValue(), originalFieldMetadata.get(entry.getKey()))); } } } else { // Field is deleted. // Remove this invalid metadata from the record object (the idea being that the record // object returned to the user should correspond to persisted state). fieldMetadataIt.remove(); } } FieldValueWriter fieldValueWriter = newFieldValueWriter(put, parentRecord); for (Entry<QName, Object> field : fields.entrySet()) { QName fieldName = field.getKey(); Object newValue = field.getValue(); boolean fieldIsNewOrDeleted = !originalFields.containsKey(fieldName); Object originalValue = originalFields.get(fieldName); if (!(((newValue == null) && (originalValue == null)) // Don't update if both are null || (isDeleteMarker(newValue) && fieldIsNewOrDeleted) // Don't delete if it doesn't exist || (newValue.equals(originalValue))) // Don't update if they didn't change || newMetadata.containsKey(field.getKey())) { // But do update if the metadata changed FieldTypeImpl fieldType = (FieldTypeImpl) fieldTypes.getFieldType(fieldName); Scope scope = fieldType.getScope(); boolean metadataOnlyUpdate = false; if (newValue == METADATA_ONLY_UPDATE) { // The metadata was updated, but the field itself not metadataOnlyUpdate = true; newValue = originalFields.get(fieldName); } // Either use new or inherit old metadata (newMetadata map contains the merged metadata) Metadata metadata = newMetadata.get(fieldName); if (metadata == null) { metadata = originalFieldMetadata.get(fieldName); } if (!metadataOnlyUpdate) { // Check if the newValue contains blobs Set<BlobReference> newReferencedBlobs = getReferencedBlobs(fieldType, newValue); referencedBlobs.addAll(newReferencedBlobs); // Check if the previousValue contained blobs which should be deleted since they are no longer used // In case of a mutable update, it is checked later if no other versions use the blob before deciding to delete it if (Scope.NON_VERSIONED.equals(scope) || (mutableUpdate && Scope.VERSIONED_MUTABLE.equals(scope))) { if (originalValue != null) { Set<BlobReference> previouslyReferencedBlobs = getReferencedBlobs(fieldType, originalValue); previouslyReferencedBlobs.removeAll(newReferencedBlobs); unReferencedBlobs.addAll(previouslyReferencedBlobs); } } } // Set the value if (Scope.NON_VERSIONED.equals(scope)) { fieldValueWriter.addFieldValue(fieldType, newValue, metadata, 1L); } else { fieldValueWriter.addFieldValue(fieldType, newValue, metadata, version); // If it is a mutable update and the next version of the field was the same as the one that is being updated, // the original value needs to be copied to that next version (due to sparseness of the table). if (originalNextFields != null && !fieldIsNewOrDeleted && originalNextFields.containsKey(fieldName)) { copyValueToNextVersionIfNeeded(parentRecord, version, fieldValueWriter, originalNextFields, fieldName, originalValue, fieldTypes); } } changedScopes.add(scope); recordEvent.addUpdatedField(fieldType.getId()); } } return changedScopes; } /** * Merges old & new metadata (metadata supports partial updating just like records). * * <p>Does not modify its arguments, but might return one of them.</p> */ private Metadata mergeMetadata(Metadata newMetadata, Metadata oldMetadata) { if (oldMetadata == null || oldMetadata.isEmpty()) { return removeFieldsToDelete(newMetadata); } if (newMetadata == null || newMetadata.isEmpty()) { return oldMetadata; } MetadataBuilder result = new MetadataBuilder(); // Run over the old values for (Entry<String, Object> entry : oldMetadata.getMap().entrySet()) { // If it's not deleted if (!newMetadata.getFieldsToDelete().contains(entry.getKey())) { // If it's not updated if (!newMetadata.contains(entry.getKey())) { result.object(entry.getKey(), entry.getValue()); } } } // Run over the new values for (Entry<String, Object> entry : newMetadata.getMap().entrySet()) { result.object(entry.getKey(), entry.getValue()); } return result.build(); } /** * Returns a metadata object with the fieldsToDelete removed. */ private Metadata removeFieldsToDelete(Metadata metadata) { if (metadata.getFieldsToDelete().size() > 0) { MetadataBuilder builder = new MetadataBuilder(); for (Map.Entry<String, Object> entry : metadata.getMap().entrySet()) { builder.object(entry.getKey(), entry.getValue()); } return builder.build(); } else { return metadata; } } public static void writeMetadataWithLengthSuffix(Metadata metadata, DataOutput output) { DataOutput tmp = new DataOutputImpl(); MetadataSerDeser.write(metadata, tmp); byte[] metadataBytes = tmp.toByteArray(); output.writeBytes(metadataBytes); output.writeInt(metadataBytes.length); } private boolean isDeleteMarker(Object fieldValue) { return (fieldValue instanceof byte[]) && FieldFlags.isDeletedField(((byte[]) fieldValue)[0]); } private Record updateMutableFields(Record record, boolean latestRecordType, List<MutationCondition> conditions, FieldTypes fieldTypes) throws RepositoryException { Record newRecord = record.cloneRecord(); RecordId recordId = record.getId(); Long version = record.getVersion(); if (version == null) { throw new InvalidRecordException("The version of the record cannot be null to update mutable fields", record.getId()); } try { Map<QName, Object> fields = getFieldsToUpdate(record); fields = filterMutableFields(fields, fieldTypes); Pair<Record, byte[]> recordAndOcc = readWithOcc(recordId, version, null, fieldTypes); Record originalRecord = new UnmodifiableRecord(recordAndOcc.getV1()); byte[] oldOccBytes = recordAndOcc.getV2(); Map<QName, Object> originalFields = filterMutableFields(originalRecord.getFields(), fieldTypes); Record originalNextRecord = null; Map<QName, Object> originalNextFields = null; try { originalNextRecord = read(recordId, version + 1, null, fieldTypes); originalNextFields = filterMutableFields(originalNextRecord.getFields(), fieldTypes); } catch (VersionNotFoundException e) { // There is no next version of the record } Put put = new Put(recordId.toBytes()); Set<BlobReference> referencedBlobs = new HashSet<BlobReference>(); Set<BlobReference> unReferencedBlobs = new HashSet<BlobReference>(); RecordEvent recordEvent = new RecordEvent(); recordEvent.setType(Type.UPDATE); recordEvent.setTableName(getTableName()); recordEvent.setVersionUpdated(version); for (RecordUpdateHook hook : updateHooks) { hook.beforeUpdate(record, originalRecord, this, fieldTypes, recordEvent); } Set<Scope> changedScopes = calculateUpdateFields(record, fields, record.getMetadataMap(), originalFields, originalRecord.getMetadataMap(), originalNextFields, version, put, recordEvent, referencedBlobs, unReferencedBlobs, true, fieldTypes); for (BlobReference referencedBlob : referencedBlobs) { referencedBlob.setRecordId(recordId); } for (BlobReference unReferencedBlob : unReferencedBlobs) { unReferencedBlob.setRecordId(recordId); } if (!changedScopes.isEmpty()) { // Check the conditions after establishing that the record really needs updating, this makes the // conditional update operation idempotent. Record conditionsRecord = MutationConditionVerifier.checkConditions(originalRecord, conditions, this, record); if (conditionsRecord != null) { return conditionsRecord; } // Update the record types // If no record type is specified explicitly, use the current one of the non-versioned scope QName recordTypeName = record.getRecordTypeName() != null ? record.getRecordTypeName() : originalRecord.getRecordTypeName(); Long recordTypeVersion; if (latestRecordType) { recordTypeVersion = null; } else if (record.getRecordTypeName() == null) { recordTypeVersion = originalRecord.getRecordTypeVersion(); } else { recordTypeVersion = record.getRecordTypeVersion(); } RecordType recordType = typeManager.getRecordTypeByName(recordTypeName, recordTypeVersion); // Update the mutable record type in the record object Scope mutableScope = Scope.VERSIONED_MUTABLE; newRecord.setRecordType(mutableScope, recordType.getName(), recordType.getVersion()); // If the record type changed, update it on the record table QName originalMutableScopeRecordTypeName = originalRecord.getRecordTypeName(mutableScope); if (originalMutableScopeRecordTypeName == null) { // There was no initial mutable record type yet put.add(RecordCf.DATA.bytes, RECORD_TYPE_ID_QUALIFIERS.get(mutableScope), version, recordType.getId().getBytes()); put.add(RecordCf.DATA.bytes, RECORD_TYPE_VERSION_QUALIFIERS.get(mutableScope), version, Bytes.toBytes(recordType.getVersion())); } else { RecordType originalMutableScopeRecordType = typeManager.getRecordTypeByName( originalMutableScopeRecordTypeName, originalRecord.getRecordTypeVersion(mutableScope)); if (!recordType.getId().equals(originalMutableScopeRecordType.getId())) { // If the next record version had the same record type name, copy the original value to that one if (originalNextRecord != null && originalMutableScopeRecordType.getName() .equals(originalNextRecord.getRecordTypeName(mutableScope))) { put.add(RecordCf.DATA.bytes, RECORD_TYPE_ID_QUALIFIERS.get(mutableScope), version + 1, originalMutableScopeRecordType.getId().getBytes()); } put.add(RecordCf.DATA.bytes, RECORD_TYPE_ID_QUALIFIERS.get(mutableScope), version, recordType.getId().getBytes()); } if (!recordType.getVersion().equals(originalMutableScopeRecordType.getVersion())) { // If the next record version had the same record type version, copy the original value to that one if (originalNextRecord != null && originalMutableScopeRecordType.getVersion() .equals(originalNextRecord.getRecordTypeVersion(mutableScope))) { put.add(RecordCf.DATA.bytes, RECORD_TYPE_ID_QUALIFIERS.get(mutableScope), version + 1, Bytes.toBytes(originalMutableScopeRecordType.getVersion())); } put.add(RecordCf.DATA.bytes, RECORD_TYPE_VERSION_QUALIFIERS.get(mutableScope), version, Bytes.toBytes(recordType.getVersion())); } } // Validate if the new values for the record are valid wrt the recordType (e.g. mandatory fields) validateRecord(newRecord, originalRecord, recordType, fieldTypes); // Reserve blobs so no other records can use them reserveBlobs(record.getId(), referencedBlobs); put.add(RecordCf.DATA.bytes, RecordColumn.PAYLOAD.bytes, 1L, recordEvent.toJsonBytes()); put.add(RecordCf.DATA.bytes, RecordColumn.OCC.bytes, 1L, nextOcc(oldOccBytes)); boolean occSuccess = recordTable.checkAndPut(put.getRow(), RecordCf.DATA.bytes, RecordColumn.OCC.bytes, oldOccBytes, put); if (!occSuccess) { throw new ConcurrentRecordUpdateException(recordId); } // The unReferencedBlobs could still be in use in another version of the mutable field, // therefore we filter them first unReferencedBlobs = filterReferencedBlobs(recordId, unReferencedBlobs, version); // Remove the used blobs from the blobIncubator blobManager.handleBlobReferences(recordId, referencedBlobs, unReferencedBlobs); newRecord.setResponseStatus(ResponseStatus.UPDATED); } else { newRecord.setResponseStatus(ResponseStatus.UP_TO_DATE); } setRecordTypesAfterUpdate(record, originalRecord, changedScopes); } catch (IOException e) { throw new RecordException("Exception occurred while updating record '" + recordId + "' on HBase table", e); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new RecordException("Exception occurred while updating record '" + recordId + "' on HBase table", e); } removeUnidirectionalState(newRecord); return newRecord; } private Map<QName, Object> filterMutableFields(Map<QName, Object> fields, FieldTypes fieldTypes) throws RecordException, TypeException, InterruptedException { Map<QName, Object> mutableFields = new HashMap<QName, Object>(); for (Entry<QName, Object> field : fields.entrySet()) { FieldType fieldType = fieldTypes.getFieldType(field.getKey()); if (Scope.VERSIONED_MUTABLE.equals(fieldType.getScope())) { mutableFields.put(field.getKey(), field.getValue()); } } return mutableFields; } /** * If the original value is the same as for the next version * this means that the cell at the next version does not contain any value yet, * and the record relies on what is in the previous cell's version. * The original value needs to be copied into it. Otherwise we loose that value. */ private void copyValueToNextVersionIfNeeded(Record parentRecord, Long version, FieldValueWriter fieldValueWriter, Map<QName, Object> originalNextFields, QName fieldName, Object originalValue, FieldTypes fieldTypes) throws RepositoryException, InterruptedException { Object originalNextValue = originalNextFields.get(fieldName); if ((originalValue == null && originalNextValue == null) || originalValue.equals(originalNextValue)) { FieldTypeImpl fieldType = (FieldTypeImpl) fieldTypes.getFieldType(fieldName); fieldValueWriter.addFieldValue(fieldType, originalValue, null, version + 1); } } @Override public void delete(RecordId recordId) throws RepositoryException { delete(recordId, null); } @Override public Record delete(RecordId recordId, List<MutationCondition> conditions) throws RepositoryException { return delete(recordId, conditions, null); } @Override public void delete(Record record) throws RepositoryException { delete(record.getId(), null, record.hasAttributes() ? record.getAttributes() : null); } private Record delete(RecordId recordId, List<MutationCondition> conditions, Map<String, String> attributes) throws RepositoryException { ArgumentValidator.notNull(recordId, "recordId"); long before = System.currentTimeMillis(); byte[] rowId = recordId.toBytes(); try { // We need to read the original record in order to put the delete marker in the non-versioned fields. // Throw RecordNotFoundException if there is no record to be deleted FieldTypes fieldTypes = typeManager.getFieldTypesSnapshot(); // The existing record is read with authorization disabled, because when deleting a full record we // must make sure we can delete all the fields in it. Pair<Record, byte[]> recordAndOcc = readWithOcc(recordId, null, null, fieldTypes, true); recordAndOcc.getV1().setAttributes(attributes); Record originalRecord = new UnmodifiableRecord(recordAndOcc.getV1()); // If the record contains any field with scope != non-versioned, do not allow to delete it if // authorization is active. This is because access to these fields will not be validated by the // first Put call below, and hence the clearData afterwards might fail, leaving us with a half-deleted // record. if (AuthorizationContextHolder.getCurrentContext() != null) { for (QName field : originalRecord.getFields().keySet()) { if (fieldTypes.getFieldType(field).getScope() != Scope.NON_VERSIONED) { throw new RepositoryException("Deleting records with versioned or versioned-mutable fields " + "is not supported when authorization is active."); } } } byte[] oldOcc = recordAndOcc.getV2(); RecordEvent recordEvent = new RecordEvent(); recordEvent.setType(Type.DELETE); recordEvent.setTableName(getTableName()); if (attributes != null && !attributes.isEmpty()) { recordEvent.getAttributes().putAll(attributes); } for (RecordUpdateHook hook : updateHooks) { hook.beforeDelete(originalRecord, this, fieldTypes, recordEvent); } if (conditions != null) { Record conditionsRecord = MutationConditionVerifier.checkConditions(originalRecord, conditions, this, null); if (conditionsRecord != null) { return conditionsRecord; } } Put put = new Put(rowId); // Mark the record as deleted put.add(RecordCf.DATA.bytes, RecordColumn.DELETED.bytes, 1L, Bytes.toBytes(true)); // Put the delete marker in the non-versioned fields instead of deleting their columns in the clearData call // This is needed to avoid non-versioned fields to be lost due to the hbase delete thombstone // See trac ticket http://dev.outerthought.org/trac/outerthought_lilyproject/ticket/297 Map<QName, Object> fields = originalRecord.getFields(); for (Entry<QName, Object> fieldEntry : fields.entrySet()) { FieldTypeImpl fieldType = (FieldTypeImpl) fieldTypes.getFieldType(fieldEntry.getKey()); if (Scope.NON_VERSIONED == fieldType.getScope()) { put.add(RecordCf.DATA.bytes, fieldType.getQualifier(), 1L, FieldFlags.getDeleteMarker()); } } put.add(RecordCf.DATA.bytes, RecordColumn.PAYLOAD.bytes, recordEvent.toJsonBytes()); put.add(RecordCf.DATA.bytes, RecordColumn.OCC.bytes, 1L, nextOcc(oldOcc)); // Hint towards the NGDATA HBase authorization coprocessor: for deletes, we need write access to all // columns, since otherwise we could end up with half-deleted records. The default behavior for puts // is to silently filter columns from the Put for which the user has no write permission. put.setAttribute(HBaseAuthzUtil.FILTER_PUT_ATT, Bytes.toBytes("f")); boolean occSuccess = recordTable.checkAndPut(put.getRow(), RecordCf.DATA.bytes, RecordColumn.OCC.bytes, oldOcc, put); if (!occSuccess) { throw new ConcurrentRecordUpdateException(recordId); } // Clear the old data and delete any referenced blobs clearData(recordId, originalRecord, originalRecord.getVersion()); } catch (IOException e) { throw new RecordException("Exception occurred while deleting record '" + recordId + "' on HBase table", e); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new RecordException("Exception occurred while deleting record '" + recordId + "' on HBase table", e); } finally { long after = System.currentTimeMillis(); metrics.report(Action.DELETE, (after - before)); } return null; } /** * Get the next OCC (Optimistic Concurrency Control) value, or an initialized occ value if the current * value is null. * * The handling of null values is needed for working with repositories that were created before the OCC * was in place. * * @param occValue current occ value */ protected static byte[] nextOcc(byte[] occValue) { if (occValue == null) { return Bytes.toBytes(1L); } byte[] newOcc = new byte[occValue.length]; System.arraycopy(occValue, 0, newOcc, 0, occValue.length); return Bytes.incrementBytes(newOcc, 1L); } // Clear all data of the recordId until the latest record version (included) // And delete any referred blobs private void clearData(RecordId recordId, Record originalRecord, Long upToVersion) throws IOException, RepositoryException, InterruptedException { Get get = new Get(recordId.toBytes()); get.addFamily(RecordCf.DATA.bytes); get.setFilter(new ColumnPrefixFilter(new byte[] { RecordColumn.DATA_PREFIX })); // Only read versions that exist(ed) at the time the record was deleted, since this code could // run concurrently with the re-creation of the same record. if (upToVersion != null) { get.setTimeRange(1 /* inclusive */, upToVersion + 1 /* exclusive */); } else { get.setTimeRange(1, 2); } get.setMaxVersions(); Result result = recordTable.get(get); if (result != null && !result.isEmpty()) { boolean dataToDelete = false; Delete delete = new Delete(recordId.toBytes()); Set<BlobReference> blobsToDelete = new HashSet<BlobReference>(); NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> map = result.getMap(); Set<Entry<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>>> familiesSet = map.entrySet(); for (Entry<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> family : familiesSet) { if (Arrays.equals(RecordCf.DATA.bytes, family.getKey())) { NavigableMap<byte[], NavigableMap<Long, byte[]>> columnsSet = family.getValue(); for (Entry<byte[], NavigableMap<Long, byte[]>> column : columnsSet.entrySet()) { try { byte[] columnQualifier = column.getKey(); SchemaId schemaId = new SchemaIdImpl( Bytes.tail(columnQualifier, columnQualifier.length - 1)); FieldType fieldType = typeManager.getFieldTypeById(schemaId); ValueType valueType = fieldType.getValueType(); NavigableMap<Long, byte[]> cells = column.getValue(); Set<Entry<Long, byte[]>> cellsSet = cells.entrySet(); for (Entry<Long, byte[]> cell : cellsSet) { // Get blobs to delete if (valueType.getDeepestValueType() instanceof BlobValueType) { Object blobValue = null; if (fieldType.getScope() == Scope.NON_VERSIONED) { // Read the blob value from the original record, // since the delete marker has already been put in the field by the delete call if (originalRecord != null) { blobValue = originalRecord.getField(fieldType.getName()); } } else { byte[] value = cell.getValue(); if (!isDeleteMarker(value)) { blobValue = valueType.read(EncodingUtil.stripPrefix(value)); } } try { if (blobValue != null) { blobsToDelete.addAll( getReferencedBlobs((FieldTypeImpl) fieldType, blobValue)); } } catch (BlobException e) { log.warn("Failure occurred while clearing blob data", e); // We do a best effort here } } // Get cells to delete // Only delete if in NON_VERSIONED scope // The NON_VERSIONED fields will get filled in with a delete marker // This is needed to avoid non-versioned fields to be lost due to the hbase delete thombstone // See trac ticket http://dev.outerthought.org/trac/outerthought_lilyproject/ticket/297 if (fieldType.getScope() != Scope.NON_VERSIONED) { delete.deleteColumn(RecordCf.DATA.bytes, columnQualifier, cell.getKey()); } dataToDelete = true; } } catch (FieldTypeNotFoundException e) { log.warn("Failure occurred while clearing blob data", e); // We do a best effort here } catch (TypeException e) { log.warn("Failure occurred while clearing blob data", e); // We do a best effort here } } } else { //skip } } // Delete the blobs blobManager.handleBlobReferences(recordId, null, blobsToDelete); // Delete data if (dataToDelete && upToVersion != null) { // Avoid a delete action when no data was found to delete // Do not delete the NON-VERSIONED record type column. // If the thombstone was not processed yet (major compaction) // a re-creation of the record would then loose its record type since the NON-VERSIONED // field is always stored at timestamp 1L // Re-creating the record will always overwrite the (NON-VERSIONED) record type. // So, there is no risk of old record type information ending up in the new record. delete.deleteColumns(RecordCf.DATA.bytes, RecordColumn.VERSIONED_RT_ID.bytes, upToVersion); delete.deleteColumns(RecordCf.DATA.bytes, RecordColumn.VERSIONED_RT_VERSION.bytes, upToVersion); delete.deleteColumns(RecordCf.DATA.bytes, RecordColumn.VERSIONED_MUTABLE_RT_ID.bytes, upToVersion); delete.deleteColumns(RecordCf.DATA.bytes, RecordColumn.VERSIONED_MUTABLE_RT_VERSION.bytes, upToVersion); recordTable.delete(delete); } } } private Set<BlobReference> getReferencedBlobs(FieldTypeImpl fieldType, Object value) throws BlobException { HashSet<BlobReference> referencedBlobs = new HashSet<BlobReference>(); ValueType valueType = fieldType.getValueType(); if ((valueType.getDeepestValueType() instanceof BlobValueType) && !isDeleteMarker(value)) { Set<Object> values = valueType.getValues(value); for (Object object : values) { referencedBlobs.add(new BlobReference((Blob) object, null, fieldType)); } } return referencedBlobs; } private void reserveBlobs(RecordId recordId, Set<BlobReference> referencedBlobs) throws IOException, InvalidRecordException { if (!referencedBlobs.isEmpty()) { // Check if the blob is newly uploaded Set<BlobReference> failedReservations = blobManager.reserveBlobs(referencedBlobs); // If not, filter those that are already used by the record failedReservations = filterReferencedBlobs(recordId, failedReservations, null); if (!failedReservations.isEmpty()) { throw new InvalidRecordException("Record references blobs which are not available for use", recordId); } } } // Checks the set of blobs and returns a subset of those blobs which are not referenced anymore private Set<BlobReference> filterReferencedBlobs(RecordId recordId, Set<BlobReference> blobs, Long ignoreVersion) throws IOException { if (recordId == null) { return blobs; } Set<BlobReference> unReferencedBlobs = new HashSet<BlobReference>(); for (BlobReference blobReference : blobs) { FieldTypeImpl fieldType = (FieldTypeImpl) blobReference.getFieldType(); byte[] recordIdBytes = recordId.toBytes(); ValueType valueType = fieldType.getValueType(); Get get = new Get(recordIdBytes); get.addColumn(RecordCf.DATA.bytes, fieldType.getQualifier()); byte[] valueToCompare = Bytes.toBytes(valueType.getNestingLevel()); // Note, if a encoding of the BlobValueType is added, this might have to change. valueToCompare = Bytes.add(valueToCompare, blobReference.getBlob().getValue()); WritableByteArrayComparable valueComparator = new ContainsValueComparator(valueToCompare); Filter filter = new SingleColumnValueFilter(RecordCf.DATA.bytes, fieldType.getQualifier(), CompareOp.EQUAL, valueComparator); get.setFilter(filter); Result result = recordTable.get(get); if (result.isEmpty()) { unReferencedBlobs.add(blobReference); } else { if (ignoreVersion != null) { boolean stillReferenced = false; List<KeyValue> column = result.getColumn(RecordCf.DATA.bytes, fieldType.getQualifier()); for (KeyValue keyValue : column) { if (keyValue.getTimestamp() != ignoreVersion) { stillReferenced = true; break; } } if (!stillReferenced) { unReferencedBlobs.add(blobReference); } } } } return unReferencedBlobs; } @Override public Set<RecordId> getVariants(RecordId recordId) throws RepositoryException { byte[] masterRecordIdBytes = recordId.getMaster().toBytes(); FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL); filterList.addFilter(new PrefixFilter(masterRecordIdBytes)); filterList.addFilter(REAL_RECORDS_FILTER); Scan scan = new Scan(masterRecordIdBytes, filterList); scan.addColumn(RecordCf.DATA.bytes, RecordColumn.DELETED.bytes); Set<RecordId> recordIds = new HashSet<RecordId>(); try { ResultScanner scanner = recordTable.getScanner(scan); Result result; while ((result = scanner.next()) != null) { RecordId id = idGenerator.fromBytes(result.getRow()); recordIds.add(id); } Closer.close(scanner); // Not closed in finally block: avoid HBase contact when there could be connection problems. } catch (IOException e) { throw new RepositoryException("Error getting list of variants of record " + recordId.getMaster(), e); } return recordIds; } @Override public RecordBuilder recordBuilder() throws RecordException { return new RecordBuilderImpl(this, getIdGenerator()); } /** * Instantiate a new {@link FieldValueWriter} linked to this instance. * * @param put put to which field values are tot be written * @param parentRecord parent record of this record * @return new FieldValueWriter */ public FieldValueWriter newFieldValueWriter(Put put, Record parentRecord) { return new FieldValueWriter(put, parentRecord); } /** * Writes encoded record fields to a {@code Put} object. */ public class FieldValueWriter { private Put put; private Record parentRecord; private FieldValueWriter(Put put, Record parentRecord) { this.put = put; this.parentRecord = parentRecord; } public FieldValueWriter addFieldValue(FieldType fieldType, Object value, Metadata metadata) throws RepositoryException, InterruptedException { return addFieldValue(fieldType, value, metadata, 1L); } public FieldValueWriter addFieldValue(FieldType fieldType, Object value, Metadata metadata, long version) throws RepositoryException, InterruptedException { byte[] encodedFieldValue = encodeFieldValue(parentRecord, fieldType, value, metadata); put.add(RecordCf.DATA.bytes, ((FieldTypeImpl) fieldType).getQualifier(), version, encodedFieldValue); return this; } private byte[] encodeFieldValue(Record parentRecord, FieldType fieldType, Object fieldValue, Metadata metadata) throws RepositoryException, InterruptedException { if (isDeleteMarker(fieldValue)) { return FieldFlags.getDeleteMarker(); } ValueType valueType = fieldType.getValueType(); // fieldValue should never be null by the time we get here, but check anyway if (fieldValue == null) { throw new RepositoryException("Null field values are not allowed. Field: " + fieldType.getName()); } if (!valueType.getType().isAssignableFrom(fieldValue.getClass())) { throw new RepositoryException(String.format( "Incorrect type of value provided for field %s. " + "Expected instance of %s but got %s.", fieldType.getName(), valueType.getType().getName(), fieldValue.getClass().getName())); } DataOutput dataOutput = new DataOutputImpl(); boolean hasMetadata = metadata != null && !metadata.getMap().isEmpty(); dataOutput.writeByte(hasMetadata ? FieldFlags.METADATA_V1 : FieldFlags.DEFAULT); try { valueType.write(fieldValue, dataOutput, new IdentityRecordStack(parentRecord)); } catch (InterruptedException e) { throw e; } catch (Exception e) { // wrap the exception so that it is known what field causes the problem throw new RepositoryException("Error serializing value for field " + fieldType.getName(), e); } if (hasMetadata) { if (fieldType.getScope() == Scope.VERSIONED_MUTABLE) { throw new RuntimeException( "Field metadata is currently not supported for versioned-mutable fields."); } if (fieldType.getValueType().getDeepestValueType().getBaseName().equals("BLOB")) { throw new RuntimeException("Field metadata is currently not supported for BLOB fields."); } writeMetadataWithLengthSuffix(metadata, dataOutput); } return dataOutput.toByteArray(); } } }