Java tutorial
/************************************************************************* * Copyright 2009-2013 Eucalyptus Systems, Inc. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; version 3 of the License. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see http://www.gnu.org/licenses/. * * Please contact Eucalyptus Systems, Inc., 6755 Hollister Ave., Goleta * CA 93117, USA or visit http://www.eucalyptus.com/licenses/ if you need * additional information or have any questions. ************************************************************************/ package com.eucalyptus.objectstorage.metadata; import java.util.ArrayList; import java.util.Collections; import java.util.Date; import java.util.HashSet; import java.util.List; import java.util.NoSuchElementException; import javax.annotation.Nonnull; import javax.annotation.Nullable; import javax.persistence.EntityTransaction; import com.eucalyptus.entities.TransactionResource; import com.eucalyptus.entities.Transactions; import com.eucalyptus.objectstorage.ObjectMetadataManagers; import com.eucalyptus.objectstorage.ObjectState; import com.eucalyptus.objectstorage.PaginatedResult; import com.eucalyptus.objectstorage.exceptions.IllegalResourceStateException; import com.eucalyptus.objectstorage.exceptions.MetadataOperationFailureException; import com.eucalyptus.objectstorage.exceptions.ObjectStorageInternalException; import com.eucalyptus.objectstorage.exceptions.s3.NoSuchUploadException; import com.google.common.base.Function; import org.apache.log4j.Logger; import org.hibernate.Criteria; import org.hibernate.criterion.Criterion; import org.hibernate.criterion.Example; import org.hibernate.criterion.MatchMode; import org.hibernate.criterion.Order; import org.hibernate.criterion.Projections; import org.hibernate.criterion.Restrictions; import com.eucalyptus.auth.principal.User; import com.eucalyptus.entities.Entities; import com.eucalyptus.entities.TransactionException; import com.eucalyptus.objectstorage.entities.Bucket; import com.eucalyptus.objectstorage.entities.ObjectEntity; import com.eucalyptus.objectstorage.exceptions.s3.InternalErrorException; import com.eucalyptus.objectstorage.exceptions.s3.NoSuchKeyException; import com.eucalyptus.objectstorage.exceptions.s3.S3Exception; import com.eucalyptus.objectstorage.util.ObjectStorageProperties; import com.eucalyptus.storage.msgs.s3.AccessControlPolicy; import com.google.common.base.Predicate; import com.google.common.base.Strings; /** * Database backed implementation of ObjectMetadataManager * */ public class DbObjectMetadataManagerImpl implements ObjectMetadataManager { private static final Logger LOG = Logger.getLogger(DbObjectMetadataManagerImpl.class); public void start() throws Exception { LOG.trace("Starting DbObjectMetadataManager"); } public void stop() throws Exception { LOG.trace("Stopping DbObjectMetadataManager"); } @Override public ObjectEntity initiateCreation(@Nonnull ObjectEntity objectToCreate) throws Exception { return this.transitionObjectToState(objectToCreate, ObjectState.creating); } @Override public ObjectEntity finalizeCreation(ObjectEntity objectToUpdate, Date updateTimestamp, String eTag) throws MetadataOperationFailureException { objectToUpdate.setObjectModifiedTimestamp(updateTimestamp); objectToUpdate.seteTag(eTag); objectToUpdate.setIsLatest(true); return this.transitionObjectToState(objectToUpdate, ObjectState.extant); } @Override public ObjectEntity finalizeMultipartInit(ObjectEntity objectToUpdate, Date updateTimestamp, String uploadId) throws MetadataOperationFailureException { objectToUpdate.setObjectModifiedTimestamp(updateTimestamp); objectToUpdate.setUploadId(uploadId); objectToUpdate.setIsLatest(false); return this.transitionObjectToState(objectToUpdate, ObjectState.mpu_pending); } @Override public List<ObjectEntity> lookupObjectsInState(@Nullable Bucket bucket, String objectKey, String versionId, ObjectState state) throws Exception { try (TransactionResource db = Entities.transactionFor(ObjectEntity.class)) { Criteria search = Entities.createCriteria(ObjectEntity.class) .add(Example.create(new ObjectEntity(bucket, objectKey, versionId).withState(state))); search.addOrder(Order.desc("objectModifiedTimestamp")); if (bucket != null) { search = getSearchByBucket(search, bucket); } List<ObjectEntity> results = search.list(); db.commit(); return results; } catch (NoSuchElementException e) { // Nothing, return empty list return new ArrayList<>(0); } catch (Exception e) { LOG.error("Error fetching pending write records for object " + (bucket == null ? "" : bucket.getBucketName()) + "/" + objectKey + "?versionId=" + versionId); throw e; } } @Override public List<ObjectEntity> lookupObjectsForReaping(Bucket bucket, String objectKeyPrefix, Date age) { List<ObjectEntity> results; try (TransactionResource tran = Entities.transactionFor(ObjectEntity.class)) { // setup example and criteria ObjectEntity example = new ObjectEntity().withState(ObjectState.extant).withBucket(bucket); Criteria search = Entities.createCriteria(ObjectEntity.class).add(Example.create(example)); search.add(Restrictions.and(Restrictions.like("objectKey", objectKeyPrefix, MatchMode.START), Restrictions.lt("creationTimestamp", age))); search = getSearchByBucket(search, bucket); results = search.list(); tran.commit(); } catch (Exception ex) { LOG.error("exception caught while retrieving objects prefix with " + objectKeyPrefix + " from bucket " + bucket.getBucketName() + ", error message - " + ex.getMessage()); return Collections.EMPTY_LIST; } return results; } /** * Provides the search criteria to handle the FK relation from ObjectEntity->Bucket * Returns a criteria for a search that matches the given bucket * @param baseCriteria * @param bucket * @return */ protected static Criteria getSearchByBucket(@Nonnull Criteria baseCriteria, @Nullable Bucket bucket) { if (bucket != null) { return baseCriteria.createCriteria("bucket").add(Restrictions.eq("naturalId", bucket.getNaturalId())); } else { return baseCriteria; } } /** * A more limited version of read-repair, it just modifies the 'islatest' * tag, but will not mark any for deletion */ private static final Predicate<ObjectEntity> SET_LATEST_PREDICATE = new Predicate<ObjectEntity>() { public boolean apply(ObjectEntity example) { try { example.setIsLatest(true); example = example.withState(ObjectState.extant); Criteria search = Entities.createCriteria(ObjectEntity.class); search.add(Example.create(example)).addOrder(Order.desc("objectModifiedTimestamp")); search = getSearchByBucket(search, example.getBucket()); List<ObjectEntity> results = search.list(); if (results != null && results.size() > 1) { try { // Set all but the first element as not latest for (ObjectEntity obj : results.subList(1, results.size())) { obj.setIsLatest(false); } } catch (IndexOutOfBoundsException e) { // Either 0 or 1 result, nothing to do } } } catch (NoSuchElementException e) { // Nothing to do. } catch (Exception e) { LOG.error("Error consolidating Object records for " + example.getResourceFullName(), e); return false; } return true; } }; @Override public void cleanupInvalidObjects(final Bucket bucket, final String objectKey) throws Exception { ObjectEntity searchExample = new ObjectEntity(bucket, objectKey, null); final Predicate<ObjectEntity> repairPredicate = new Predicate<ObjectEntity>() { public boolean apply(ObjectEntity example) { try { //Find object versions that need updated ObjectEntity searchExample = new ObjectEntity().withKey(example.getObjectKey()) .withBucket(example.getBucket()).withState(ObjectState.extant); Criteria searchCriteria = Entities.createCriteria(ObjectEntity.class); searchCriteria.add(Example.create(searchExample)); searchCriteria.add( Restrictions.or(Restrictions.eq("versionId", ObjectStorageProperties.NULL_VERSION_ID), Restrictions.eq("isLatest", Boolean.TRUE))); searchCriteria.addOrder(Order.desc("objectModifiedTimestamp")); searchCriteria = getSearchByBucket(searchCriteria, example.getBucket()); List<ObjectEntity> results = searchCriteria.list(); if (results.size() <= 1) { //nothing to do return true; } ObjectEntity latest = results.get(0); latest.setIsLatest(Boolean.TRUE); // Set all but the first element as not latest for (ObjectEntity obj : results.subList(1, results.size())) { LOG.trace("Marking object " + obj.getObjectUuid() + " as no longer latest version"); obj.setIsLatest(Boolean.FALSE); if (latest.getVersionId() != null && ObjectStorageProperties.NULL_VERSION_ID.equals(latest.getVersionId()) && obj.getVersionId() != null && ObjectStorageProperties.NULL_VERSION_ID.equals(obj.getVersionId())) { transitionObjectToState(obj, ObjectState.deleting); } } } catch (NoSuchElementException e) { // Nothing to do. } catch (Exception e) { LOG.error("Error consolidationg Object records for " + example.getBucket().getBucketName() + "/" + example.getObjectKey()); return false; } return true; } }; try { Entities.asTransaction(repairPredicate).apply(searchExample); } catch (final Throwable f) { LOG.error("Error in version/null repair", f); } } @Override public void cleanupAllNullVersionedObjectRecords(final Bucket bucket, final String objectKey) throws Exception { ObjectEntity searchExample = new ObjectEntity(bucket, objectKey, null); final Predicate<ObjectEntity> repairPredicate = new Predicate<ObjectEntity>() { public boolean apply(ObjectEntity example) { try { //Find all null-versioned objects and mark them for deletion. ObjectEntity searchExample = new ObjectEntity().withKey(example.getObjectKey()) .withBucket(example.getBucket()).withState(ObjectState.extant) .withVersionId(ObjectStorageProperties.NULL_VERSION_ID); Criteria searchCriteria = Entities.createCriteria(ObjectEntity.class); searchCriteria.add(Example.create(searchExample)); searchCriteria = getSearchByBucket(searchCriteria, bucket); // Set all but the first element as not latest for (ObjectEntity obj : (List<ObjectEntity>) searchCriteria.list()) { LOG.trace("Marking object " + obj.getObjectUuid() + " as no longer latest version"); obj.setIsLatest(false); obj = transitionObjectToState(obj, ObjectState.deleting); } } catch (NoSuchElementException e) { // Nothing to do. } catch (Exception e) { LOG.error("Error consolidationg Object records for " + example.getBucket().getBucketName() + "/" + example.getObjectKey()); return false; } return true; } }; try { Entities.asTransaction(repairPredicate).apply(searchExample); } catch (final Throwable f) { LOG.error("Error in version/null repair", f); } } /** * Returns the ObjectEntities that are in 'creating' for too long and * thus should be considered failed */ @Override public List<ObjectEntity> lookupFailedObjects() throws MetadataOperationFailureException { // Return the latest version based on the created date. try (TransactionResource trans = Entities.transactionFor(ObjectEntity.class)) { ObjectEntity searchExample = new ObjectEntity().withState(ObjectState.creating); Criteria search = Entities.createCriteria(ObjectEntity.class); List<ObjectEntity> results = search.add(Example.create(searchExample)) .add(Restrictions.lt("creationExpiration", System.currentTimeMillis())).list(); trans.commit(); return results; } catch (NoSuchElementException e) { // Swallow this exception return new ArrayList(0); } catch (Exception e) { LOG.warn("Error fetching failed or deleted object records"); throw new MetadataOperationFailureException(e); } } @Override public ObjectEntity lookupObject(Bucket bucket, String objectKey, String versionId) throws NoSuchElementException, MetadataOperationFailureException { try { // Return the latest version based on the created date. try (TransactionResource trans = Entities.transactionFor(ObjectEntity.class)) { ObjectEntity searchExample = new ObjectEntity().withBucket(bucket).withKey(objectKey) .withState(ObjectState.extant); if (Strings.isNullOrEmpty(versionId)) { searchExample.setIsLatest(true); } else { searchExample = searchExample.withVersionId(versionId); } Criteria search = Entities.createCriteria(ObjectEntity.class).add(Example.create(searchExample)) .addOrder(Order.desc("objectModifiedTimestamp")).setMaxResults(1); search = getSearchByBucket(search, bucket); List<ObjectEntity> results = search.list(); if (results == null || results.size() < 1) { throw new NoSuchElementException(); } else if (results.size() > 1) { //this.repairObjectLatest(bucket, objectKey); //Do async repair if necessary to remove old data if overwritten //fireRepairTask(bucket, objectKey); } trans.commit(); return results.get(0); } } catch (NoSuchElementException ex) { throw ex; } catch (Exception e) { LOG.error("Error getting object entity for " + bucket.getBucketName() + "/" + objectKey + "?version=" + versionId, e); throw new MetadataOperationFailureException(e); } } @Override public ObjectEntity generateAndPersistDeleteMarker(@Nonnull ObjectEntity currentObject, @Nonnull AccessControlPolicy acp, @Nonnull User owningUser) throws MetadataOperationFailureException { final ObjectEntity deleteMarker = currentObject.generateNewDeleteMarkerFrom(); try (TransactionResource trans = Entities.transactionFor(ObjectEntity.class)) { deleteMarker.setOwnerCanonicalId(owningUser.getAccount().getCanonicalId()); deleteMarker.setOwnerDisplayName(owningUser.getAccount().getName()); deleteMarker.setOwnerIamUserDisplayName(owningUser.getName()); deleteMarker.setOwnerIamUserId(owningUser.getUserId()); deleteMarker.setAcl(acp); ObjectEntity persistedDeleteMarker = Entities.persist(deleteMarker); persistedDeleteMarker = ObjectMetadataManagers.getInstance() .transitionObjectToState(persistedDeleteMarker, ObjectState.extant); trans.commit(); return persistedDeleteMarker; } catch (Exception e) { LOG.warn("Failed to persist the delete marker " + deleteMarker.getObjectUuid()); throw new MetadataOperationFailureException(e); } } @Override public void delete(final @Nonnull ObjectEntity objectToDelete) throws IllegalResourceStateException, MetadataOperationFailureException { try { //Delete markers can be just removed not state transitioned. if (objectToDelete.getIsDeleteMarker()) { Transactions.delete(objectToDelete); return; } boolean success = Entities .asTransaction(ObjectEntity.class, ObjectStateTransitions.TRANSITION_TO_DELETED) .apply(objectToDelete); if (!success) { throw new MetadataOperationFailureException("Delete operation returned false"); } } catch (MetadataOperationFailureException | IllegalResourceStateException e) { throw e; } catch (Exception e) { throw new MetadataOperationFailureException(e); } } @Override public void flushUploads(Bucket bucket) throws Exception { EntityTransaction db = Entities.get(ObjectEntity.class); try { Criteria search = Entities.createCriteria(ObjectEntity.class); ObjectEntity searchExample = new ObjectEntity().withBucket(bucket).withState(ObjectState.mpu_pending); search.add(Example.create(searchExample)); search = getSearchByBucket(search, bucket); List<ObjectEntity> uploads = search.list(); for (ObjectEntity e : uploads) { Entities.delete(e); } db.commit(); } catch (Exception e) { throw new MetadataOperationFailureException(e); } finally { if (db != null && db.isActive()) { db.rollback(); } } } @Override public ObjectEntity lookupUpload(Bucket bucket, String uploadId) throws Exception { EntityTransaction db = Entities.get(ObjectEntity.class); try { Criteria search = Entities.createCriteria(ObjectEntity.class); ObjectEntity searchExample = new ObjectEntity().withBucket(bucket).withState(ObjectState.mpu_pending); searchExample.setUploadId(uploadId); search.add(Example.create(searchExample)); search = getSearchByBucket(search, bucket); List<ObjectEntity> results = search.list(); db.commit(); if (results.size() > 0) { return results.get(0); } else { throw new NoSuchUploadException(uploadId); } } finally { if (db != null && db.isActive()) { db.rollback(); } } } @Override public PaginatedResult<ObjectEntity> listUploads(Bucket bucket, int maxUploads, String prefix, String delimiter, String keyMarker, String uploadIdMarker) throws Exception { try (TransactionResource trans = Entities.transactionFor(ObjectEntity.class)) { PaginatedResult<ObjectEntity> result = new PaginatedResult<ObjectEntity>(); HashSet<String> commonPrefixes = new HashSet<String>(); // Include zero since 'istruncated' is still valid if (maxUploads >= 0) { final int queryStrideSize = maxUploads + 1; ObjectEntity searchObj = new ObjectEntity(); searchObj.withBucket(bucket); //This doesn't actually filter, but do it anyway searchObj.withState(ObjectState.mpu_pending); Criteria objCriteria = Entities.createCriteria(ObjectEntity.class); objCriteria.setReadOnly(true); objCriteria.setFetchSize(queryStrideSize); objCriteria.add(Example.create(searchObj)); objCriteria.addOrder(Order.asc("objectKey")); objCriteria.setMaxResults(queryStrideSize); if (!Strings.isNullOrEmpty(keyMarker)) { objCriteria.add(Restrictions.gt("objectKey", keyMarker)); } else { keyMarker = ""; } if (!Strings.isNullOrEmpty(uploadIdMarker)) { objCriteria.add(Restrictions.gt("uploadId", uploadIdMarker)); } else { uploadIdMarker = ""; } if (!Strings.isNullOrEmpty(prefix)) { objCriteria.add(Restrictions.like("objectKey", prefix, MatchMode.START)); } else { prefix = ""; } //Be sure to add the bucket restriction last objCriteria = getSearchByBucket(objCriteria, bucket); // Ensure not null. if (Strings.isNullOrEmpty(delimiter)) { delimiter = ""; } List<ObjectEntity> objectInfos = null; int resultKeyCount = 0; String[] parts = null; String prefixString = null; boolean useDelimiter = !Strings.isNullOrEmpty(delimiter); int pages = 0; // Iterate over result sets of size maxkeys + 1 since // commonPrefixes collapse the list, we may examine many more // records than maxkeys + 1 do { parts = null; prefixString = null; // Skip ahead the next page of 'queryStrideSize' results. objCriteria.setFirstResult(pages++ * queryStrideSize); objectInfos = (List<ObjectEntity>) objCriteria.list(); if (objectInfos == null) { // nothing to do. break; } for (ObjectEntity objectRecord : objectInfos) { if (useDelimiter) { // Check if it will get aggregated as a commonprefix parts = objectRecord.getObjectKey().substring(prefix.length()).split(delimiter); if (parts.length > 1) { prefixString = prefix + parts[0] + delimiter; if (!prefixString.equals(keyMarker) && !commonPrefixes.contains(prefixString)) { if (resultKeyCount == maxUploads) { // This is a new record, so we know // we're truncating if this is true result.setIsTruncated(true); resultKeyCount++; break; } else { // Add it to the common prefix set commonPrefixes.add(prefixString); result.setLastEntry(prefixString); // count the unique commonprefix as a // single return entry resultKeyCount++; } } else { // Already have this prefix, so skip } continue; } } if (resultKeyCount == maxUploads) { // This is a new (non-commonprefix) record, so // we know we're truncating result.setIsTruncated(true); resultKeyCount++; break; } result.getEntityList().add(objectRecord); result.setLastEntry(objectRecord); resultKeyCount++; } if (resultKeyCount <= maxUploads && objectInfos.size() <= maxUploads) { break; } } while (resultKeyCount <= maxUploads); // Sort the prefixes from the hashtable and add to the reply if (commonPrefixes != null) { result.getCommonPrefixes().addAll(commonPrefixes); Collections.sort(result.getCommonPrefixes()); } } else { throw new IllegalArgumentException("max uploads must be positive integer"); } return result; } catch (Exception e) { LOG.error("Error generating paginated multipart upload list for bucket " + bucket.getBucketName(), e); throw e; } } @Override public ObjectEntity setAcp(ObjectEntity object, AccessControlPolicy acp) throws S3Exception, TransactionException { try (TransactionResource trans = Entities.transactionFor(ObjectEntity.class)) { // Do record swap if existing record is found. ObjectEntity extantEntity = Entities.merge(object); extantEntity.setAcl(acp); trans.commit(); return extantEntity; } catch (Exception e) { LOG.error("Error setting ACP on backend for object: " + object.getResourceFullName()); throw new InternalErrorException(object.getResourceFullName() + "?versionId=" + object.getVersionId()); } } @Override public PaginatedResult<ObjectEntity> listPaginated(final Bucket bucket, int maxKeys, String prefix, String delimiter, String startKey) throws Exception { return listVersionsPaginated(bucket, maxKeys, prefix, delimiter, startKey, null, true); } @Override public List<ObjectEntity> lookupObjectVersions(Bucket bucket, String objectKey, int numResults) throws Exception { ObjectEntity searchObj = new ObjectEntity().withBucket(bucket).withState(ObjectState.extant) .withKey(objectKey); List<ObjectEntity> objectInfos = null; try (TransactionResource tran = Entities.transactionFor(ObjectEntity.class)) { Criteria objCriteria = Entities.createCriteria(ObjectEntity.class); objCriteria.setMaxResults(numResults); objCriteria.setReadOnly(true); objCriteria.add(Example.create(searchObj)); objCriteria.addOrder(Order.desc("objectModifiedTimestamp")); objCriteria = getSearchByBucket(objCriteria, bucket); objectInfos = objCriteria.list(); tran.commit(); } catch (Exception ex) { LOG.warn("exception caught while retrieving all versions of object " + objectKey + " in bucket " + bucket.getBucketName()); throw ex; } return objectInfos; } @Override public ObjectEntity makeLatest(ObjectEntity entity) throws Exception { ObjectEntity retrieved = null; try (TransactionResource tran = Entities.transactionFor(ObjectEntity.class)) { retrieved = lookupObject(entity.getBucket(), entity.getObjectKey(), entity.getVersionId()); retrieved.setIsLatest(Boolean.TRUE); Entities.mergeDirect(retrieved); tran.commit(); } catch (Exception ex) { LOG.warn( "while attempting to set isLatest = true on the newest remaining object version, an exception was encountered: ", ex); throw ex; } return retrieved; } private ObjectEntity makeNotLatest(ObjectEntity entity) throws Exception { try (TransactionResource tran = Entities.transactionFor(ObjectEntity.class)) { ObjectEntity retrieved = Entities.merge(entity); retrieved.setIsLatest(Boolean.FALSE); tran.commit(); return retrieved; } catch (Exception ex) { LOG.warn( "while attempting to set isLatest = true on the newest remaining object version, an exception was encountered: ", ex); throw ex; } } @Override public PaginatedResult<ObjectEntity> listVersionsPaginated(final Bucket bucket, int maxEntries, String prefix, String delimiter, String fromKeyMarker, String fromVersionId, boolean latestOnly) throws Exception { EntityTransaction db = Entities.get(ObjectEntity.class); try { PaginatedResult<ObjectEntity> result = new PaginatedResult<ObjectEntity>(); HashSet<String> commonPrefixes = new HashSet<String>(); // Include zero since 'istruncated' is still valid if (maxEntries >= 0) { final int queryStrideSize = maxEntries + 1; ObjectEntity searchObj = new ObjectEntity().withBucket(bucket).withState(ObjectState.extant); // Return latest version, so exclude delete markers as well. // This makes listVersion act like listObjects if (latestOnly) { searchObj.setIsLatest(true); searchObj.setIsDeleteMarker(false); } Criteria objCriteria = Entities.createCriteria(ObjectEntity.class); objCriteria.setReadOnly(true); objCriteria.setFetchSize(queryStrideSize); objCriteria.add(Example.create(searchObj)); objCriteria.addOrder(Order.asc("objectKey")); objCriteria.addOrder(Order.desc("objectModifiedTimestamp")); objCriteria.setMaxResults(queryStrideSize); if (!Strings.isNullOrEmpty(fromKeyMarker)) { if (!Strings.isNullOrEmpty(fromVersionId)) { // Look for the key that matches the key-marker and version-id-marker ObjectEntity searchObject = new ObjectEntity(bucket, fromKeyMarker, fromVersionId); ObjectEntity matchingObject = null; try { matchingObject = Entities.uniqueResult(searchObject); if (matchingObject == null || matchingObject.getObjectModifiedTimestamp() == null) { throw new NoSuchKeyException(bucket.getBucketName() + "/" + fromKeyMarker + "?versionId=" + fromVersionId); } } catch (Exception e) { LOG.warn("No matching object found for key-marker=" + fromKeyMarker + " and version-id-marker=" + fromVersionId); throw new NoSuchKeyException( bucket.getBucketName() + "/" + fromKeyMarker + "?versionId=" + fromVersionId); } // The result set should be exclusive of the key with the key-marker version-id-marker pair. Look for keys that chronologically // follow the version-id-marker for the given key-marker and also the keys that follow the key-marker. objCriteria.add(Restrictions.or( Restrictions.and(Restrictions.eq("objectKey", fromKeyMarker), Restrictions.lt("objectModifiedTimestamp", matchingObject.getObjectModifiedTimestamp())), Restrictions.gt("objectKey", fromKeyMarker))); } else { // No version-id-marker, just set the criteria the key-marker objCriteria.add(Restrictions.gt("objectKey", fromKeyMarker)); } } else { // No criteria to be set } if (!Strings.isNullOrEmpty(prefix)) { objCriteria.add(Restrictions.like("objectKey", prefix, MatchMode.START)); } else { prefix = ""; } objCriteria = getSearchByBucket(objCriteria, bucket); // Ensure not null. if (Strings.isNullOrEmpty(delimiter)) { delimiter = ""; } List<ObjectEntity> objectInfos = null; int resultKeyCount = 0; String[] parts = null; String prefixString = null; boolean useDelimiter = !Strings.isNullOrEmpty(delimiter); int pages = 0; // Iterate over result sets of size maxkeys + 1 since // commonPrefixes collapse the list, we may examine many more // records than maxkeys + 1 do { parts = null; prefixString = null; // Skip ahead the next page of 'queryStrideSize' results. objCriteria.setFirstResult(pages++ * queryStrideSize); objectInfos = (List<ObjectEntity>) objCriteria.list(); if (objectInfos == null) { // nothing to do. break; } for (ObjectEntity objectRecord : objectInfos) { if (useDelimiter) { // Check if it will get aggregated as a commonprefix parts = objectRecord.getObjectKey().substring(prefix.length()).split(delimiter); if (parts.length > 1) { prefixString = prefix + parts[0] + delimiter; if (!prefixString.equals(fromKeyMarker) && !commonPrefixes.contains(prefixString)) { if (resultKeyCount == maxEntries) { // This is a new record, so we know // we're truncating if this is true result.setIsTruncated(true); resultKeyCount++; break; } else { // Add it to the common prefix set commonPrefixes.add(prefixString); result.setLastEntry(prefixString); // count the unique commonprefix as a // single return entry resultKeyCount++; } } else { // Already have this prefix, so skip } continue; } } if (resultKeyCount == maxEntries) { // This is a new (non-commonprefix) record, so // we know we're truncating result.setIsTruncated(true); resultKeyCount++; break; } result.getEntityList().add(objectRecord); result.setLastEntry(objectRecord); resultKeyCount++; } if (resultKeyCount <= maxEntries && objectInfos.size() <= maxEntries) { break; } } while (resultKeyCount <= maxEntries); // Sort the prefixes from the hashtable and add to the reply if (commonPrefixes != null) { result.getCommonPrefixes().addAll(commonPrefixes); Collections.sort(result.getCommonPrefixes()); } } else { throw new IllegalArgumentException("MaxKeys must be positive integer"); } return result; } catch (Exception e) { LOG.error("Error generating paginated object list of bucket " + bucket.getBucketName(), e); throw e; } finally { db.rollback(); } } @Override public long countValid(Bucket bucket) throws Exception { try (TransactionResource trans = Entities.transactionFor(ObjectEntity.class)) { /*Criteria queryCriteria = Entities.createCriteria(ObjectEntity.class); queryCriteria.add(Restrictions.eq("state", ObjectState.extant)) .createCriteria("bucket").add(Restrictions.eq("naturalId", bucket.getNaturalId()))*/ Criteria queryCriteria = Entities.createCriteria(ObjectEntity.class) .add(Restrictions.eq("state", ObjectState.extant)).setProjection(Projections.rowCount()); queryCriteria = getSearchByBucket(queryCriteria, bucket); queryCriteria.setReadOnly(true); final Number count = (Number) queryCriteria.uniqueResult(); trans.commit(); return count.longValue(); } catch (Throwable e) { LOG.error("Error getting object count for bucket " + bucket.getBucketName(), e); throw new Exception(e); } } @Override public long getTotalSize(Bucket bucket) throws Exception { try (TransactionResource trans = Entities.transactionFor(ObjectEntity.class)) { Criteria queryCriteria = Entities.createCriteria(ObjectEntity.class) .add(Restrictions.or(Restrictions.eq("state", ObjectState.creating), Restrictions.eq("state", ObjectState.extant))) .setProjection(Projections.sum("size")); if (bucket != null) { queryCriteria = getSearchByBucket(queryCriteria, bucket); } queryCriteria.setReadOnly(true); final Number count = (Number) queryCriteria.uniqueResult(); return count == null ? 0 : count.longValue(); } catch (Throwable e) { LOG.error("Error getting object total size for " + (bucket == null ? "all buckets" : "bucket " + bucket.getBucketName()), e); throw new Exception(e); } } @Override public ObjectEntity transitionObjectToState(@Nonnull final ObjectEntity entity, @Nonnull ObjectState destState) throws IllegalResourceStateException, MetadataOperationFailureException { Function<ObjectEntity, ObjectEntity> transitionFunction; switch (destState) { case creating: transitionFunction = ObjectStateTransitions.TRANSITION_TO_CREATING; break; case extant: transitionFunction = ObjectStateTransitions.TRANSITION_TO_EXTANT; break; case mpu_pending: transitionFunction = ObjectStateTransitions.TRANSITION_TO_MPU_PENDING; break; case deleting: transitionFunction = ObjectStateTransitions.TRANSITION_TO_DELETING; break; default: LOG.error("Unexpected destination state: " + destState); throw new IllegalArgumentException(); } try { ObjectEntity result = Entities.asTransaction(ObjectEntity.class, transitionFunction).apply(entity); return result; } catch (ObjectStorageInternalException e) { throw e; } catch (Exception e) { throw new MetadataOperationFailureException(e); } } /** * Update the progress timeout field in the object entity. Will set it * to current-time + ObjectStorageProperties.PROGRESS_TIMEOUT_SEC * @param entity * @throws Exception */ public ObjectEntity updateCreationTimeout(ObjectEntity entity) throws Exception { try (TransactionResource trans = Entities.transactionFor(ObjectEntity.class)) { ObjectEntity mergedEntity = Entities.merge(entity); if (ObjectState.creating.equals(mergedEntity.getState())) { mergedEntity.updateCreationExpiration(); } Entities.flush(mergedEntity); //Ensure it is pushed right away trans.commit(); return mergedEntity; } catch (Exception e) { LOG.error("Error updating progress timeout for object " + entity.getObjectUuid()); throw e; } } }