com.eucalyptus.objectstorage.metadata.DbMpuPartMetadataManagerImpl.java Source code

Java tutorial

Introduction

Here is the source code for com.eucalyptus.objectstorage.metadata.DbMpuPartMetadataManagerImpl.java

Source

/*
 * Copyright 2009-2014 Eucalyptus Systems, Inc.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; version 3 of the License.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see http://www.gnu.org/licenses/.
 *
 * Please contact Eucalyptus Systems, Inc., 6755 Hollister Ave., Goleta
 * CA 93117, USA or visit http://www.eucalyptus.com/licenses/ if you need
 * additional information or have any questions.
 */

package com.eucalyptus.objectstorage.metadata;

import com.eucalyptus.entities.Entities;
import com.eucalyptus.entities.TransactionResource;
import com.eucalyptus.entities.Transactions;
import com.eucalyptus.objectstorage.BucketMetadataManagers;
import com.eucalyptus.objectstorage.MpuPartMetadataManagers;
import com.eucalyptus.objectstorage.ObjectState;
import com.eucalyptus.objectstorage.PaginatedResult;
import com.eucalyptus.objectstorage.entities.Bucket;
import com.eucalyptus.objectstorage.entities.PartEntity;
import com.eucalyptus.objectstorage.exceptions.IllegalResourceStateException;
import com.eucalyptus.objectstorage.exceptions.MetadataOperationFailureException;
import com.eucalyptus.objectstorage.exceptions.ObjectStorageInternalException;
import com.eucalyptus.objectstorage.exceptions.s3.EntityTooSmallException;
import com.eucalyptus.objectstorage.exceptions.s3.InvalidPartException;
import com.eucalyptus.objectstorage.exceptions.s3.InvalidPartOrderException;
import com.eucalyptus.objectstorage.exceptions.s3.S3Exception;
import com.eucalyptus.objectstorage.util.ObjectStorageProperties;
import com.eucalyptus.storage.msgs.s3.Part;
import com.google.common.base.Function;
import com.google.common.base.Objects;
import com.google.common.base.Predicate;
import org.apache.log4j.Logger;
import org.hibernate.Criteria;
import org.hibernate.criterion.Example;
import org.hibernate.criterion.Order;
import org.hibernate.criterion.Projections;
import org.hibernate.criterion.Restrictions;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import javax.persistence.EntityTransaction;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

/**
 * Database backed implementation of ObjectMetadataManager
 * 
 */
public class DbMpuPartMetadataManagerImpl implements MpuPartMetadataManager {
    private static final Logger LOG = Logger.getLogger(DbMpuPartMetadataManagerImpl.class);

    @Override
    public void start() throws Exception {
        // Do nothing
    }

    @Override
    public void stop() throws Exception {
    }

    @Override
    public PartEntity initiatePartCreation(@Nonnull PartEntity objectToCreate) throws Exception {
        return this.transitionPartToState(objectToCreate, ObjectState.creating);
    }

    @Override
    public PartEntity finalizeCreation(PartEntity objectToUpdate, Date updateTimestamp, String eTag)
            throws MetadataOperationFailureException {
        objectToUpdate.setObjectModifiedTimestamp(updateTimestamp);
        objectToUpdate.seteTag(eTag);
        objectToUpdate.setIsLatest(true);
        return this.transitionPartToState(objectToUpdate, ObjectState.extant);
    }

    /**
     * Provides the search criteria to handle the FK relation from PartEntity->Bucket
     * Returns a criteria for a search that matches the given bucket
     * @param baseCriteria
     * @param bucket
     * @return
     */
    protected static Criteria getSearchByBucket(@Nonnull Criteria baseCriteria, @Nullable Bucket bucket) {
        if (bucket != null) {
            return baseCriteria.createCriteria("bucket").add(Restrictions.eq("naturalId", bucket.getNaturalId()));
        } else {
            return baseCriteria;
        }
    }

    /**
     * A more limited version of read-repair, it just modifies the 'islatest'
     * tag, but will not mark any for deletion
     */
    private static final Predicate<PartEntity> SET_LATEST_PREDICATE = new Predicate<PartEntity>() {
        public boolean apply(PartEntity example) {
            try {
                example.setIsLatest(true);
                example = example.withState(ObjectState.extant);
                Criteria search = Entities.createCriteria(PartEntity.class);
                search.add(Example.create(example)).addOrder(Order.desc("objectModifiedTimestamp"));
                search = getSearchByBucket(search, example.getBucket());
                List<PartEntity> results = search.list();

                if (results != null && results.size() > 1) {
                    try {
                        // Set all but the first element as not latest
                        for (PartEntity obj : results.subList(1, results.size())) {
                            obj.setIsLatest(false);
                        }
                    } catch (IndexOutOfBoundsException e) {
                        // Either 0 or 1 result, nothing to do
                    }
                }
            } catch (NoSuchElementException e) {
                // Nothing to do.
            } catch (Exception e) {
                LOG.error("Error consolidating Object records for " + example.getResourceFullName(), e);
                return false;
            }
            return true;

        }
    };

    private static final Comparator timestampComparator = new Comparator<PartEntity>() {

        @Override
        public int compare(PartEntity objectEntity, PartEntity objectEntity2) {
            return objectEntity2.getObjectModifiedTimestamp().compareTo(objectEntity.getObjectModifiedTimestamp());
        }
    };

    @Override
    public void cleanupInvalidParts(final Bucket bucket, final String objectKey, final String uploadId,
            final int partNumber) throws Exception {
        final PartEntity searchExample = new PartEntity(bucket, objectKey, uploadId).withPartNumber(partNumber);

        final Predicate<PartEntity> repairPredicate = new Predicate<PartEntity>() {
            public boolean apply(PartEntity example) {
                try {

                    //Find not-latest null-versioned objects and mark them for deletion.
                    PartEntity searchExample = new PartEntity().withKey(example.getObjectKey())
                            .withBucket(example.getBucket()).withState(ObjectState.extant)
                            .withUploadId(example.getUploadId()).withPartNumber(example.getPartNumber());
                    Criteria searchCriteria = Entities.createCriteria(PartEntity.class);
                    searchCriteria.add(Example.create(searchExample))
                            .addOrder(Order.desc("objectModifiedTimestamp"));
                    searchCriteria = getSearchByBucket(searchCriteria, example.getBucket());
                    List<PartEntity> results = searchCriteria.list();
                    if (results.size() <= 1) {
                        //nothing to do
                        return true;
                    }

                    results.get(0).setIsLatest(true);
                    // Set all but the first element as not latest
                    for (PartEntity obj : results.subList(1, results.size())) {
                        LOG.trace("Marking mpu part " + obj.getPartUuid()
                                + " as no longer latest version and for cleanup");
                        obj.setIsLatest(false);
                        obj = transitionPartToState(obj, ObjectState.deleting);
                    }
                } catch (NoSuchElementException e) {
                    // Nothing to do.
                } catch (Exception e) {
                    LOG.error("Error consolidating PartEntity records for " + example.getBucket().getBucketName()
                            + "/" + example.getObjectKey() + " uploadId = " + example.getUploadId()
                            + " partNumber = " + example.getPartNumber());
                    return false;
                }
                return true;
            }
        };

        try {
            Entities.asTransaction(repairPredicate).apply(searchExample);
        } catch (final Throwable f) {
            LOG.error("Error in version/null repair", f);
        }
    }

    /**
     * Returns the ObjectEntities that are in 'creating' for too long and
     * thus should be considered failed
     */
    @Override
    public List<PartEntity> lookupFailedParts() throws MetadataOperationFailureException {
        // Return the latest version based on the created date.
        try (TransactionResource trans = Entities.transactionFor(PartEntity.class)) {
            PartEntity searchExample = new PartEntity().withState(ObjectState.creating);
            Criteria search = Entities.createCriteria(PartEntity.class);
            List<PartEntity> results = search.add(Example.create(searchExample))
                    .add(Restrictions.lt("creationExpiration", System.currentTimeMillis())).list();
            trans.commit();
            return results;
        } catch (NoSuchElementException e) {
            // Swallow this exception
            return new ArrayList(0);
        } catch (Exception e) {
            LOG.warn("Error fetching failed or deleted object records");
            throw new MetadataOperationFailureException(e);
        }
    }

    @Override
    public void delete(final @Nonnull PartEntity objectToDelete)
            throws IllegalResourceStateException, MetadataOperationFailureException {
        try {
            Transactions.delete(objectToDelete);
        } catch (MetadataOperationFailureException | IllegalResourceStateException e) {
            throw e;
        } catch (Exception e) {
            throw new MetadataOperationFailureException(e);
        }
    }

    @Override
    public List<PartEntity> lookupPartsInState(Bucket searchBucket, String searchKey, String uploadId,
            ObjectState state) throws Exception {
        EntityTransaction db = Entities.get(PartEntity.class);
        try {
            Criteria search = Entities.createCriteria(PartEntity.class);
            PartEntity searchExample = new PartEntity().withBucket(searchBucket).withKey(searchKey)
                    .withUploadId(uploadId).withState(state);
            search.add(Example.create(searchExample));
            if (searchBucket != null) {
                search = getSearchByBucket(search, searchBucket);
            }
            List<PartEntity> results = search.list();
            db.commit();
            return results;
        } finally {
            if (db != null && db.isActive()) {
                db.rollback();
            }
        }
    }

    @Override
    public void removeParts(final Bucket bucket, final String uploadId) throws Exception {
        Predicate<String> removePredicate = new Predicate<String>() {
            public boolean apply(String uploadId) {
                try (TransactionResource db = Entities.transactionFor(PartEntity.class)) {
                    //Calculate the sum size of the parts to update the bucket size.
                    PartEntity searchExample = new PartEntity().withUploadId(uploadId)
                            .withState(ObjectState.extant);
                    long size = Objects.firstNonNull(
                            (Number) Entities.createCriteria(PartEntity.class).add(Example.create(searchExample))
                                    .setProjection(Projections.sum("size")).setReadOnly(true).uniqueResult(),
                            0).longValue();

                    //Remove all part records with this upload id
                    Entities.deleteAllMatching(PartEntity.class,
                            "where part_number IS NOT NULL and upload_id=:uploadId",
                            Collections.singletonMap("uploadId", uploadId));
                    db.commit();
                } catch (Exception e) {
                    LOG.trace("Error finalizing part-removal transaction. Will retry.", e);
                    throw new RuntimeException(e);
                }

                return true;
            }
        };

        Entities.asTransaction(PartEntity.class, removePredicate).apply(uploadId);
    }

    @Override
    public void flushAllParts(Bucket bucket) throws Exception {
        try (TransactionResource db = Entities.transactionFor(PartEntity.class)) {
            Criteria search = Entities.createCriteria(PartEntity.class);
            PartEntity searchExample = new PartEntity().withBucket(bucket);
            search.add(Example.create(searchExample));
            search = getSearchByBucket(search, bucket);
            List<PartEntity> uploads = search.list();
            for (PartEntity e : uploads) {
                Entities.delete(e);
            }
            db.commit();
        }
    }

    @Override
    public PartEntity transitionPartToState(@Nonnull final PartEntity entity, @Nonnull ObjectState destState)
            throws IllegalResourceStateException, MetadataOperationFailureException {
        Function<PartEntity, PartEntity> transitionFunction;

        switch (destState) {
        case creating:
            transitionFunction = MpuPartStateTransitions.TRANSITION_TO_CREATING;
            break;
        case extant:
            transitionFunction = MpuPartStateTransitions.TRANSITION_TO_EXTANT;
            break;
        case deleting:
            transitionFunction = MpuPartStateTransitions.TRANSITION_TO_DELETING;
            break;
        default:
            LOG.error("Unexpected destination state: " + destState);
            throw new IllegalArgumentException();
        }

        try {
            return Entities.asTransaction(PartEntity.class, transitionFunction).apply(entity);
        } catch (ObjectStorageInternalException e) {
            throw e;
        } catch (Exception e) {
            throw new MetadataOperationFailureException(e);
        }
    }

    /**
     * Update the progress timeout field in the object entity. Will set it
     * to current-time + ObjectStorageProperties.PROGRESS_TIMEOUT_SEC
     * @param entity
     * @throws Exception
     */
    @Override
    public PartEntity updateCreationTimeout(PartEntity entity) throws Exception {
        try (TransactionResource trans = Entities.transactionFor(PartEntity.class)) {
            PartEntity mergedEntity = Entities.merge(entity);
            if (ObjectState.creating.equals(mergedEntity.getState())) {
                mergedEntity.updateCreationExpiration();
            }
            Entities.flush(mergedEntity); //Ensure it is pushed right away
            trans.commit();
            return mergedEntity;
        } catch (Exception e) {
            LOG.error("Error updating progress timeout for object " + entity.getPartUuid());
            throw e;
        }
    }

    @Override
    public HashMap<Integer, PartEntity> getParts(Bucket bucket, String objectKey, String uploadId)
            throws Exception {
        HashMap<Integer, PartEntity> parts = new HashMap<>();
        try (TransactionResource trans = Entities.transactionFor(PartEntity.class)) {
            Criteria search = Entities.createCriteria(PartEntity.class);
            PartEntity searchExample = new PartEntity(bucket, objectKey, uploadId).withState(ObjectState.extant);
            search.add(Example.create(searchExample));
            search = getSearchByBucket(search, bucket);
            List<PartEntity> results = search.list();
            trans.commit();
            for (PartEntity result : results) {
                parts.put(result.getPartNumber(), result);
            }
            return parts;
        } catch (Exception e) {
            LOG.warn("Error looking up parts for MPU id : " + uploadId);
            throw e;
        }
    }

    private void doConsolidateParts(Bucket bucket, String objectKey, String uploadId, Integer partNumber) {
        PartEntity searchExample = new PartEntity(bucket, objectKey, uploadId).withPartNumber(partNumber);
        searchExample = searchExample.withState(ObjectState.extant);

        final Predicate<PartEntity> repairPredicate = new Predicate<PartEntity>() {
            public boolean apply(PartEntity example) {
                //Remove all but latest entry
                try {
                    Criteria search = Entities.createCriteria(PartEntity.class);
                    List<PartEntity> results = search.add(Example.create(example))
                            .addOrder(Order.desc("objectModifiedTimestamp")).list();

                    if (results != null && results.size() > 0) {
                        try {
                            for (PartEntity partEntity : results.subList(1, results.size())) {
                                LOG.trace("Marking part " + partEntity.getBucket().getBucketName() + " uploadId: "
                                        + partEntity.getUploadId() + " partNumber: " + partEntity.getPartNumber()
                                        + " for deletion because it is not latest.");
                                //partEntity.setState(ObjectState.deleting);
                                MpuPartMetadataManagers.getInstance().transitionPartToState(partEntity,
                                        ObjectState.deleting);
                            }
                        } catch (IndexOutOfBoundsException e) {
                            // Either 0 or 1 result, nothing to do
                        }
                    }
                } catch (NoSuchElementException e) {
                    // Nothing to do.
                } catch (Exception e) {
                    LOG.error("Error consolidationg Part records for " + example.getBucket().getBucketName()
                            + " uploadId: " + example.getUploadId() + " partNumber: " + example.getPartNumber());
                    return false;
                }
                return true;
            }
        };
        try {
            Entities.asTransaction(repairPredicate).apply(searchExample);
        } catch (final Throwable f) {
            LOG.error("Error in part repair", f);
        }
    }

    @Override
    public long processPartListAndGetSize(List<Part> partsInManifest, HashMap<Integer, PartEntity> availableParts)
            throws S3Exception {
        int lastPartNumber = 0;
        long objectSize = 0;
        int numPartsProcessed = 0;
        for (Part partInManifest : partsInManifest) {
            Integer partNumber = partInManifest.getPartNumber();
            if (partNumber <= lastPartNumber) {
                throw new InvalidPartOrderException("partNumber: " + partNumber);
            }
            PartEntity actualPart = availableParts.get(partNumber);
            if (actualPart == null) {
                throw new InvalidPartException("partNumber: " + partNumber);
            }
            final long actualPartSize = actualPart.getSize();
            if ((++numPartsProcessed) < partsInManifest.size()
                    && actualPartSize < ObjectStorageProperties.MPU_PART_MIN_SIZE) {
                throw new EntityTooSmallException(
                        "uploadId: " + actualPart.getUploadId() + " partNumber: " + partNumber);
            }
            objectSize += actualPartSize;
            lastPartNumber = partNumber;
        }
        return objectSize;
    }

    @Override
    public long getTotalSize(Bucket bucket) throws Exception {
        try (TransactionResource trans = Entities.transactionFor(PartEntity.class)) {
            Criteria queryCriteria = Entities.createCriteria(PartEntity.class)
                    .add(Restrictions.or(Restrictions.eq("state", ObjectState.creating),
                            Restrictions.eq("state", ObjectState.extant)))
                    .setProjection(Projections.sum("size"));
            if (bucket != null) {
                queryCriteria = getSearchByBucket(queryCriteria, bucket);
            }
            queryCriteria.setReadOnly(true);
            final Number count = (Number) queryCriteria.uniqueResult();
            return count == null ? 0 : count.longValue();
        } catch (Throwable e) {
            LOG.error("Error getting part total size for bucket " + bucket.getBucketName(), e);
            throw new Exception(e);
        }
    }

    @Override
    public PaginatedResult<PartEntity> listPartsForUpload(final Bucket bucket, final String objectKey,
            final String uploadId, final Integer partNumberMarker, final Integer maxParts) throws Exception {

        EntityTransaction db = Entities.get(PartEntity.class);
        try {
            PaginatedResult<PartEntity> result = new PaginatedResult<PartEntity>();
            HashSet<String> commonPrefixes = new HashSet<String>();

            // Include zero since 'istruncated' is still valid
            if (maxParts >= 0) {
                final int queryStrideSize = maxParts + 1;
                PartEntity searchPart = new PartEntity(bucket, objectKey, uploadId).withState(ObjectState.extant);

                Criteria objCriteria = Entities.createCriteria(PartEntity.class);
                objCriteria.setReadOnly(true);
                objCriteria.setFetchSize(queryStrideSize);
                objCriteria.add(Example.create(searchPart));
                objCriteria.addOrder(Order.asc("partNumber"));
                objCriteria.addOrder(Order.desc("objectModifiedTimestamp"));
                objCriteria.setMaxResults(queryStrideSize);

                if (partNumberMarker != null) {
                    objCriteria.add(Restrictions.gt("partNumber", partNumberMarker));
                }
                objCriteria = getSearchByBucket(objCriteria, bucket);

                List<PartEntity> partInfos = null;
                int resultKeyCount = 0;
                String[] parts = null;
                int pages = 0;

                // Iterate over result sets of size maxkeys + 1 since
                // commonPrefixes collapse the list, we may examine many more
                // records than maxkeys + 1
                do {
                    parts = null;

                    // Skip ahead the next page of 'queryStrideSize' results.
                    objCriteria.setFirstResult(pages++ * queryStrideSize);

                    partInfos = (List<PartEntity>) objCriteria.list();
                    if (partInfos == null) {
                        // nothing to do.
                        break;
                    }

                    for (PartEntity partRecord : partInfos) {

                        if (resultKeyCount == maxParts) {
                            result.setIsTruncated(true);
                            resultKeyCount++;
                            break;
                        }

                        result.getEntityList().add(partRecord);
                        result.setLastEntry(partRecord);
                        resultKeyCount++;
                    }

                    if (resultKeyCount <= maxParts && partInfos.size() <= maxParts) {
                        break;
                    }
                } while (resultKeyCount <= maxParts);
            } else {
                throw new IllegalArgumentException("MaxKeys must be positive integer");
            }

            return result;
        } catch (Exception e) {
            LOG.error("Error generating paginated parts list for upload ID " + uploadId, e);
            throw e;
        } finally {
            db.rollback();
        }
    }
}