com.avanza.ymer.MirroredObjectLoader.java Source code

Java tutorial

Introduction

Here is the source code for com.avanza.ymer.MirroredObjectLoader.java

Source

/*
 * Copyright 2015 Avanza Bank AB
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.avanza.ymer;

import java.util.List;
import java.util.Optional;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.ForkJoinTask;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.avanza.ymer.plugin.PostReadProcessor;
import com.avanza.ymer.util.OptionalUtil;
import com.mongodb.BasicDBObject;
import com.mongodb.DBObject;

/**
 * Loads mirrored objects from an external (persistent) source.
 *
 * Implementation note: multithreaded patching to increase throughput.
 *
 * @author Elias Lindholm (elilin), Kristoffer Erlandsson, Andreas Skoog
 */
final class MirroredObjectLoader<T> {
    private static final int NUM_THREADS = 15;
    private final Logger log = LoggerFactory.getLogger(getClass());

    private final ForkJoinPool forkJoinPool = new ForkJoinPool(NUM_THREADS);
    private final MirroredObject<T> mirroredObject;
    private final DocumentCollection documentCollection;
    private final SpaceObjectFilter<T> spaceObjectFilter;
    private final DocumentConverter documentConverter;
    private final AtomicLong numLoadedObjects = new AtomicLong();
    private final MirrorContextProperties contextProperties;
    private final PostReadProcessor postReadProcessor;

    MirroredObjectLoader(DocumentCollection documentCollection, DocumentConverter documentConverter,
            MirroredObject<T> mirroredObject, SpaceObjectFilter<T> spaceObjectFilter,
            MirrorContextProperties contextProperties, PostReadProcessor postReadProcessor) {
        this.documentConverter = documentConverter;
        this.spaceObjectFilter = spaceObjectFilter;
        this.documentCollection = documentCollection;
        this.mirroredObject = mirroredObject;
        this.contextProperties = contextProperties;
        this.postReadProcessor = postReadProcessor;
    }

    List<LoadedDocument<T>> loadAllObjects() {
        return streamAllObjects().collect(Collectors.toList());
    }

    Stream<LoadedDocument<T>> streamAllObjects() {
        log.info("Begin loadAllObjects. targetCollection={}", mirroredObject.getCollectionName());
        ForkJoinTask<Stream<LoadedDocument<T>>> loadedDocuments = forkJoinPool.submit(() -> {
            return loadDocuments().parallel().map(this::tryPatchAndConvert).flatMap(OptionalUtil::asStream);

        });
        try {
            Stream<LoadedDocument<T>> result = loadedDocuments.get();
            return result;
        } catch (InterruptedException | ExecutionException e) {
            throw new RuntimeException(e);
        }
    }

    public void destroy() {
        this.forkJoinPool.shutdown();
    }

    private Stream<DBObject> loadDocuments() {
        if (mirroredObject.hasCustomInitialLoadTemplate()) {
            BasicDBObject template = mirroredObject.getCustomInitialLoadTemplateFactory()
                    .create(contextProperties.getPartitionCount(), contextProperties.getInstanceId());
            return documentCollection.findByTemplate(template);
        }
        if (mirroredObject.loadDocumentsRouted()) {
            return documentCollection.findAll(spaceObjectFilter);
        }
        return documentCollection.findAll();
    }

    private Optional<LoadedDocument<T>> tryPatchAndConvert(DBObject object) {
        try {
            Optional<LoadedDocument<T>> result;
            try {
                result = patchAndConvert(new BasicDBObject(object.toMap()));
            } catch (RuntimeException e) {
                // MongoConverter is not thread-safe due to a bug in AbstractMappingContext.addPersistentEntity().
                // The bug occurs at most once or twice per collection but will produce objects without any properties set
                // Resolve it temporarily by retrying.
                log.warn("Failed to load dbObject=" + object + ". Retrying.", e);
                result = patchAndConvert(new BasicDBObject(object.toMap()));
            }
            long loaded = numLoadedObjects.incrementAndGet();
            if (loaded % 10000 == 0) {
                log.info("Status: loaded {} records for collection {}", loaded, mirroredObject.getCollectionName());
            }
            return result;
        } catch (RuntimeException e) {
            log.error("Unable to load dbObject=" + object, e);
            throw e;
        }
    }

    Optional<LoadedDocument<T>> loadById(Object id) {
        BasicDBObject document = findById(id);
        if (document == null) {
            return Optional.empty();
        }
        // TODO: Why throw when spaceObjectFilter rejects but not when not found by findById???
        LoadedDocument<T> result = patchAndConvert(document).orElseThrow(
                () -> new IllegalArgumentException("Space object not accepted by filter (id=" + id + ")"));
        return Optional.of(result);
    }

    List<LoadedDocument<T>> loadByQuery(T template) {
        return documentCollection.findByQuery(documentConverter.toQuery(template)).map(BasicDBObject.class::cast)
                .map(this::patchAndConvert).flatMap(OptionalUtil::asStream).collect(Collectors.toList());
    }

    private BasicDBObject findById(Object id) {
        final Object convertedId = documentConverter.convertToMongoObject(id);
        final DBObject dbObject = documentCollection.findById(convertedId);
        return dbObject != null ? new BasicDBObject(dbObject.toMap()) : null;
    }

    private Optional<LoadedDocument<T>> patchAndConvert(BasicDBObject dbObject) {
        BasicDBObject currentVersion = dbObject;
        boolean patched = false;
        if (this.mirroredObject.requiresPatching(dbObject)) {
            patched = true;
            try {
                currentVersion = (BasicDBObject) dbObject.copy();
                postReadProcessor.postRead(currentVersion);
                currentVersion = this.mirroredObject.patch(currentVersion);
            } catch (RuntimeException e) {
                log.error(
                        "Patch of document failed! document=" + mirroredObject + "currentVersion=" + currentVersion,
                        e);
                throw e;
            }
        } else {
            postReadProcessor.postRead(currentVersion);
        }
        T mirroredObject = documentConverter.convert(this.mirroredObject.getMirroredType(), currentVersion);
        if (!spaceObjectFilter.accept(mirroredObject)) {
            return Optional.empty();
        }
        if (patched) {
            return Optional.of(new LoadedDocument<T>(postProcess(mirroredObject),
                    Optional.of(new PatchedDocument(dbObject, currentVersion))));
        }
        return Optional.of(new LoadedDocument<T>(postProcess(mirroredObject), Optional.empty()));
    }

    private T postProcess(T mirroredObject) {
        if (mirroredObject instanceof ReloadableSpaceObject) {
            ReloadableSpaceObjectUtil.markReloaded((ReloadableSpaceObject) mirroredObject);
        }
        return mirroredObject;
    }

    /*
     * Holds the space representation of a document loaded form an external data source (typically mongo)
     * and also an Optional {@link PatchedDocument} which is present if the document was patched during
     * the loading
     */
    static class LoadedDocument<T> {
        private final T document;
        private final Optional<PatchedDocument> patchedDocument;

        public LoadedDocument(T document, Optional<PatchedDocument> patchedDocument) {
            this.document = document;
            this.patchedDocument = patchedDocument;
        }

        public T getDocument() {
            return document;
        }

        public Optional<PatchedDocument> getPatchedDocument() {
            return patchedDocument;
        }

    }

}