Java tutorial
/* * Copyright 2016 The Simple File Server Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.sfs.elasticsearch; import com.google.common.base.Optional; import io.vertx.core.json.JsonObject; import io.vertx.core.logging.Logger; import io.vertx.core.logging.LoggerFactory; import org.sfs.Server; import org.sfs.VertxContext; import org.sfs.jobs.VerifyRepairAllContainerObjects; import org.sfs.nodes.ClusterInfo; import org.sfs.nodes.all.blobreference.AcknowledgeBlobReference; import org.sfs.nodes.all.blobreference.DeleteBlobReference; import org.sfs.nodes.all.blobreference.VerifyBlobReference; import org.sfs.nodes.all.segment.RebalanceSegment; import org.sfs.nodes.compute.object.PruneObject; import org.sfs.rx.Defer; import org.sfs.rx.RxHelper; import org.sfs.rx.ToType; import org.sfs.vo.PersistentAccount; import org.sfs.vo.PersistentContainer; import org.sfs.vo.PersistentObject; import org.sfs.vo.TransientBlobReference; import org.sfs.vo.TransientSegment; import org.sfs.vo.TransientServiceDef; import org.sfs.vo.TransientVersion; import rx.Observable; import rx.Scheduler; import java.util.Iterator; import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; import static com.google.common.base.Optional.absent; import static com.google.common.base.Optional.of; import static com.google.common.collect.FluentIterable.from; import static java.lang.Boolean.FALSE; import static java.lang.Boolean.TRUE; import static java.lang.System.currentTimeMillis; import static org.sfs.rx.Defer.aVoid; import static org.sfs.rx.Defer.just; import static rx.Observable.defer; public class SearchHitMaintainObjectEndableWrite extends AbstractBulkUpdateEndableWriteStream { private static final Logger LOGGER = LoggerFactory.getLogger(SearchHitMaintainObjectEndableWrite.class); private final ClusterInfo clusterInfo; private final CachedAccount cachedAccount; private final CachedContainer cachedContainer; private final List<TransientServiceDef> dataNodes; private final Scheduler scheduler; private final Set<String> forceRemoveVolumes; public SearchHitMaintainObjectEndableWrite(VertxContext<Server> vertxContext, Set<String> forceRemoveVolumes) { super(vertxContext); this.clusterInfo = vertxContext.verticle().getClusterInfo(); this.dataNodes = from(clusterInfo.getDataNodes()).transform(TransientServiceDef::copy).toList(); this.scheduler = RxHelper.scheduler(vertxContext.vertx()); this.cachedAccount = new CachedAccount(vertxContext); this.cachedContainer = new CachedContainer(vertxContext); this.forceRemoveVolumes = forceRemoveVolumes; if (!forceRemoveVolumes.isEmpty()) { LOGGER.info("forceRemoveVolumes: " + forceRemoveVolumes); } } @Override protected Observable<Optional<JsonObject>> transform(final JsonObject data, String id, long version) { return toPersistentObject(data, id, version).map(this::forceRemoveVolumes) // delete versions that are to old to attempt verification,ack and rebalance .flatMap(this::deleteOldUnAckdVersions) .timeout(3, TimeUnit.MINUTES, Observable.error(new RuntimeException( "Timeout on deleteOldUnAckdVersions " + data.encodePrettily())), scheduler) // attempt to delete versions that need deleting .flatMap(this::pruneObject) .timeout(3, TimeUnit.MINUTES, Observable.error(new RuntimeException("Timeout on pruneObject " + data.encodePrettily())), scheduler) // verifyAck before rebalance // so that in cases where the verify/ack // failed to persist to the index // we're able to recreate the state // needed for rebalancing .flatMap(this::verifyAck) .timeout(3, TimeUnit.MINUTES, Observable.error(new RuntimeException("Timeout on verifyAck " + data.encodePrettily())), scheduler) // rebalance the objects, including the ones that were just re-verified // disable rebalance because more testing is needed .flatMap(this::reBalance) .timeout(3, TimeUnit.MINUTES, Observable.error(new RuntimeException("Timeout on reBalance " + data.encodePrettily())), scheduler) .map(persistentObject -> { if (persistentObject.getVersions().isEmpty()) { return absent(); } else { JsonObject jsonObject = persistentObject.toJsonObject(); return of(jsonObject); } }); } protected PersistentObject forceRemoveVolumes(PersistentObject persistentObject) { if (!forceRemoveVolumes.isEmpty()) { for (TransientVersion version : persistentObject.getVersions()) { for (TransientSegment segment : version.getSegments()) { if (!segment.isTinyData()) { Iterator<TransientBlobReference> iterator = segment.getBlobs().iterator(); while (iterator.hasNext()) { TransientBlobReference blob = iterator.next(); String volumeId = blob.getVolumeId().orNull(); if (volumeId != null && forceRemoveVolumes.contains(volumeId)) { iterator.remove(); } } } } } } return persistentObject; } protected Observable<PersistentObject> verifyAck(PersistentObject persistentObject) { return just(persistentObject).flatMap(persistentObject1 -> Observable.from(persistentObject1.getVersions())) .filter(version -> !version.isDeleted()) .flatMap(transientVersion -> Observable.from(transientVersion.getSegments())) .filter(transientSegment -> !transientSegment.isTinyData()) .flatMap(transientSegment -> Observable.from(transientSegment.getBlobs())) .filter(transientBlobReference -> !transientBlobReference.isDeleted()) .flatMap(transientBlobReference -> { boolean alreadyAckd = transientBlobReference.isAcknowledged(); Optional<Integer> oVerifyFailCount = transientBlobReference.getVerifyFailCount(); int verifyFailCount = oVerifyFailCount.isPresent() ? oVerifyFailCount.get() : 0; return just(transientBlobReference).flatMap(new VerifyBlobReference(vertxContext)) .map(verified -> { // we do this here to unAck blob refs // in cases where the referenced volume has somehow // become corrupted if (!verified) { if (verifyFailCount >= VerifyRepairAllContainerObjects.VERIFY_RETRY_COUNT) { transientBlobReference.setAcknowledged(FALSE); } else { transientBlobReference.setVerifyFailCount(verifyFailCount + 1); } } else { transientBlobReference.setVerifyFailCount(0); transientBlobReference.setAcknowledged(TRUE); } return verified; }) // only call ack on the volume if this blob reference // was successfully verified and not already ackd since // there's not sense in re-ack'ing .filter(verified -> verified && !alreadyAckd).map(aVoid -> transientBlobReference) .flatMap(new AcknowledgeBlobReference(vertxContext)); }).count().map(new ToType<>(persistentObject)); } protected Observable<PersistentObject> reBalance(PersistentObject persistentObject) { return just(persistentObject).flatMap(persistentObject1 -> Observable.from(persistentObject1.getVersions())) .filter(version -> !version.isDeleted()) .flatMap(transientVersion -> Observable.from(transientVersion.getSegments())) .flatMap(transientSegment -> just(transientSegment) .flatMap(new RebalanceSegment(vertxContext, dataNodes)).map(reBalanced -> (Void) null)) .count().map(new ToType<>(persistentObject)); } protected Observable<PersistentObject> pruneObject(PersistentObject persistentObject) { return just(persistentObject).flatMap(new PruneObject(vertxContext)).map(modified -> persistentObject); } protected Observable<PersistentObject> deleteOldUnAckdVersions(PersistentObject persistentObject) { return defer(() -> { long now = currentTimeMillis(); return aVoid() .filter(aVoid -> now - persistentObject.getUpdateTs() .getTimeInMillis() >= VerifyRepairAllContainerObjects.CONSISTENCY_THRESHOLD) .map(aVoid -> persistentObject) .flatMap(persistentObject1 -> Observable.from(persistentObject1.getVersions())) // don't bother trying to delete old versions that are marked as // deleted since we have another method that will take care of that // at some point in the future .filter(version -> !version.isDeleted()) .flatMap(transientVersion -> Observable.from(transientVersion.getSegments())) .filter(transientSegment -> !transientSegment.isTinyData()) .flatMap(transientSegment -> Observable.from(transientSegment.getBlobs())) .filter(transientBlobReference -> !transientBlobReference.isAcknowledged()) .filter(transientBlobReference -> { Optional<Integer> oVerifyFailCount = transientBlobReference.getVerifyFailCount(); return oVerifyFailCount.isPresent() && oVerifyFailCount.get() >= VerifyRepairAllContainerObjects.VERIFY_RETRY_COUNT; }) .flatMap(transientBlobReference -> just(transientBlobReference) .flatMap(new DeleteBlobReference(vertxContext)).filter(deleted -> deleted) .map(deleted -> { if (Boolean.TRUE.equals(deleted)) { transientBlobReference.setDeleted(deleted); } return (Void) null; })) .onErrorResumeNext(throwable -> { LOGGER.warn("Handling Error", throwable); return Defer.aVoid(); }).count().map(new ToType<>(persistentObject)); }); } protected Observable<PersistentObject> toPersistentObject(JsonObject jsonObject, String id, long version) { return defer(() -> { final String accountId = jsonObject.getString("account_id"); final String containerId = jsonObject.getString("container_id"); return just(jsonObject).filter(jsonObject1 -> accountId != null && containerId != null) .flatMap(document -> getAccount(accountId)) .flatMap(persistentAccount -> getContainer(persistentAccount, containerId)) .map(persistentContainer -> new PersistentObject(persistentContainer, id, version) .merge(jsonObject)); }); } protected Observable<PersistentAccount> getAccount(String accountId) { return cachedAccount.get(accountId); } protected Observable<PersistentContainer> getContainer(PersistentAccount persistentAccount, String containerId) { return cachedContainer.get(persistentAccount, containerId); } }