org.sfs.jobs.AssignDocumentsToNodeJob.java Source code

Introduction

Here is the source code for org.sfs.jobs.AssignDocumentsToNodeJob.java

Source

/*
 * Copyright 2016 The Simple File Server Authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.sfs.jobs;

import com.google.common.base.Optional;
import com.google.common.collect.Iterables;
import io.vertx.core.MultiMap;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.sfs.Server;
import org.sfs.VertxContext;
import org.sfs.elasticsearch.Elasticsearch;
import org.sfs.elasticsearch.ListSfsStorageIndexes;
import org.sfs.elasticsearch.ScanAndScrollStreamProducer;
import org.sfs.elasticsearch.SearchHitEndableWriteStreamUpdateNodeId;
import org.sfs.io.AsyncIO;
import org.sfs.nodes.ClusterInfo;
import org.sfs.rx.Defer;
import org.sfs.rx.ToVoid;
import rx.Observable;

import java.util.HashMap;

import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
import static org.elasticsearch.index.query.QueryBuilders.existsQuery;
import static org.elasticsearch.index.query.QueryBuilders.termsQuery;

public class AssignDocumentsToNodeJob extends AbstractJob {

    private ScanAndScrollStreamProducer producer;
    private boolean aborted = false;

    @Override
    public String id() {
        return Jobs.ID.ASSIGN_DOCUMENTS_TO_NODE;
    }

    @Override
    public Observable<Void> executeImpl(VertxContext<Server> vertxContext, MultiMap parameters) {
        return execute0(vertxContext);
    }

    @Override
    public Observable<Void> stopImpl(VertxContext<Server> vertxContext) {
        aborted = true;
        if (producer != null) {
            producer.abort();
        }
        return Defer.aVoid();
    }

    protected Observable<Void> execute0(VertxContext<Server> vertxContext) {
        return Defer.aVoid().flatMap(aVoid -> {
            ClusterInfo clusterInfo = vertxContext.verticle().getClusterInfo();
            return Observable.from(clusterInfo.getDataNodes());
        }).reduce(new HashMap<String, Long>(), (documentCountsByNode, persistentServiceDef) -> {
            Optional<Long> documentCount = persistentServiceDef.getDocumentCount();
            if (documentCount.isPresent()) {
                documentCountsByNode.put(persistentServiceDef.getId(), documentCount.get());
            }
            return documentCountsByNode;
        }).filter(stringLongHashMap -> !stringLongHashMap.isEmpty()).flatMap(documentCountsByNode -> {
            Elasticsearch elasticsearch = vertxContext.verticle().elasticsearch();

            String[] activeNodeIds = Iterables.toArray(documentCountsByNode.keySet(), String.class);

            BoolQueryBuilder c0 = boolQuery().mustNot(existsQuery("node_id"));

            BoolQueryBuilder c1 = boolQuery().mustNot(termsQuery("node_id", activeNodeIds));

            BoolQueryBuilder query = boolQuery().should(c0).should(c1).minimumNumberShouldMatch(1);

            return Defer.aVoid().flatMap(new ListSfsStorageIndexes(vertxContext)).flatMap(index -> {
                producer = new ScanAndScrollStreamProducer(vertxContext, query).setIndeces(index)
                        .setTypes(elasticsearch.defaultType()).setReturnVersion(true);

                SearchHitEndableWriteStreamUpdateNodeId consumer = new SearchHitEndableWriteStreamUpdateNodeId(
                        vertxContext, documentCountsByNode);

                if (aborted) {
                    producer.abort();
                }

                return AsyncIO.pump(producer, consumer);
            }).count().map(new ToVoid<>());
        }).singleOrDefault(null);
    }
}