org.cyclop.service.importer.intern.ParallelQueryImporter.java Source code

Introduction

Here is the source code for org.cyclop.service.importer.intern.ParallelQueryImporter.java

Source

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.cyclop.service.importer.intern;

import java.util.ArrayList;
import java.util.List;
import java.util.Scanner;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;

import javax.inject.Inject;
import javax.inject.Named;

import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.time.StopWatch;
import org.cyclop.model.CqlQuery;
import org.cyclop.model.CqlQueryType;
import org.cyclop.model.QueryHistory;
import org.cyclop.service.cassandra.CassandraSession;
import org.cyclop.service.importer.QueryImporter;
import org.cyclop.service.importer.ResultWriter;
import org.cyclop.service.importer.model.ImportConfig;
import org.cyclop.service.queryprotocoling.HistoryService;
import org.cyclop.validation.EnableValidation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.datastax.driver.core.Session;
import com.google.common.collect.ImmutableList;

/** @author Maciej Miklas */
@Named(QueryImporter.IMPL_PARALLEL)
@EnableValidation
public class ParallelQueryImporter extends AbstractImporter {

    private final static Logger LOG = LoggerFactory.getLogger(ParallelQueryImporter.class);

    @Inject
    protected HistoryService historyService;

    @Inject
    private CassandraSession session;

    @Inject
    @Named("importExecutor")
    private ExecutorService executor;

    @Override
    void execImport(Scanner scanner, ResultWriter resultWriter, StatsCollector status, ImportConfig iconfig) {
        ImmutableList<CqlQuery> queries = parse(scanner);
        if (queries.isEmpty()) {
            LOG.debug("No data to import");
            return;
        }

        QueryHistory history = historyService.read();

        List<Future<Void>> futures = startWorkers(queries, resultWriter, status, iconfig, history);
        waitForImport(futures);

        if (iconfig.isUpdateHistory()) {
            historyService.store(history);
        }
    }

    private List<Future<Void>> startWorkers(ImmutableList<CqlQuery> queries, ResultWriter resultWriter,
            StatsCollector status, ImportConfig iconfig, QueryHistory history) {

        final int queriesSize = queries.size();
        final int proThread = Math.max(1, queriesSize / conf.queryImport.maxThreadsProImport);

        LOG.debug("Starting parallel import for {} queries, {} pro thread", queriesSize, proThread);
        Session cassSession = session.getSession();

        List<Future<Void>> futures = new ArrayList<>();
        int startIndex = 0;
        int remaining = queriesSize;
        while (remaining > 0) {
            remaining -= proThread;
            int amount = remaining > 0 ? proThread : remaining + proThread;

            LOG.debug("Starting thread with start index: {} and amount: {}, remaining: {}", startIndex, amount,
                    remaining);

            ImportWorker task = new ImportWorker(startIndex, amount, queries, status, iconfig, resultWriter,
                    cassSession, history);

            Future<Void> future = executor.submit(task);
            futures.add(future);
            startIndex += amount;
        }
        return futures;
    }

    private void waitForImport(List<Future<Void>> futures) {
        LOG.debug("Tasks submitted - waiting for results");
        for (Future<Void> future : futures) {
            try {
                future.get();
            } catch (InterruptedException e) {
                Thread.interrupted();
                LOG.warn("Import executor interrupted", e);
            } catch (Exception e) {
                LOG.error("Import executor error", e);
            }
        }
    }

    private ImmutableList<CqlQuery> parse(Scanner scanner) {
        StopWatch timer = null;
        if (LOG.isDebugEnabled()) {
            timer = new StopWatch();
            timer.start();
        }

        ImmutableList.Builder<CqlQuery> build = ImmutableList.builder();
        while (scanner.hasNext()) {
            String nextStr = StringUtils.trimToNull(scanner.next());
            if (nextStr == null) {
                continue;
            }
            CqlQuery query = new CqlQuery(CqlQueryType.UNKNOWN, nextStr);
            build.add(query);
        }
        ImmutableList<CqlQuery> res = build.build();

        if (LOG.isDebugEnabled()) {
            timer.stop();
            LOG.debug("Parsed {} queries in {}", res.size(), timer.toString());
        }
        return res;
    }

}