com.baidu.rigel.biplatform.tesseract.isservice.search.service.impl.SearchIndexServiceImpl.java Source code

Java tutorial

Introduction

Here is the source code for com.baidu.rigel.biplatform.tesseract.isservice.search.service.impl.SearchIndexServiceImpl.java

Source

/**
 * Copyright (c) 2014 Baidu, Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
/**
 * 
 */
package com.baidu.rigel.biplatform.tesseract.isservice.search.service.impl;

import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorCompletionService;

import javax.annotation.Resource;

import org.apache.commons.collections.CollectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.task.TaskExecutor;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;

import com.baidu.rigel.biplatform.tesseract.dataquery.service.DataQueryService;
import com.baidu.rigel.biplatform.tesseract.datasource.DataSourcePoolService;
import com.baidu.rigel.biplatform.tesseract.datasource.impl.SqlDataSourceWrap;
import com.baidu.rigel.biplatform.tesseract.exception.DataSourceException;
import com.baidu.rigel.biplatform.tesseract.isservice.exception.IndexAndSearchException;
import com.baidu.rigel.biplatform.tesseract.isservice.exception.IndexAndSearchExceptionType;
import com.baidu.rigel.biplatform.tesseract.isservice.index.service.IndexMetaService;
import com.baidu.rigel.biplatform.tesseract.isservice.meta.IndexMeta;
import com.baidu.rigel.biplatform.tesseract.isservice.meta.IndexShard;
import com.baidu.rigel.biplatform.tesseract.isservice.meta.IndexState;
import com.baidu.rigel.biplatform.tesseract.isservice.meta.SqlQuery;
import com.baidu.rigel.biplatform.tesseract.isservice.search.service.SearchService;
import com.baidu.rigel.biplatform.tesseract.node.meta.Node;
import com.baidu.rigel.biplatform.tesseract.node.service.IndexAndSearchClient;
import com.baidu.rigel.biplatform.tesseract.node.service.IsNodeService;
import com.baidu.rigel.biplatform.tesseract.qsservice.query.vo.Expression;
import com.baidu.rigel.biplatform.tesseract.qsservice.query.vo.QueryMeasure;
import com.baidu.rigel.biplatform.tesseract.qsservice.query.vo.QueryRequest;
import com.baidu.rigel.biplatform.tesseract.resultset.isservice.SearchIndexResultSet;
import com.baidu.rigel.biplatform.tesseract.util.QueryRequestUtil;
import com.baidu.rigel.biplatform.tesseract.util.TesseractExceptionUtils;
import com.baidu.rigel.biplatform.tesseract.util.isservice.LogInfoConstants;

/**
 * SearchService 
 * 
 * @author lijin
 *
 */
@Service("searchService")
public class SearchIndexServiceImpl implements SearchService {

    private static final Logger LOGGER = LoggerFactory.getLogger(SearchIndexServiceImpl.class);

    /**
     * IndexAndSearchClient
     */

    private IndexAndSearchClient isClient;

    /**
     * ??
     */
    @Resource(name = "indexMetaService")
    private IndexMetaService idxMetaService;

    @Resource
    private IsNodeService isNodeService;

    /**
     * dataQueryService
     */
    @Resource(name = "sqlDataQueryService")
    private DataQueryService dataQueryService;

    /**
     * dataSourcePoolService
     */
    @Resource
    private DataSourcePoolService dataSourcePoolService;

    @Autowired
    private TaskExecutor taskExecutor;

    /**
     * Constructor by no param
     */
    public SearchIndexServiceImpl() {
        super();
        this.isClient = IndexAndSearchClient.getNodeClient();
    }

    /*
     * (non-Javadoc)
     * 
     * @see com.baidu.rigel.biplatform.tesseract.isservice.search.SearchService#query
     * (com.baidu.rigel.biplatform.tesseract.qsservice.query.vo.QueryRequest)
     */
    @Override
    public SearchIndexResultSet query(QueryRequest query) throws IndexAndSearchException {
        ExecutorCompletionService<SearchIndexResultSet> completionService = new ExecutorCompletionService<>(
                taskExecutor);
        LOGGER.info(String.format(LogInfoConstants.INFO_PATTERN_FUNCTION_BEGIN, "query", "[query:" + query + "]"));
        // 1. Does all the existed index cover this query
        // 2. get index meta and index shard
        // 3. trans query to Query that can used for searching
        // 4. dispatch search query
        // 5. do search
        // 6. merge result
        // 7. return

        if (query == null || StringUtils.isEmpty(query.getCubeId())) {
            LOGGER.error(String.format(LogInfoConstants.INFO_PATTERN_FUNCTION_EXCEPTION, "query",
                    "[query:" + query + "]"));
            throw new IndexAndSearchException(
                    TesseractExceptionUtils.getExceptionMessage(IndexAndSearchException.QUERYEXCEPTION_MESSAGE,
                            IndexAndSearchExceptionType.ILLEGALARGUMENT_EXCEPTION),
                    IndexAndSearchExceptionType.ILLEGALARGUMENT_EXCEPTION);
        }
        IndexMeta idxMeta = this.idxMetaService.getIndexMetaByCubeId(query.getCubeId(),
                query.getDataSourceInfo().getDataSourceKey());

        SearchIndexResultSet result = null;
        long current = System.currentTimeMillis();
        if (idxMeta == null || idxMeta.getIdxState().equals(IndexState.INDEX_UNAVAILABLE)
                || idxMeta.getIdxState().equals(IndexState.INDEX_UNINIT) || !query.isUseIndex()
                || (query.getFrom() != null && query.getFrom().getFrom() != null
                        && !idxMeta.getDataDescInfo().getTableNameList().contains(query.getFrom().getFrom()))
                || !indexMetaContains(idxMeta, query)) {
            LOGGER.info(String.format(LogInfoConstants.INFO_PATTERN_FUNCTION_PROCESS_NO_PARAM, "query",
                    "use database"));
            // index does not exist or unavailable,use db query
            SqlQuery sqlQuery = QueryRequestUtil.transQueryRequest2SqlQuery(query);
            SqlDataSourceWrap dataSourceWrape = null;
            try {
                dataSourceWrape = (SqlDataSourceWrap) this.dataSourcePoolService
                        .getDataSourceByKey(query.getDataSourceInfo());
            } catch (DataSourceException e) {
                LOGGER.error(String.format(LogInfoConstants.INFO_PATTERN_FUNCTION_EXCEPTION, "query",
                        "[query:" + query + "]", e));
                throw new IndexAndSearchException(
                        TesseractExceptionUtils.getExceptionMessage(IndexAndSearchException.QUERYEXCEPTION_MESSAGE,
                                IndexAndSearchExceptionType.SQL_EXCEPTION),
                        e, IndexAndSearchExceptionType.SQL_EXCEPTION);
            }
            if (dataSourceWrape == null) {
                throw new IllegalArgumentException();
            }

            long limitStart = 0;
            long limitSize = 0;
            if (query.getLimit() != null) {
                limitStart = query.getLimit().getStart();
                if (query.getLimit().getSize() > 0) {
                    limitSize = query.getLimit().getSize();
                }

            }
            SearchIndexResultSet currResult = this.dataQueryService.queryForListWithSQLQueryAndGroupBy(sqlQuery,
                    dataSourceWrape, limitStart, limitSize, query);
            LOGGER.info(String.format(LogInfoConstants.INFO_PATTERN_FUNCTION_PROCESS_NO_PARAM, "query",
                    "db return " + currResult.size() + " records"));
            result = currResult;
        } else {
            LOGGER.info(
                    String.format(LogInfoConstants.INFO_PATTERN_FUNCTION_PROCESS_NO_PARAM, "query", "use index"));

            LOGGER.info("cost :" + (System.currentTimeMillis() - current) + " before prepare get record.");
            current = System.currentTimeMillis();

            List<SearchIndexResultSet> idxShardResultSetList = new ArrayList<SearchIndexResultSet>();
            for (IndexShard idxShard : idxMeta.getIdxShardList()) {

                if (idxShard.getIdxState().equals(IndexState.INDEX_UNINIT)) {
                    continue;
                }

                completionService.submit(new Callable<SearchIndexResultSet>() {

                    @Override
                    public SearchIndexResultSet call() throws Exception {
                        try {
                            long current = System.currentTimeMillis();
                            Node searchNode = isNodeService.getFreeSearchNodeByIndexShard(idxShard,
                                    idxMeta.getClusterName());
                            searchNode.searchRequestCountAdd();
                            isNodeService.saveOrUpdateNodeInfo(searchNode);
                            LOGGER.info("begin search in shard:{}", idxShard);
                            SearchIndexResultSet result = (SearchIndexResultSet) isClient
                                    .search(query, idxShard, searchNode).getMessageBody();
                            searchNode.searchrequestCountSub();
                            isNodeService.saveOrUpdateNodeInfo(searchNode);
                            LOGGER.info("compelete search in shard:{},take:{} ms", idxShard,
                                    System.currentTimeMillis() - current);
                            return result;
                        } catch (Exception e) {
                            throw new IndexAndSearchException(
                                    TesseractExceptionUtils.getExceptionMessage(
                                            IndexAndSearchException.QUERYEXCEPTION_MESSAGE,
                                            IndexAndSearchExceptionType.NETWORK_EXCEPTION),
                                    e, IndexAndSearchExceptionType.NETWORK_EXCEPTION);
                        }

                    }
                });
            }
            for (int i = 0; i < idxMeta.getIdxShardList().size(); i++) {
                try {
                    idxShardResultSetList.add(completionService.take().get());
                } catch (InterruptedException | ExecutionException e) {
                    throw new IndexAndSearchException(
                            TesseractExceptionUtils.getExceptionMessage(
                                    IndexAndSearchException.QUERYEXCEPTION_MESSAGE,
                                    IndexAndSearchExceptionType.NETWORK_EXCEPTION),
                            e, IndexAndSearchExceptionType.NETWORK_EXCEPTION);
                }
            }
            LOGGER.info(String.format(LogInfoConstants.INFO_PATTERN_FUNCTION_PROCESS_NO_PARAM, "query",
                    "merging result from multiple index"));
            result = mergeResultSet(idxShardResultSetList, query);
            StringBuilder sb = new StringBuilder();
            sb.append("cost :").append(System.currentTimeMillis() - current)
                    .append(" in get result record,result size:").append(result.size()).append(" shard size:")
                    .append(idxShardResultSetList.size());

            LOGGER.info(sb.toString());
            current = System.currentTimeMillis();
        }

        LOGGER.info(String.format(LogInfoConstants.INFO_PATTERN_FUNCTION_PROCESS_NO_PARAM, "query",
                "merging final result"));

        LOGGER.info(String.format(LogInfoConstants.INFO_PATTERN_FUNCTION_END, "query", "[query:" + query + "]"));
        return result;
    }

    /**
     * 
     * mergeResultSet
     * 
     * @param resultList ??TesseractResultSet?
     * @return TesseractResultSet
     */
    private SearchIndexResultSet mergeResultSet(List<SearchIndexResultSet> resultList, QueryRequest query) {
        int totalSize = 0;
        for (SearchIndexResultSet tr : resultList) {
            totalSize += tr.size();
        }

        SearchIndexResultSet result = new SearchIndexResultSet(resultList.get(0).getMeta(), totalSize);
        resultList.forEach(set -> {
            result.getDataList().addAll(set.getDataList());
        });
        return result;

    }

    /**
     * 
     * ???
     * @param idxMeta ?
     * @param query 
     * @return boolean
     */
    private boolean indexMetaContains(IndexMeta idxMeta, QueryRequest query) {
        boolean result = false;
        if (idxMeta == null || query == null) {
            return result;
        }
        Set<String> idxSelect = idxMeta.getSelectList(false);

        if (!CollectionUtils.isEmpty(idxSelect) && idxSelect.containsAll(query.getSelect().getQueryProperties())) {
            for (QueryMeasure qm : query.getSelect().getQueryMeasures()) {
                if (!idxSelect.contains(qm.getProperties())) {
                    result = false;
                    break;
                } else {
                    result = true;
                }
            }

            if (query.getWhere() != null && query.getWhere().getAndList() != null && result) {
                for (Expression ex : query.getWhere().getAndList()) {
                    if (!idxSelect.contains(ex.getProperties())) {
                        result = false;
                        break;
                    } else {
                        result = true;
                    }
                }
            }

        }
        return result;

    }

}