com.baidu.rigel.biplatform.tesseract.isservice.search.service.impl.CallbackSearchServiceImpl.java Source code

Java tutorial

Introduction

Here is the source code for com.baidu.rigel.biplatform.tesseract.isservice.search.service.impl.CallbackSearchServiceImpl.java

Source

/**
 * Copyright (c) 2014 Baidu, Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
/**
 * 
 */
package com.baidu.rigel.biplatform.tesseract.isservice.search.service.impl;

import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.Future;
import java.util.function.Consumer;
import java.util.stream.Collectors;

import org.apache.commons.collections.CollectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;

import com.baidu.rigel.biplatform.ac.minicube.CallbackMeasure;
import com.baidu.rigel.biplatform.ac.minicube.MiniCubeMeasure;
import com.baidu.rigel.biplatform.ac.model.MeasureType;
import com.baidu.rigel.biplatform.ac.model.callback.CallbackMeasureVaue;
import com.baidu.rigel.biplatform.ac.model.callback.CallbackResponse;
import com.baidu.rigel.biplatform.ac.model.callback.CallbackServiceInvoker;
import com.baidu.rigel.biplatform.ac.model.callback.CallbackType;
import com.baidu.rigel.biplatform.ac.util.AnswerCoreConstant;
import com.baidu.rigel.biplatform.ac.util.ThreadLocalPlaceholder;
import com.baidu.rigel.biplatform.tesseract.dataquery.udf.condition.QueryContextAdapter;
import com.baidu.rigel.biplatform.tesseract.isservice.exception.IndexAndSearchException;
import com.baidu.rigel.biplatform.tesseract.isservice.exception.IndexAndSearchExceptionType;
import com.baidu.rigel.biplatform.tesseract.isservice.meta.SqlQuery;
import com.baidu.rigel.biplatform.tesseract.qsservice.query.vo.Expression;
import com.baidu.rigel.biplatform.tesseract.qsservice.query.vo.QueryContext;
import com.baidu.rigel.biplatform.tesseract.qsservice.query.vo.QueryRequest;
import com.baidu.rigel.biplatform.tesseract.resultset.isservice.Meta;
import com.baidu.rigel.biplatform.tesseract.resultset.isservice.SearchIndexResultRecord;
import com.baidu.rigel.biplatform.tesseract.resultset.isservice.SearchIndexResultSet;
import com.baidu.rigel.biplatform.tesseract.util.QueryRequestUtil;
import com.baidu.rigel.biplatform.tesseract.util.TesseractConstant;
import com.baidu.rigel.biplatform.tesseract.util.TesseractExceptionUtils;
import com.baidu.rigel.biplatform.tesseract.util.isservice.LogInfoConstants;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;

/**
 * SearchService ?API
 * 
 * @author mengran
 *
 */
@Service("callbackSearchService")
public class CallbackSearchServiceImpl {

    private static final Logger LOGGER = LoggerFactory.getLogger(CallbackSearchServiceImpl.class);

    private static final String MEASURE_NAMES = "measureNames";
    private static final String GROUP_BY = "groupBy";
    private static final String FILTER = "filter";
    private static final String RESPONSE_VALUE_SPLIT = "$";

    @Autowired
    private ThreadPoolTaskExecutor taskExecutor;

    @Value(value = "${callback.measure.timeout}")
    private long callbackTimeout;

    /**
     * Constructor by no param
     */
    public CallbackSearchServiceImpl() {
        super();
    }

    /**
     * ?????
     * @author mengran
     *
     */
    private class CallbackExecutor implements Callable<CallbackResponse> {

        private Entry<String, List<MiniCubeMeasure>> group;
        private String groupbyParams;
        private String whereParams;
        // For external usage only
        private String callbackMeasures;

        /**
         * @param group
         */
        public CallbackExecutor(Entry<String, List<MiniCubeMeasure>> group,
                LinkedHashMap<String, List<String>> groupbyParams,
                LinkedHashMap<String, List<String>> whereParams) {
            super();
            this.group = group;
            this.groupbyParams = AnswerCoreConstant.GSON.toJson(groupbyParams);
            this.whereParams = AnswerCoreConstant.GSON.toJson(whereParams);
        }

        public List<String> getCallbackMeasures() {

            return Arrays.asList(StringUtils.delimitedListToStringArray(callbackMeasures, ","));
        }

        @Override
        public CallbackResponse call() throws Exception {

            MiniCubeMeasure maxTimeout = group.getValue().stream().max(new Comparator<MiniCubeMeasure>() {

                @Override
                public int compare(MiniCubeMeasure o1, MiniCubeMeasure o2) {
                    return new Long(
                            ((CallbackMeasure) o1).getSocketTimeOut() - ((CallbackMeasure) o2).getSocketTimeOut())
                                    .intValue();
                }

            }).get();

            // Build call-back parameter
            Map<String, String> merged = new HashMap<String, String>();
            StringBuilder sb = new StringBuilder();
            for (MiniCubeMeasure m : group.getValue()) {
                sb.append(",").append(m.getName());
            }
            if (sb.length() > 0) {
                sb.deleteCharAt(0);
            }
            // Prepared required parameters (other parameters depend on invoke methods)
            merged.put(MEASURE_NAMES, callbackMeasures = sb.toString());
            merged.put(GROUP_BY, groupbyParams);
            merged.put(FILTER, whereParams);

            group.getValue().stream().forEach(new Consumer<MiniCubeMeasure>() {

                @Override
                public void accept(MiniCubeMeasure m) {
                    if (((CallbackMeasure) m).getCallbackParams() != null
                            && !((CallbackMeasure) m).getCallbackParams().isEmpty()) {
                        merged.putAll(((CallbackMeasure) m).getCallbackParams());
                    }
                }
            });
            return CallbackServiceInvoker.invokeCallback(group.getKey(), merged, CallbackType.MEASURE,
                    ((CallbackMeasure) maxTimeout).getSocketTimeOut());
        }

    }

    /**
     * @param context ?
     * @param query ?
     * @return 
     * @throws IndexAndSearchException exception occurred when 
     */
    public SearchIndexResultSet query(QueryContext context, QueryRequest query) throws IndexAndSearchException {
        LOGGER.info(String.format(LogInfoConstants.INFO_PATTERN_FUNCTION_BEGIN, "callbackquery",
                "[callbackquery:" + query + "]"));
        if (query == null || context == null || StringUtils.isEmpty(query.getCubeId())) {
            LOGGER.error(String.format(LogInfoConstants.INFO_PATTERN_FUNCTION_EXCEPTION, "callbackquery",
                    "[callbackquery:" + query + "]"));
            throw new IndexAndSearchException(
                    TesseractExceptionUtils.getExceptionMessage(IndexAndSearchException.QUERYEXCEPTION_MESSAGE,
                            IndexAndSearchExceptionType.ILLEGALARGUMENT_EXCEPTION),
                    IndexAndSearchExceptionType.ILLEGALARGUMENT_EXCEPTION);
        }
        // TODO ???
        if (query.getGroupBy() == null || query.getSelect() == null) {
            return null;
        }
        Map<String, String> requestParams = ((QueryContextAdapter) context).getQuestionModel().getRequestParams();
        // Build query target map
        Map<String, List<MiniCubeMeasure>> callbackMeasures = context.getQueryMeasures().stream()
                .filter(m -> m.getType().equals(MeasureType.CALLBACK)).map(m -> {
                    CallbackMeasure tmp = (CallbackMeasure) m;
                    for (Map.Entry<String, String> entry : tmp.getCallbackParams().entrySet()) {
                        if (requestParams.containsKey(entry.getKey())) {
                            tmp.getCallbackParams().put(entry.getKey(), requestParams.get(entry.getKey()));
                        }
                    }
                    return m;
                }).collect(Collectors.groupingBy(c -> ((CallbackMeasure) c).getCallbackUrl(), Collectors.toList()));
        if (callbackMeasures == null || callbackMeasures.isEmpty()) {
            LOGGER.error(String.format(LogInfoConstants.INFO_PATTERN_FUNCTION_EXCEPTION, "Empty callback measure",
                    "[callbackquery:" + query + "]"));
            throw new IndexAndSearchException(
                    TesseractExceptionUtils.getExceptionMessage(IndexAndSearchException.QUERYEXCEPTION_MESSAGE,
                            IndexAndSearchExceptionType.ILLEGALARGUMENT_EXCEPTION),
                    IndexAndSearchExceptionType.ILLEGALARGUMENT_EXCEPTION);
        }
        LOGGER.info("Find callback targets " + callbackMeasures);

        // Keep group-by sequence.
        List<String> groupby = new ArrayList<String>(query.getGroupBy().getGroups());
        LinkedHashMap<String, List<String>> groupbyParams = new LinkedHashMap<String, List<String>>(groupby.size());
        for (String g : groupby) {
            groupbyParams.put(g, new ArrayList<String>());
        }

        LinkedHashMap<String, List<String>> whereParams = new LinkedHashMap<String, List<String>>();
        for (Expression e : query.getWhere().getAndList()) {
            List<String> l = e.getQueryValues().stream().filter(v -> !StringUtils.isEmpty(v.getValue()))
                    .map(v -> v.getValue()).collect(Collectors.toList());
            if (groupbyParams.containsKey(e.getProperties())) {
                // if not contains SUMMARY_KEY, add it into group by list
                if (!l.contains(TesseractConstant.SUMMARY_KEY)) {
                    l.add(TesseractConstant.SUMMARY_KEY);
                }
                // Put it into group by field
                groupbyParams.get(e.getProperties()).addAll(l);
            } else {
                // Put it into filter field
                if (CollectionUtils.isEmpty(l)) {
                    List<Set<String>> tmp = e.getQueryValues().stream().map(v -> v.getLeafValues())
                            .collect(Collectors.toList());
                    List<String> values = Lists.newArrayList();
                    tmp.forEach(t -> values.addAll(t));
                    whereParams.put(e.getProperties(), values);
                } else {
                    whereParams.put(e.getProperties(), new ArrayList<String>(l));
                }
            }
        }

        // Prepare query tools
        //        CountDownLatch latch = new CountDownLatch(response.size());
        //        List<Future<CallbackResponse>> results = Lists.newArrayList();
        Map<CallbackExecutor, Future<CallbackResponse>> results = Maps.newHashMap();
        ExecutorCompletionService<CallbackResponse> service = new ExecutorCompletionService<CallbackResponse>(
                taskExecutor);
        StringBuilder callbackMeasureNames = new StringBuilder();
        for (Entry<String, List<MiniCubeMeasure>> e : callbackMeasures.entrySet()) {
            CallbackExecutor ce = new CallbackExecutor(e, groupbyParams, whereParams);
            results.put(ce, service.submit(ce));
            e.getValue().forEach(m -> {
                callbackMeasureNames.append(" " + m.getCaption() + " ");
            });
        }
        //        }
        Map<CallbackExecutor, CallbackResponse> response = new ConcurrentHashMap<CallbackExecutor, CallbackResponse>(
                callbackMeasures.size());
        StringBuffer sb = new StringBuffer();
        results.forEach((k, v) -> {
            try {
                response.put(k, v.get());
            } catch (Exception e1) {
                LOGGER.error(e1.getMessage(), e1);
                sb.append(": " + callbackMeasureNames.toString()
                        + " ??, ?");
            }
        });
        if (!StringUtils.isEmpty(sb.toString())) {
            if (ThreadLocalPlaceholder.getProperty(ThreadLocalPlaceholder.ERROR_MSG_KEY) != null) {
                ThreadLocalPlaceholder.unbindProperty(ThreadLocalPlaceholder.ERROR_MSG_KEY);
            }
            ThreadLocalPlaceholder.bindProperty(ThreadLocalPlaceholder.ERROR_MSG_KEY, sb.toString());
        }
        // Package result
        SqlQuery sqlQuery = QueryRequestUtil.transQueryRequest2SqlQuery(query);
        SearchIndexResultSet result = null;
        if (!response.isEmpty()) {
            result = packageResultRecords(query, sqlQuery, response);
        } else {
            result = new SearchIndexResultSet(new Meta(query.getGroupBy().getGroups().toArray(new String[0])), 0);
        }

        LOGGER.info(String.format(LogInfoConstants.INFO_PATTERN_FUNCTION_END, "query", "[query:" + query + "]"));
        return result;
    }

    private SearchIndexResultSet packageResultRecords(QueryRequest query, SqlQuery sqlQuery,
            Map<CallbackExecutor, CallbackResponse> response) {
        List<String> groupby = new ArrayList<String>(query.getGroupBy().getGroups());
        // Confirm meta sequence
        List<Entry<CallbackExecutor, CallbackResponse>> fieldValuesHolderList = new ArrayList<Entry<CallbackExecutor, CallbackResponse>>(
                response.size());
        for (Entry<CallbackExecutor, CallbackResponse> e : response.entrySet()) {
            groupby.addAll(e.getKey().getCallbackMeasures());
            fieldValuesHolderList.add(e);
        }

        Meta meta = new Meta(groupby.toArray(new String[0]));
        // default result size 500
        SearchIndexResultSet result = new SearchIndexResultSet(meta, 500);
        // Use first response as base SEQ. Weak implementation. FIXME: WANGYUXUE.
        List<String> fieldValues = null;
        for (int index = 0; index < fieldValuesHolderList.get(0).getValue().getData().size(); index++) {
            fieldValues = new ArrayList<String>(groupby.size());
            CallbackMeasureVaue mv = (CallbackMeasureVaue) fieldValuesHolderList.get(0).getValue().getData()
                    .get(index);
            String key = mv.keySet().iterator().next();
            fieldValues.addAll(Arrays.asList(StringUtils.delimitedListToStringArray(key, RESPONSE_VALUE_SPLIT)));
            fieldValues.addAll(mv.get(key));
            for (int i = 1; i < fieldValuesHolderList.size(); i++) {
                CallbackMeasureVaue callbackMeasureValue = (CallbackMeasureVaue) fieldValuesHolderList.get(i)
                        .getValue().getData().get(index);
                if (key.equals(callbackMeasureValue.keySet().iterator().next())) {
                    fieldValues.addAll(callbackMeasureValue.get(key));
                } else {
                    LOGGER.error("Wrong SEQ of callback response of {} : {}",
                            fieldValuesHolderList.get(i).getKey().group.getKey(), callbackMeasureValue);
                }
            }
            SearchIndexResultRecord record = new SearchIndexResultRecord(fieldValues.toArray(new Serializable[0]),
                    StringUtils.collectionToCommaDelimitedString(fieldValues));

            result.addRecord(record);
        }

        return result;
    }

}