com.espertech.esper.epl.join.base.JoinSetComposerFactoryImpl.java Source code

Java tutorial

Introduction

Here is the source code for com.espertech.esper.epl.join.base.JoinSetComposerFactoryImpl.java

Source

/**************************************************************************************
 * Copyright (C) 2008 EsperTech, Inc. All rights reserved.                            *
 * http://esper.codehaus.org                                                          *
 * http://www.espertech.com                                                           *
 * ---------------------------------------------------------------------------------- *
 * The software in this package is published under the terms of the GPL license       *
 * a copy of which has been included with this distribution in the license.txt file.  *
 **************************************************************************************/
package com.espertech.esper.epl.join.base;

import com.espertech.esper.client.EventType;
import com.espertech.esper.collection.Pair;
import com.espertech.esper.core.StreamJoinAnalysisResult;
import com.espertech.esper.epl.expression.*;
import com.espertech.esper.epl.join.exec.base.ExecNode;
import com.espertech.esper.epl.join.plan.*;
import com.espertech.esper.epl.join.pollindex.*;
import com.espertech.esper.epl.join.table.EventTable;
import com.espertech.esper.epl.join.table.EventTableFactory;
import com.espertech.esper.epl.join.table.HistoricalStreamIndexList;
import com.espertech.esper.epl.spec.OuterJoinDesc;
import com.espertech.esper.epl.spec.SelectClauseStreamSelectorEnum;
import com.espertech.esper.type.OuterJoinType;
import com.espertech.esper.util.AuditPath;
import com.espertech.esper.util.DependencyGraph;
import com.espertech.esper.view.HistoricalEventViewable;
import com.espertech.esper.view.Viewable;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import java.lang.annotation.Annotation;
import java.util.*;

/**
 * Factory for building a {@link JoinSetComposer} from analyzing filter nodes, for
 * fast join tuple result set composition.
 */
public class JoinSetComposerFactoryImpl implements JoinSetComposerFactory {
    private static final Log queryPlanLog = LogFactory.getLog(AuditPath.QUERYPLAN_LOG);

    /**
     * Builds join tuple composer.
     * @param outerJoinDescList - list of descriptors for outer join criteria
     * @param optionalFilterNode - filter tree for analysis to build indexes for fast access
     * @param streamTypes - types of streams
     * @param streamNames - names of streams
     * @param streamViews - leaf view per stream
     * @param selectStreamSelectorEnum - indicator for rstream or istream-only, for optimization
     * @return composer implementation
     * @throws ExprValidationException is thrown to indicate that
     * validation of view use in joins failed.
     */
    public JoinSetComposerDesc makeComposer(List<OuterJoinDesc> outerJoinDescList, ExprNode optionalFilterNode,
            EventType[] streamTypes, String[] streamNames, Viewable[] streamViews,
            SelectClauseStreamSelectorEnum selectStreamSelectorEnum,
            StreamJoinAnalysisResult streamJoinAnalysisResult, ExprEvaluatorContext exprEvaluatorContext,
            boolean queryPlanLogging, Annotation[] annotations) throws ExprValidationException {
        // Determine if there is a historical stream, and what dependencies exist
        DependencyGraph historicalDependencyGraph = new DependencyGraph(streamTypes.length);
        boolean[] isHistorical = new boolean[streamViews.length];
        boolean hasHistorical = false;
        for (int i = 0; i < streamViews.length; i++) {
            if (streamViews[i] instanceof HistoricalEventViewable) {
                HistoricalEventViewable historicalViewable = (HistoricalEventViewable) streamViews[i];
                isHistorical[i] = true;
                hasHistorical = true;
                SortedSet<Integer> streamsThisStreamDependsOn = historicalViewable.getRequiredStreams();
                historicalDependencyGraph.addDependency(i, streamsThisStreamDependsOn);
            }
        }

        if (log.isDebugEnabled()) {
            log.debug("Dependency graph: " + historicalDependencyGraph);
        }

        QueryStrategy[] queryStrategies;

        // Handle a join with a database or other historical data source for 2 streams
        if ((hasHistorical) && (streamViews.length == 2)) {
            return makeComposerHistorical2Stream(outerJoinDescList, optionalFilterNode, streamTypes, streamViews,
                    exprEvaluatorContext, queryPlanLogging);
        }

        boolean isOuterJoins = !OuterJoinDesc.consistsOfAllInnerJoins(outerJoinDescList);

        // Query graph for graph relationships between streams/historicals
        // For outer joins the query graph will just contain outer join relationships
        QueryGraph queryGraph = new QueryGraph(streamTypes.length);
        if (!outerJoinDescList.isEmpty()) {
            OuterJoinAnalyzer.analyze(outerJoinDescList, queryGraph);
            if (log.isDebugEnabled()) {
                log.debug(".makeComposer After outer join queryGraph=\n" + queryGraph);
            }
        }

        // Let the query graph reflect the where-clause
        if (optionalFilterNode != null) {
            // Analyze relationships between streams using the optional filter expression.
            // Relationships are properties in AND and EQUALS nodes of joins.
            FilterExprAnalyzer.analyze(optionalFilterNode, queryGraph, isOuterJoins);
            if (log.isDebugEnabled()) {
                log.debug(".makeComposer After filter expression queryGraph=\n" + queryGraph);
            }

            // Add navigation entries based on key and index property equivalency (a=b, b=c follows a=c)
            QueryGraph.fillEquivalentNav(streamTypes, queryGraph);
            if (log.isDebugEnabled()) {
                log.debug(".makeComposer After fill equiv. nav. queryGraph=\n" + queryGraph);
            }
        }

        // Historical index lists
        HistoricalStreamIndexList[] historicalStreamIndexLists = new HistoricalStreamIndexList[streamTypes.length];

        QueryPlan queryPlan = QueryPlanBuilder.getPlan(streamTypes, outerJoinDescList, queryGraph, streamNames,
                hasHistorical, isHistorical, historicalDependencyGraph, historicalStreamIndexLists,
                exprEvaluatorContext, streamJoinAnalysisResult, queryPlanLogging, annotations);

        // remove unused indexes - consider all streams or all unidirectional
        HashSet<String> usedIndexes = new HashSet<String>();
        QueryPlanIndex[] indexSpecs = queryPlan.getIndexSpecs();
        for (int streamNum = 0; streamNum < queryPlan.getExecNodeSpecs().length; streamNum++) {
            QueryPlanNode planNode = queryPlan.getExecNodeSpecs()[streamNum];
            if (planNode != null) {
                planNode.addIndexes(usedIndexes);
            }
        }
        for (QueryPlanIndex indexSpec : indexSpecs) {
            if (indexSpec == null) {
                continue;
            }
            Map<String, QueryPlanIndexItem> items = indexSpec.getItems();
            String[] indexNames = items.keySet().toArray(new String[items.size()]);
            for (String indexName : indexNames) {
                if (!usedIndexes.contains(indexName)) {
                    items.remove(indexName);
                }
            }
        }

        if (queryPlanLogging && queryPlanLog.isInfoEnabled()) {
            queryPlanLog.info("Query plan: " + queryPlan.toQueryPlan());
        }

        // Build indexes
        Map<String, EventTable>[] indexesPerStream = new HashMap[indexSpecs.length];
        for (int streamNo = 0; streamNo < indexSpecs.length; streamNo++) {
            if (indexSpecs[streamNo] == null) {
                continue;
            }

            Map<String, QueryPlanIndexItem> items = indexSpecs[streamNo].getItems();
            indexesPerStream[streamNo] = new LinkedHashMap<String, EventTable>();

            for (Map.Entry<String, QueryPlanIndexItem> entry : items.entrySet()) {
                EventTable index;
                if (streamJoinAnalysisResult.getViewExternal()[streamNo] != null) {
                    index = streamJoinAnalysisResult.getViewExternal()[streamNo]
                            .getJoinIndexTable(items.get(entry.getKey()));
                } else {
                    index = EventTableFactory.buildIndex(streamNo, items.get(entry.getKey()), streamTypes[streamNo],
                            false);
                }
                indexesPerStream[streamNo].put(entry.getKey(), index);
            }
        }

        // Build strategies
        QueryPlanNode[] queryExecSpecs = queryPlan.getExecNodeSpecs();
        queryStrategies = new QueryStrategy[queryExecSpecs.length];
        for (int i = 0; i < queryExecSpecs.length; i++) {
            QueryPlanNode planNode = queryExecSpecs[i];
            if (planNode == null) {
                log.debug(".makeComposer No execution node for stream " + i + " '" + streamNames[i] + "'");
                continue;
            }

            ExecNode executionNode = planNode.makeExec(indexesPerStream, streamTypes, streamViews,
                    historicalStreamIndexLists, streamJoinAnalysisResult.getViewExternal());

            if (log.isDebugEnabled()) {
                log.debug(".makeComposer Execution nodes for stream " + i + " '" + streamNames[i] + "' : \n"
                        + ExecNode.print(executionNode));
            }

            queryStrategies[i] = new ExecNodeQueryStrategy(i, streamTypes.length, executionNode);
        }

        // If this is not unidirectional and not a self-join (excluding self-outer-join)
        if ((!streamJoinAnalysisResult.isUnidirectional())
                && (!streamJoinAnalysisResult.isPureSelfJoin() || !outerJoinDescList.isEmpty())) {
            JoinSetComposer composer;
            if (hasHistorical) {
                composer = new JoinSetComposerHistoricalImpl(indexesPerStream, queryStrategies, streamViews,
                        exprEvaluatorContext);
            } else {
                composer = new JoinSetComposerImpl(indexesPerStream, queryStrategies,
                        streamJoinAnalysisResult.isPureSelfJoin(), exprEvaluatorContext);
            }

            // rewrite the filter expression for all-inner joins in case "on"-clause outer join syntax was used to include those expressions
            ExprNode filterExpression = getFilterExpressionInclOnClause(optionalFilterNode, outerJoinDescList);

            ExprEvaluator postJoinEval = filterExpression == null ? null : filterExpression.getExprEvaluator();
            return new JoinSetComposerDesc(composer, postJoinEval);
        } else {
            QueryStrategy driver;
            int unidirectionalStream;
            if (streamJoinAnalysisResult.getUnidirectionalStreamNumber() != -1) {
                unidirectionalStream = streamJoinAnalysisResult.getUnidirectionalStreamNumber();
                driver = queryStrategies[unidirectionalStream];
            } else {
                unidirectionalStream = 0;
                driver = queryStrategies[0];
            }
            JoinSetComposer composer = new JoinSetComposerStreamToWinImpl(indexesPerStream,
                    streamJoinAnalysisResult.isPureSelfJoin(), unidirectionalStream, driver,
                    streamJoinAnalysisResult.getUnidirectionalNonDriving());
            ExprEvaluator postJoinEval = optionalFilterNode == null ? null : optionalFilterNode.getExprEvaluator();
            return new JoinSetComposerDesc(composer, postJoinEval);
        }
    }

    private ExprNode getFilterExpressionInclOnClause(ExprNode optionalFilterNode,
            List<OuterJoinDesc> outerJoinDescList) throws ExprValidationException {
        if (optionalFilterNode == null) { // no need to add as query planning is fully based on on-clause
            return null;
        }
        if (outerJoinDescList.isEmpty()) { // not an outer-join syntax
            return optionalFilterNode;
        }
        if (!OuterJoinDesc.consistsOfAllInnerJoins(outerJoinDescList)) { // all-inner joins
            return optionalFilterNode;
        }
        ExprAndNode andNode = new ExprAndNodeImpl();
        andNode.addChildNode(optionalFilterNode);
        for (OuterJoinDesc outerJoinDesc : outerJoinDescList) {
            andNode.addChildNode(outerJoinDesc.makeExprNode(null));
        }
        andNode.validate(null);
        return andNode;
    }

    private JoinSetComposerDesc makeComposerHistorical2Stream(List<OuterJoinDesc> outerJoinDescList,
            ExprNode optionalFilterNode, EventType[] streamTypes, Viewable[] streamViews,
            ExprEvaluatorContext exprEvaluatorContext, boolean queryPlanLogging) throws ExprValidationException {
        QueryStrategy[] queryStrategies;

        // No tables for any streams
        queryStrategies = new QueryStrategy[streamTypes.length];

        int polledViewNum = 0;
        int streamViewNum = 1;
        if (streamViews[1] instanceof HistoricalEventViewable) {
            streamViewNum = 0;
            polledViewNum = 1;
        }

        // if all-historical join, check dependency
        boolean isAllHistoricalNoSubordinate = false;
        if ((streamViews[0] instanceof HistoricalEventViewable)
                && (streamViews[1] instanceof HistoricalEventViewable)) {
            DependencyGraph graph = new DependencyGraph(2);
            graph.addDependency(0, ((HistoricalEventViewable) streamViews[0]).getRequiredStreams());
            graph.addDependency(1, ((HistoricalEventViewable) streamViews[1]).getRequiredStreams());
            if (graph.getFirstCircularDependency() != null) {
                throw new ExprValidationException("Circular dependency detected between historical streams");
            }

            // if both streams are independent
            if (graph.getRootNodes().size() == 2) {
                isAllHistoricalNoSubordinate = true; // No parameters used by either historical
            } else {
                if ((graph.getDependenciesForStream(0).size() == 0)) {
                    streamViewNum = 0;
                    polledViewNum = 1;
                } else {
                    streamViewNum = 1;
                    polledViewNum = 0;
                }
            }
        }

        // Build an outer join expression node
        boolean isOuterJoin = false;
        ExprNode outerJoinEqualsNode = null;
        if (!outerJoinDescList.isEmpty()) {
            OuterJoinDesc outerJoinDesc = outerJoinDescList.get(0);
            if (outerJoinDesc.getOuterJoinType().equals(OuterJoinType.FULL)) {
                isOuterJoin = true;
            } else if ((outerJoinDesc.getOuterJoinType().equals(OuterJoinType.LEFT)) && (streamViewNum == 0)) {
                isOuterJoin = true;
            } else if ((outerJoinDesc.getOuterJoinType().equals(OuterJoinType.RIGHT)) && (streamViewNum == 1)) {
                isOuterJoin = true;
            }

            outerJoinEqualsNode = outerJoinDesc.makeExprNode(exprEvaluatorContext);
        }

        // Determine filter for indexing purposes
        ExprNode filterForIndexing = null;
        if ((outerJoinEqualsNode != null) && (optionalFilterNode != null)) // both filter and outer join, add
        {
            filterForIndexing = new ExprAndNodeImpl();
            filterForIndexing.addChildNode(optionalFilterNode);
            filterForIndexing.addChildNode(outerJoinEqualsNode);
        } else if ((outerJoinEqualsNode == null) && (optionalFilterNode != null)) {
            filterForIndexing = optionalFilterNode;
        } else if (outerJoinEqualsNode != null) {
            filterForIndexing = outerJoinEqualsNode;
        }

        Pair<HistoricalIndexLookupStrategy, PollResultIndexingStrategy> indexStrategies = determineIndexing(
                filterForIndexing, streamTypes[polledViewNum], streamTypes[streamViewNum], polledViewNum,
                streamViewNum);

        if (queryPlanLogging && queryPlanLog.isInfoEnabled()) {
            queryPlanLog.info("historical lookup strategy: " + indexStrategies.getFirst().toQueryPlan());
            queryPlanLog.info("historical index strategy: " + indexStrategies.getSecond().toQueryPlan());
        }

        HistoricalEventViewable viewable = (HistoricalEventViewable) streamViews[polledViewNum];
        ExprEvaluator outerJoinEqualsNodeEval = outerJoinEqualsNode == null ? null
                : outerJoinEqualsNode.getExprEvaluator();
        queryStrategies[streamViewNum] = new HistoricalDataQueryStrategy(streamViewNum, polledViewNum, viewable,
                isOuterJoin, outerJoinEqualsNodeEval, indexStrategies.getFirst(), indexStrategies.getSecond());

        // for strictly historical joins, create a query strategy for the non-subordinate historical view
        if (isAllHistoricalNoSubordinate) {
            isOuterJoin = false;
            if (!outerJoinDescList.isEmpty()) {
                OuterJoinDesc outerJoinDesc = outerJoinDescList.get(0);
                if (outerJoinDesc.getOuterJoinType().equals(OuterJoinType.FULL)) {
                    isOuterJoin = true;
                } else if ((outerJoinDesc.getOuterJoinType().equals(OuterJoinType.LEFT)) && (polledViewNum == 0)) {
                    isOuterJoin = true;
                } else if ((outerJoinDesc.getOuterJoinType().equals(OuterJoinType.RIGHT)) && (polledViewNum == 1)) {
                    isOuterJoin = true;
                }
            }

            viewable = (HistoricalEventViewable) streamViews[streamViewNum];
            queryStrategies[polledViewNum] = new HistoricalDataQueryStrategy(polledViewNum, streamViewNum, viewable,
                    isOuterJoin, outerJoinEqualsNodeEval, new HistoricalIndexLookupStrategyNoIndex(),
                    new PollResultIndexingStrategyNoIndex());
        }

        JoinSetComposer composer = new JoinSetComposerHistoricalImpl(null, queryStrategies, streamViews,
                exprEvaluatorContext);
        ExprEvaluator postJoinEval = optionalFilterNode == null ? null : optionalFilterNode.getExprEvaluator();
        return new JoinSetComposerDesc(composer, postJoinEval);
    }

    private static Pair<HistoricalIndexLookupStrategy, PollResultIndexingStrategy> determineIndexing(
            ExprNode filterForIndexing, EventType polledViewType, EventType streamViewType, int polledViewStreamNum,
            int streamViewStreamNum) {
        // No filter means unindexed event tables
        if (filterForIndexing == null) {
            return new Pair<HistoricalIndexLookupStrategy, PollResultIndexingStrategy>(
                    new HistoricalIndexLookupStrategyNoIndex(), new PollResultIndexingStrategyNoIndex());
        }

        // analyze query graph; Whereas stream0=named window, stream1=delete-expr filter
        QueryGraph queryGraph = new QueryGraph(2);
        FilterExprAnalyzer.analyze(filterForIndexing, queryGraph, false);

        return determineIndexing(queryGraph, polledViewType, streamViewType, polledViewStreamNum,
                streamViewStreamNum);
    }

    /**
     * Constructs indexing and lookup strategy for a given relationship that a historical stream may have with another
     * stream (historical or not) that looks up into results of a poll of a historical stream.
     * <p>
     * The term "polled" refers to the assumed-historical stream.
     * @param queryGraph relationship representation of where-clause filter and outer join on-expressions
     * @param polledViewType the event type of the historical that is indexed
     * @param streamViewType the event type of the stream looking up in indexes
     * @param polledViewStreamNum the stream number of the historical that is indexed
     * @param streamViewStreamNum the stream number of the historical that is looking up
     * @return indexing and lookup strategy pair
     */
    public static Pair<HistoricalIndexLookupStrategy, PollResultIndexingStrategy> determineIndexing(
            QueryGraph queryGraph, EventType polledViewType, EventType streamViewType, int polledViewStreamNum,
            int streamViewStreamNum) {
        QueryGraphValue queryGraphValue = queryGraph.getGraphValue(streamViewStreamNum, polledViewStreamNum);
        QueryGraphValuePairHashKeyIndex hashKeysAndIndes = queryGraphValue.getHashKeyProps();
        QueryGraphValuePairRangeIndex rangeKeysAndIndex = queryGraphValue.getRangeProps();

        // index and key property names
        List<QueryGraphValueEntryHashKeyed> hashKeys = hashKeysAndIndes.getKeys();
        String[] hashIndexes = hashKeysAndIndes.getIndexed();
        List<QueryGraphValueEntryRange> rangeKeys = rangeKeysAndIndex.getKeys();
        String[] rangeIndexes = rangeKeysAndIndex.getIndexed();

        // If the analysis revealed no join columns, must use the brute-force full table scan
        if (hashKeys.isEmpty() && rangeKeys.isEmpty()) {
            return new Pair<HistoricalIndexLookupStrategy, PollResultIndexingStrategy>(
                    new HistoricalIndexLookupStrategyNoIndex(), new PollResultIndexingStrategyNoIndex());
        }

        CoercionDesc keyCoercionTypes = CoercionUtil.getCoercionTypesHash(
                new EventType[] { streamViewType, polledViewType }, 0, 1, hashKeys, hashIndexes);

        if (rangeKeys.isEmpty()) {
            // No coercion
            if (!keyCoercionTypes.isCoerce()) {
                if (hashIndexes.length == 1) {
                    PollResultIndexingStrategyIndexSingle indexing = new PollResultIndexingStrategyIndexSingle(
                            polledViewStreamNum, polledViewType, hashIndexes[0]);
                    HistoricalIndexLookupStrategy strategy = new HistoricalIndexLookupStrategyIndexSingle(
                            streamViewStreamNum, hashKeys.get(0));
                    return new Pair<HistoricalIndexLookupStrategy, PollResultIndexingStrategy>(strategy, indexing);
                } else {
                    PollResultIndexingStrategyIndex indexing = new PollResultIndexingStrategyIndex(
                            polledViewStreamNum, polledViewType, hashIndexes);
                    HistoricalIndexLookupStrategy strategy = new HistoricalIndexLookupStrategyIndex(streamViewType,
                            streamViewStreamNum, hashKeys);
                    return new Pair<HistoricalIndexLookupStrategy, PollResultIndexingStrategy>(strategy, indexing);
                }
            }

            // With coercion, same lookup strategy as the index coerces
            if (hashIndexes.length == 1) {
                PollResultIndexingStrategy indexing = new PollResultIndexingStrategyIndexCoerceSingle(
                        polledViewStreamNum, polledViewType, hashIndexes[0],
                        keyCoercionTypes.getCoercionTypes()[0]);
                HistoricalIndexLookupStrategy strategy = new HistoricalIndexLookupStrategyIndexSingle(
                        streamViewStreamNum, hashKeys.get(0));
                return new Pair<HistoricalIndexLookupStrategy, PollResultIndexingStrategy>(strategy, indexing);
            } else {
                PollResultIndexingStrategy indexing = new PollResultIndexingStrategyIndexCoerce(polledViewStreamNum,
                        polledViewType, hashIndexes, keyCoercionTypes.getCoercionTypes());
                HistoricalIndexLookupStrategy strategy = new HistoricalIndexLookupStrategyIndex(streamViewType,
                        streamViewStreamNum, hashKeys);
                return new Pair<HistoricalIndexLookupStrategy, PollResultIndexingStrategy>(strategy, indexing);
            }
        } else {
            CoercionDesc rangeCoercionTypes = CoercionUtil.getCoercionTypesRange(
                    new EventType[] { streamViewType, polledViewType }, 1, rangeIndexes, rangeKeys);
            if (rangeKeys.size() == 1 && hashKeys.size() == 0) {
                Class rangeCoercionType = rangeCoercionTypes.isCoerce() ? rangeCoercionTypes.getCoercionTypes()[0]
                        : null;
                PollResultIndexingStrategySorted indexing = new PollResultIndexingStrategySorted(
                        polledViewStreamNum, polledViewType, rangeIndexes[0], rangeCoercionType);
                HistoricalIndexLookupStrategy strategy = new HistoricalIndexLookupStrategySorted(
                        streamViewStreamNum, rangeKeys.get(0));
                return new Pair<HistoricalIndexLookupStrategy, PollResultIndexingStrategy>(strategy, indexing);
            } else {
                PollResultIndexingStrategyComposite indexing = new PollResultIndexingStrategyComposite(
                        polledViewStreamNum, polledViewType, hashIndexes, keyCoercionTypes.getCoercionTypes(),
                        rangeIndexes, rangeCoercionTypes.getCoercionTypes());
                HistoricalIndexLookupStrategy strategy = new HistoricalIndexLookupStrategyComposite(
                        streamViewStreamNum, hashKeys, keyCoercionTypes.getCoercionTypes(), rangeKeys,
                        rangeCoercionTypes.getCoercionTypes());
                return new Pair<HistoricalIndexLookupStrategy, PollResultIndexingStrategy>(strategy, indexing);
            }
        }
    }

    private static final Log log = LogFactory.getLog(JoinSetComposerFactoryImpl.class);
}