Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one or more contributor license * agreements. See the NOTICE file distributed with this work for additional information regarding * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. You may obtain a * copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package org.apache.hadoop.hive.ql.optimizer; import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; import java.util.BitSet; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.Stack; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.ColumnInfo; import org.apache.hadoop.hive.ql.exec.FileSinkOperator; import org.apache.hadoop.hive.ql.exec.FilterOperator; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; import org.apache.hadoop.hive.ql.exec.GroupByOperator; import org.apache.hadoop.hive.ql.exec.JoinOperator; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; import org.apache.hadoop.hive.ql.exec.RowSchema; import org.apache.hadoop.hive.ql.exec.SelectOperator; import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.exec.UDF; import org.apache.hadoop.hive.ql.exec.UDFArgumentException; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.VirtualColumn; import org.apache.hadoop.hive.ql.optimizer.ConstantPropagateProcCtx.ConstantPropagateOption; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.DynamicPartitionCtx; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils; import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; import org.apache.hadoop.hive.ql.plan.FileSinkDesc; import org.apache.hadoop.hive.ql.plan.GroupByDesc; import org.apache.hadoop.hive.ql.plan.JoinCondDesc; import org.apache.hadoop.hive.ql.plan.JoinDesc; import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc; import org.apache.hadoop.hive.ql.plan.TableScanDesc; import org.apache.hadoop.hive.ql.udf.UDFType; import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseCompare; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFCase; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFNvl; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNot; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotEqual; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotNull; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNull; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFStruct; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToUnixTimeStamp; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFUnixTimeStamp; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFWhen; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.StandardConstantStructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; /** * Factory for generating the different node processors used by ConstantPropagate. */ public final class ConstantPropagateProcFactory { protected static final Logger LOG = LoggerFactory.getLogger(ConstantPropagateProcFactory.class.getName()); protected static Set<Class<?>> propagatableUdfs = new HashSet<Class<?>>(); static { propagatableUdfs.add(GenericUDFOPAnd.class); }; private ConstantPropagateProcFactory() { // prevent instantiation } /** * Get ColumnInfo from column expression. * * @param rr * @param desc * @return */ public static ColumnInfo resolveColumn(RowSchema rs, ExprNodeColumnDesc desc) { ColumnInfo ci = rs.getColumnInfo(desc.getTabAlias(), desc.getColumn()); if (ci == null) { ci = rs.getColumnInfo(desc.getColumn()); } if (ci == null) { return null; } return ci; } private static final Set<PrimitiveCategory> unSupportedTypes = ImmutableSet.<PrimitiveCategory>builder() .add(PrimitiveCategory.VARCHAR).add(PrimitiveCategory.CHAR).build(); private static final Set<PrimitiveCategory> unsafeConversionTypes = ImmutableSet.<PrimitiveCategory>builder() .add(PrimitiveCategory.STRING).add(PrimitiveCategory.VARCHAR).add(PrimitiveCategory.CHAR).build(); /** * Cast type from expression type to expected type ti. * * @param desc constant expression * @param ti expected type info * @return cast constant, or null if the type cast failed. */ private static ExprNodeConstantDesc typeCast(ExprNodeDesc desc, TypeInfo ti) { return typeCast(desc, ti, false); } /** * Cast type from expression type to expected type ti. * * @param desc constant expression * @param ti expected type info * @param performSafeTypeCast when true then don't perform typecast because it could be unsafe (loosing leading zeroes etc.) * @return cast constant, or null if the type cast failed. */ private static ExprNodeConstantDesc typeCast(ExprNodeDesc desc, TypeInfo ti, boolean performSafeTypeCast) { if (desc instanceof ExprNodeConstantDesc && null == ((ExprNodeConstantDesc) desc).getValue()) { return null; } if (!(ti instanceof PrimitiveTypeInfo) || !(desc.getTypeInfo() instanceof PrimitiveTypeInfo)) { return null; } PrimitiveTypeInfo priti = (PrimitiveTypeInfo) ti; PrimitiveTypeInfo descti = (PrimitiveTypeInfo) desc.getTypeInfo(); if (unSupportedTypes.contains(priti.getPrimitiveCategory()) || unSupportedTypes.contains(descti.getPrimitiveCategory())) { // FIXME: support template types. It currently has conflict with ExprNodeConstantDesc if (LOG.isDebugEnabled()) { LOG.debug("Unsupported types " + priti + "; " + descti); } return null; } // We shouldn't cast strings to other types because that can break original data in cases of // leading zeros or zeros trailing after decimal point. // Example: "000126" => 126 => "126" boolean brokenDataTypesCombination = unsafeConversionTypes.contains(priti.getPrimitiveCategory()) && !unsafeConversionTypes.contains(descti.getPrimitiveCategory()); if (performSafeTypeCast && brokenDataTypesCombination) { if (LOG.isDebugEnabled()) { LOG.debug("Unsupported cast " + priti + "; " + descti); } return null; } if (LOG.isDebugEnabled()) { LOG.debug("Casting " + desc + " to type " + ti); } ExprNodeConstantDesc c = (ExprNodeConstantDesc) desc; if (null != c.getFoldedFromVal() && priti.getTypeName().equals(serdeConstants.STRING_TYPE_NAME)) { // avoid double casting to preserve original string representation of constant. return new ExprNodeConstantDesc(c.getFoldedFromVal()); } ObjectInspector origOI = TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(desc.getTypeInfo()); ObjectInspector oi = TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(ti); Converter converter = ObjectInspectorConverters.getConverter(origOI, oi); Object convObj = converter.convert(c.getValue()); // Convert integer related types because converters are not sufficient if (convObj instanceof Integer) { switch (priti.getPrimitiveCategory()) { case BYTE: convObj = new Byte((byte) (((Integer) convObj).intValue())); break; case SHORT: convObj = new Short((short) ((Integer) convObj).intValue()); break; case LONG: convObj = new Long(((Integer) convObj).intValue()); default: } } return new ExprNodeConstantDesc(ti, convObj); } public static ExprNodeDesc foldExpr(ExprNodeGenericFuncDesc funcDesc) { GenericUDF udf = funcDesc.getGenericUDF(); if (!isDeterministicUdf(udf, funcDesc.getChildren())) { return funcDesc; } return evaluateFunction(funcDesc.getGenericUDF(), funcDesc.getChildren(), funcDesc.getChildren()); } /** * Fold input expression desc. * * @param desc folding expression * @param constants current propagated constant map * @param cppCtx * @param op processing operator * @param propagate if true, assignment expressions will be added to constants. * @return fold expression * @throws UDFArgumentException */ private static ExprNodeDesc foldExpr(ExprNodeDesc desc, Map<ColumnInfo, ExprNodeDesc> constants, ConstantPropagateProcCtx cppCtx, Operator<? extends Serializable> op, int tag, boolean propagate) throws UDFArgumentException { if (cppCtx.getConstantPropagateOption() == ConstantPropagateOption.SHORTCUT) { return foldExprShortcut(desc, constants, cppCtx, op, tag, propagate); } return foldExprFull(desc, constants, cppCtx, op, tag, propagate); } /** * Combines the logical not() operator with the child operator if possible. * @param desc the expression to be evaluated * @return the new expression to be replaced * @throws UDFArgumentException */ private static ExprNodeDesc foldNegative(ExprNodeDesc desc) throws UDFArgumentException { if (desc instanceof ExprNodeGenericFuncDesc) { ExprNodeGenericFuncDesc funcDesc = (ExprNodeGenericFuncDesc) desc; GenericUDF udf = funcDesc.getGenericUDF(); if (udf instanceof GenericUDFOPNot) { ExprNodeDesc child = funcDesc.getChildren().get(0); if (child instanceof ExprNodeGenericFuncDesc) { ExprNodeGenericFuncDesc childDesc = (ExprNodeGenericFuncDesc) child; GenericUDF childUDF = childDesc.getGenericUDF(); List<ExprNodeDesc> grandChildren = child.getChildren(); if (childUDF instanceof GenericUDFBaseCompare || childUDF instanceof GenericUDFOPNull || childUDF instanceof GenericUDFOPNotNull) { List<ExprNodeDesc> newGrandChildren = new ArrayList<ExprNodeDesc>(); for (ExprNodeDesc grandChild : grandChildren) { newGrandChildren.add(foldNegative(grandChild)); } return ExprNodeGenericFuncDesc.newInstance(childUDF.negative(), newGrandChildren); } else if (childUDF instanceof GenericUDFOPAnd || childUDF instanceof GenericUDFOPOr) { List<ExprNodeDesc> newGrandChildren = new ArrayList<ExprNodeDesc>(); for (ExprNodeDesc grandChild : grandChildren) { newGrandChildren.add(foldNegative(ExprNodeGenericFuncDesc .newInstance(new GenericUDFOPNot(), Arrays.asList(grandChild)))); } return ExprNodeGenericFuncDesc.newInstance(childUDF.negative(), newGrandChildren); } else if (childUDF instanceof GenericUDFOPNot) { return foldNegative(child.getChildren().get(0)); } else { // For operator like if() that cannot be handled, leave not() as it // is and continue processing the children List<ExprNodeDesc> newGrandChildren = new ArrayList<ExprNodeDesc>(); for (ExprNodeDesc grandChild : grandChildren) { newGrandChildren.add(foldNegative(grandChild)); } childDesc.setChildren(newGrandChildren); return funcDesc; } } } } return desc; } /** * Fold input expression desc, only performing short-cutting. * * Unnecessary AND/OR operations involving a constant true/false value will be eliminated. * * @param desc folding expression * @param constants current propagated constant map * @param cppCtx * @param op processing operator * @param propagate if true, assignment expressions will be added to constants. * @return fold expression * @throws UDFArgumentException */ private static ExprNodeDesc foldExprShortcut(ExprNodeDesc desc, Map<ColumnInfo, ExprNodeDesc> constants, ConstantPropagateProcCtx cppCtx, Operator<? extends Serializable> op, int tag, boolean propagate) throws UDFArgumentException { // Combine NOT operator with the child operator. Otherwise, the following optimization // from bottom up could lead to incorrect result, such as not(x > 3 and x is not null), // should not be optimized to not(x > 3), but (x <=3 or x is null). desc = foldNegative(desc); if (desc instanceof ExprNodeGenericFuncDesc) { ExprNodeGenericFuncDesc funcDesc = (ExprNodeGenericFuncDesc) desc; GenericUDF udf = funcDesc.getGenericUDF(); boolean propagateNext = propagate && propagatableUdfs.contains(udf.getClass()); List<ExprNodeDesc> newExprs = new ArrayList<ExprNodeDesc>(); for (ExprNodeDesc childExpr : desc.getChildren()) { newExprs.add(foldExpr(childExpr, constants, cppCtx, op, tag, propagateNext)); } // Don't evaluate nondeterministic function since the value can only calculate during runtime. if (!isDeterministicUdf(udf, newExprs)) { if (LOG.isDebugEnabled()) { LOG.debug("Function " + udf.getClass() + " is undeterministic. Don't evaluate immediately."); } ((ExprNodeGenericFuncDesc) desc).setChildren(newExprs); return desc; } // Check if the function can be short cut. ExprNodeDesc shortcut = shortcutFunction(udf, newExprs, op); if (shortcut != null) { if (LOG.isDebugEnabled()) { LOG.debug("Folding expression:" + desc + " -> " + shortcut); } return shortcut; } ((ExprNodeGenericFuncDesc) desc).setChildren(newExprs); } return desc; } /** * Fold input expression desc. * * This function recursively checks if any subexpression of a specified expression * can be evaluated to be constant and replaces such subexpression with the constant. * If the expression is a deterministic UDF and all the subexpressions are constants, * the value will be calculated immediately (during compilation time vs. runtime). * e.g.: * concat(year, month) => 200112 for year=2001, month=12 since concat is deterministic UDF * unix_timestamp(time) => unix_timestamp(123) for time=123 since unix_timestamp is nondeterministic UDF * @param desc folding expression * @param constants current propagated constant map * @param cppCtx * @param op processing operator * @param propagate if true, assignment expressions will be added to constants. * @return fold expression * @throws UDFArgumentException */ private static ExprNodeDesc foldExprFull(ExprNodeDesc desc, Map<ColumnInfo, ExprNodeDesc> constants, ConstantPropagateProcCtx cppCtx, Operator<? extends Serializable> op, int tag, boolean propagate) throws UDFArgumentException { // Combine NOT operator with the child operator. Otherwise, the following optimization // from bottom up could lead to incorrect result, such as not(x > 3 and x is not null), // should not be optimized to not(x > 3), but (x <=3 or x is null). desc = foldNegative(desc); if (desc instanceof ExprNodeGenericFuncDesc) { ExprNodeGenericFuncDesc funcDesc = (ExprNodeGenericFuncDesc) desc; GenericUDF udf = funcDesc.getGenericUDF(); boolean propagateNext = propagate && propagatableUdfs.contains(udf.getClass()); List<ExprNodeDesc> newExprs = new ArrayList<ExprNodeDesc>(); for (ExprNodeDesc childExpr : desc.getChildren()) { newExprs.add(foldExpr(childExpr, constants, cppCtx, op, tag, propagateNext)); } // Don't evaluate nondeterministic function since the value can only calculate during runtime. if (!isDeterministicUdf(udf, newExprs)) { if (LOG.isDebugEnabled()) { LOG.debug("Function " + udf.getClass() + " is undeterministic. Don't evaluate immediately."); } ((ExprNodeGenericFuncDesc) desc).setChildren(newExprs); return desc; } else { // If all child expressions of deterministic function are constants, evaluate such UDF immediately ExprNodeDesc constant = evaluateFunction(udf, newExprs, desc.getChildren()); if (constant != null) { if (LOG.isDebugEnabled()) { LOG.debug("Folding expression:" + desc + " -> " + constant); } return constant; } else { // Check if the function can be short cut. ExprNodeDesc shortcut = shortcutFunction(udf, newExprs, op); if (shortcut != null) { if (LOG.isDebugEnabled()) { LOG.debug("Folding expression:" + desc + " -> " + shortcut); } return shortcut; } ((ExprNodeGenericFuncDesc) desc).setChildren(newExprs); } // If in some selected binary operators (=, is null, etc), one of the // expressions are // constant, add them to colToConstants as half-deterministic columns. if (propagate) { propagate(udf, newExprs, op.getSchema(), constants); } } return desc; } else if (desc instanceof ExprNodeColumnDesc) { if (op.getParentOperators() == null || op.getParentOperators().isEmpty()) { return desc; } Operator<? extends Serializable> parent = op.getParentOperators().get(tag); ExprNodeDesc col = evaluateColumn((ExprNodeColumnDesc) desc, cppCtx, parent); if (col != null) { if (LOG.isDebugEnabled()) { LOG.debug("Folding expression:" + desc + " -> " + col); } return col; } } return desc; } private static boolean isDeterministicUdf(GenericUDF udf, List<ExprNodeDesc> children) { UDFType udfType = udf.getClass().getAnnotation(UDFType.class); if (udf instanceof GenericUDFBridge) { udfType = ((GenericUDFBridge) udf).getUdfClass().getAnnotation(UDFType.class); } if (udfType.deterministic() == false) { if (udf.getClass().equals(GenericUDFUnixTimeStamp.class) && children != null && children.size() > 0) { // unix_timestamp is polymorphic (ignore class annotations) return true; } return false; } // If udf is requiring additional jars, we can't determine the result in // compile time. String[] files; String[] jars; if (udf instanceof GenericUDFBridge) { GenericUDFBridge bridge = (GenericUDFBridge) udf; String udfClassName = bridge.getUdfClassName(); try { UDF udfInternal = (UDF) Class .forName(bridge.getUdfClassName(), true, Utilities.getSessionSpecifiedClassLoader()) .newInstance(); files = udfInternal.getRequiredFiles(); jars = udfInternal.getRequiredJars(); } catch (Exception e) { LOG.error("The UDF implementation class '" + udfClassName + "' is not present in the class path"); return false; } } else { files = udf.getRequiredFiles(); jars = udf.getRequiredJars(); } if (files != null || jars != null) { return false; } return true; } /** * Propagate assignment expression, adding an entry into constant map constants. * * @param udf expression UDF, currently only 2 UDFs are supported: '=' and 'is null'. * @param newExprs child expressions (parameters). * @param cppCtx * @param op * @param constants */ private static void propagate(GenericUDF udf, List<ExprNodeDesc> newExprs, RowSchema rs, Map<ColumnInfo, ExprNodeDesc> constants) { if (udf instanceof GenericUDFOPEqual) { ExprNodeDesc lOperand = newExprs.get(0); ExprNodeDesc rOperand = newExprs.get(1); ExprNodeConstantDesc v; if (lOperand instanceof ExprNodeConstantDesc) { v = (ExprNodeConstantDesc) lOperand; } else if (rOperand instanceof ExprNodeConstantDesc) { v = (ExprNodeConstantDesc) rOperand; } else { // we need a constant on one side. return; } // If both sides are constants, there is nothing to propagate ExprNodeColumnDesc c = ExprNodeDescUtils.getColumnExpr(lOperand); if (null == c) { c = ExprNodeDescUtils.getColumnExpr(rOperand); } if (null == c) { // we need a column expression on other side. return; } ColumnInfo ci = resolveColumn(rs, c); if (ci != null) { if (LOG.isDebugEnabled()) { LOG.debug("Filter " + udf + " is identified as a value assignment, propagate it."); } if (!v.getTypeInfo().equals(ci.getType())) { v = typeCast(v, ci.getType(), true); } if (v != null) { constants.put(ci, v); } } } else if (udf instanceof GenericUDFOPNull) { ExprNodeDesc operand = newExprs.get(0); if (operand instanceof ExprNodeColumnDesc) { if (LOG.isDebugEnabled()) { LOG.debug("Filter " + udf + " is identified as a value assignment, propagate it."); } ExprNodeColumnDesc c = (ExprNodeColumnDesc) operand; ColumnInfo ci = resolveColumn(rs, c); if (ci != null) { constants.put(ci, new ExprNodeConstantDesc(ci.getType(), null)); } } } } private static ExprNodeDesc shortcutFunction(GenericUDF udf, List<ExprNodeDesc> newExprs, Operator<? extends Serializable> op) throws UDFArgumentException { if (udf instanceof GenericUDFOPEqual) { assert newExprs.size() == 2; boolean foundUDFInFirst = false; ExprNodeGenericFuncDesc caseOrWhenexpr = null; if (newExprs.get(0) instanceof ExprNodeGenericFuncDesc) { caseOrWhenexpr = (ExprNodeGenericFuncDesc) newExprs.get(0); if (caseOrWhenexpr.getGenericUDF() instanceof GenericUDFWhen || caseOrWhenexpr.getGenericUDF() instanceof GenericUDFCase) { foundUDFInFirst = true; } } if (!foundUDFInFirst && newExprs.get(1) instanceof ExprNodeGenericFuncDesc) { caseOrWhenexpr = (ExprNodeGenericFuncDesc) newExprs.get(1); if (!(caseOrWhenexpr.getGenericUDF() instanceof GenericUDFWhen || caseOrWhenexpr.getGenericUDF() instanceof GenericUDFCase)) { return null; } } if (null == caseOrWhenexpr) { // we didn't find case or when udf return null; } GenericUDF childUDF = caseOrWhenexpr.getGenericUDF(); List<ExprNodeDesc> children = caseOrWhenexpr.getChildren(); int i; if (childUDF instanceof GenericUDFWhen) { for (i = 1; i < children.size(); i += 2) { children.set(i, ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPEqual(), Lists.newArrayList(children.get(i), newExprs.get(foundUDFInFirst ? 1 : 0)))); } if (children.size() % 2 == 1) { i = children.size() - 1; children.set(i, ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPEqual(), Lists.newArrayList(children.get(i), newExprs.get(foundUDFInFirst ? 1 : 0)))); } // after constant folding of child expression the return type of UDFWhen might have changed, // so recreate the expression ExprNodeGenericFuncDesc newCaseOrWhenExpr = ExprNodeGenericFuncDesc.newInstance(childUDF, caseOrWhenexpr.getFuncText(), children); return newCaseOrWhenExpr; } else if (childUDF instanceof GenericUDFCase) { for (i = 2; i < children.size(); i += 2) { children.set(i, ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPEqual(), Lists.newArrayList(children.get(i), newExprs.get(foundUDFInFirst ? 1 : 0)))); } if (children.size() % 2 == 0) { i = children.size() - 1; children.set(i, ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPEqual(), Lists.newArrayList(children.get(i), newExprs.get(foundUDFInFirst ? 1 : 0)))); } // after constant folding of child expression the return type of UDFCase might have changed, // so recreate the expression ExprNodeGenericFuncDesc newCaseOrWhenExpr = ExprNodeGenericFuncDesc.newInstance(childUDF, caseOrWhenexpr.getFuncText(), children); return newCaseOrWhenExpr; } else { // cant happen return null; } } if (udf instanceof GenericUDFOPAnd) { final BitSet positionsToRemove = new BitSet(); final List<ExprNodeDesc> notNullExprs = new ArrayList<ExprNodeDesc>(); final List<Integer> notNullExprsPositions = new ArrayList<Integer>(); final List<ExprNodeDesc> compareExprs = new ArrayList<ExprNodeDesc>(); for (int i = 0; i < newExprs.size(); i++) { ExprNodeDesc childExpr = newExprs.get(i); if (childExpr instanceof ExprNodeConstantDesc) { ExprNodeConstantDesc c = (ExprNodeConstantDesc) childExpr; if (Boolean.TRUE.equals(c.getValue())) { // if true, prune it positionsToRemove.set(i); } else { // if false, return false return childExpr; } } else if (childExpr instanceof ExprNodeGenericFuncDesc && ((ExprNodeGenericFuncDesc) childExpr).getGenericUDF() instanceof GenericUDFOPNotNull && childExpr.getChildren().get(0) instanceof ExprNodeColumnDesc) { notNullExprs.add(childExpr.getChildren().get(0)); notNullExprsPositions.add(i); } else if (childExpr instanceof ExprNodeGenericFuncDesc && ((ExprNodeGenericFuncDesc) childExpr).getGenericUDF() instanceof GenericUDFBaseCompare && !(((ExprNodeGenericFuncDesc) childExpr).getGenericUDF() instanceof GenericUDFOPNotEqual) && childExpr.getChildren().size() == 2) { // Try to fold (key <op> 86) and (key is not null) to (key <op> 86) // where <op> can be "=", ">=", "<=", ">", "<". // Note: (key <> 86) and (key is not null) cannot be folded ExprNodeColumnDesc colDesc = ExprNodeDescUtils.getColumnExpr(childExpr.getChildren().get(0)); if (null == colDesc) { colDesc = ExprNodeDescUtils.getColumnExpr(childExpr.getChildren().get(1)); } if (colDesc != null) { compareExprs.add(colDesc); } } } // Try to fold (key = 86) and (key is not null) to (key = 86) for (int i = 0; i < notNullExprs.size(); i++) { for (ExprNodeDesc other : compareExprs) { if (notNullExprs.get(i).isSame(other)) { positionsToRemove.set(notNullExprsPositions.get(i)); break; } } } // Remove unnecessary expressions int pos = 0; int removed = 0; while ((pos = positionsToRemove.nextSetBit(pos)) != -1) { newExprs.remove(pos - removed); pos++; removed++; } if (newExprs.size() == 0) { return new ExprNodeConstantDesc(TypeInfoFactory.booleanTypeInfo, Boolean.TRUE); } if (newExprs.size() == 1) { return newExprs.get(0); } } if (udf instanceof GenericUDFOPOr) { final BitSet positionsToRemove = new BitSet(); for (int i = 0; i < newExprs.size(); i++) { ExprNodeDesc childExpr = newExprs.get(i); if (childExpr instanceof ExprNodeConstantDesc) { ExprNodeConstantDesc c = (ExprNodeConstantDesc) childExpr; if (Boolean.FALSE.equals(c.getValue())) { // if false, prune it positionsToRemove.set(i); } else if (Boolean.TRUE.equals(c.getValue())) { // if true return true return childExpr; } } } int pos = 0; int removed = 0; while ((pos = positionsToRemove.nextSetBit(pos)) != -1) { newExprs.remove(pos - removed); pos++; removed++; } if (newExprs.size() == 0) { return new ExprNodeConstantDesc(TypeInfoFactory.booleanTypeInfo, Boolean.FALSE); } if (newExprs.size() == 1) { return newExprs.get(0); } } if (udf instanceof GenericUDFWhen) { if (!(newExprs.size() == 2 || newExprs.size() == 3)) { // In general, when can have unlimited # of branches, // we currently only handle either 1 or 2 branch. return null; } ExprNodeDesc thenExpr = newExprs.get(1); ExprNodeDesc elseExpr = newExprs.size() == 3 ? newExprs.get(2) : new ExprNodeConstantDesc(newExprs.get(1).getTypeInfo(), null); ExprNodeDesc whenExpr = newExprs.get(0); if (whenExpr instanceof ExprNodeConstantDesc) { Boolean whenVal = (Boolean) ((ExprNodeConstantDesc) whenExpr).getValue(); return (whenVal == null || Boolean.FALSE.equals(whenVal)) ? elseExpr : thenExpr; } if (thenExpr instanceof ExprNodeConstantDesc && elseExpr instanceof ExprNodeConstantDesc) { ExprNodeConstantDesc constThen = (ExprNodeConstantDesc) thenExpr; ExprNodeConstantDesc constElse = (ExprNodeConstantDesc) elseExpr; Object thenVal = constThen.getValue(); Object elseVal = constElse.getValue(); if (thenVal == null) { if (elseVal == null) { // both branches are null. return thenExpr; } else if (op instanceof FilterOperator) { // we can still fold, since here null is equivalent to false. return Boolean.TRUE.equals(elseVal) ? ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPNot(), newExprs.subList(0, 1)) : Boolean.FALSE.equals(elseVal) ? elseExpr : null; } else { // can't do much, expression is not in context of filter, so we can't treat null as equivalent to false here. return null; } } else if (elseVal == null && op instanceof FilterOperator) { return Boolean.TRUE.equals(thenVal) ? whenExpr : Boolean.FALSE.equals(thenVal) ? thenExpr : null; } else if (thenVal.equals(elseVal)) { return thenExpr; } else if (thenVal instanceof Boolean && elseVal instanceof Boolean) { List<ExprNodeDesc> children = new ArrayList<>(); children.add(whenExpr); children.add(new ExprNodeConstantDesc(false)); ExprNodeGenericFuncDesc func = ExprNodeGenericFuncDesc.newInstance(new GenericUDFNvl(), children); if (Boolean.TRUE.equals(thenVal)) { return func; } else { List<ExprNodeDesc> exprs = new ArrayList<>(); exprs.add(func); return ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPNot(), exprs); } } else { return null; } } } if (udf instanceof GenericUDFCase) { // HIVE-9644 Attempt to fold expression like : // where (case ss_sold_date when '1998-01-01' then 1=1 else null=1 end); // where ss_sold_date= '1998-01-01' ; if (!(newExprs.size() == 3 || newExprs.size() == 4)) { // In general case can have unlimited # of branches, // we currently only handle either 1 or 2 branch. return null; } ExprNodeDesc thenExpr = newExprs.get(2); ExprNodeDesc elseExpr = newExprs.size() == 4 ? newExprs.get(3) : new ExprNodeConstantDesc(newExprs.get(2).getTypeInfo(), null); if (thenExpr instanceof ExprNodeConstantDesc && elseExpr instanceof ExprNodeConstantDesc) { ExprNodeConstantDesc constThen = (ExprNodeConstantDesc) thenExpr; ExprNodeConstantDesc constElse = (ExprNodeConstantDesc) elseExpr; Object thenVal = constThen.getValue(); Object elseVal = constElse.getValue(); if (thenVal == null) { if (null == elseVal) { return thenExpr; } else if (op instanceof FilterOperator) { return Boolean.TRUE.equals(elseVal) ? ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPNotEqual(), newExprs.subList(0, 2)) : Boolean.FALSE.equals(elseVal) ? elseExpr : null; } else { return null; } } else if (null == elseVal && op instanceof FilterOperator) { return Boolean.TRUE.equals(thenVal) ? ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPEqual(), newExprs.subList(0, 2)) : Boolean.FALSE.equals(thenVal) ? thenExpr : null; } else if (thenVal.equals(elseVal)) { return thenExpr; } else if (thenVal instanceof Boolean && elseVal instanceof Boolean) { ExprNodeGenericFuncDesc equal = ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPEqual(), newExprs.subList(0, 2)); List<ExprNodeDesc> children = new ArrayList<>(); children.add(equal); children.add(new ExprNodeConstantDesc(false)); ExprNodeGenericFuncDesc func = ExprNodeGenericFuncDesc.newInstance(new GenericUDFNvl(), children); if (Boolean.TRUE.equals(thenVal)) { return func; } else { List<ExprNodeDesc> exprs = new ArrayList<>(); exprs.add(func); return ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPNot(), exprs); } } else { return null; } } } if (udf instanceof GenericUDFUnixTimeStamp) { if (newExprs.size() >= 1) { // unix_timestamp(args) -> to_unix_timestamp(args) return ExprNodeGenericFuncDesc.newInstance(new GenericUDFToUnixTimeStamp(), newExprs); } } return null; } /** * Evaluate column, replace the deterministic columns with constants if possible * * @param desc * @param ctx * @param op * @param colToConstants * @return */ private static ExprNodeDesc evaluateColumn(ExprNodeColumnDesc desc, ConstantPropagateProcCtx cppCtx, Operator<? extends Serializable> parent) { RowSchema rs = parent.getSchema(); ColumnInfo ci = rs.getColumnInfo(desc.getColumn()); if (ci == null) { if (LOG.isErrorEnabled()) { LOG.error("Reverse look up of column " + desc + " error!"); } ci = rs.getColumnInfo(desc.getTabAlias(), desc.getColumn()); } if (ci == null) { if (LOG.isErrorEnabled()) { LOG.error("Can't resolve " + desc.getTabAlias() + "." + desc.getColumn()); } return null; } ExprNodeDesc constant = null; // Additional work for union operator, see union27.q if (ci.getAlias() == null) { for (Entry<ColumnInfo, ExprNodeDesc> e : cppCtx.getOpToConstantExprs().get(parent).entrySet()) { if (e.getKey().getInternalName().equals(ci.getInternalName())) { constant = e.getValue(); break; } } } else { constant = cppCtx.getOpToConstantExprs().get(parent).get(ci); } if (constant != null) { if (constant instanceof ExprNodeConstantDesc && !constant.getTypeInfo().equals(desc.getTypeInfo())) { return typeCast(constant, desc.getTypeInfo()); } return constant; } else { return null; } } /** * Evaluate UDF * * @param udf UDF object * @param exprs * @param oldExprs * @return null if expression cannot be evaluated (not all parameters are constants). Or evaluated * ExprNodeConstantDesc if possible. * @throws HiveException */ private static ExprNodeDesc evaluateFunction(GenericUDF udf, List<ExprNodeDesc> exprs, List<ExprNodeDesc> oldExprs) { DeferredJavaObject[] arguments = new DeferredJavaObject[exprs.size()]; ObjectInspector[] argois = new ObjectInspector[exprs.size()]; for (int i = 0; i < exprs.size(); i++) { ExprNodeDesc desc = exprs.get(i); if (desc instanceof ExprNodeConstantDesc) { ExprNodeConstantDesc constant = (ExprNodeConstantDesc) exprs.get(i); if (!constant.getTypeInfo().equals(oldExprs.get(i).getTypeInfo())) { constant = typeCast(constant, oldExprs.get(i).getTypeInfo()); if (constant == null) { return null; } } if (constant.getTypeInfo().getCategory() != Category.PRIMITIVE) { // nested complex types cannot be folded cleanly return null; } Object value = constant.getValue(); PrimitiveTypeInfo pti = (PrimitiveTypeInfo) constant.getTypeInfo(); Object writableValue = null == value ? value : PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(pti) .getPrimitiveWritableObject(value); arguments[i] = new DeferredJavaObject(writableValue); argois[i] = ObjectInspectorUtils.getConstantObjectInspector(constant.getWritableObjectInspector(), writableValue); } else if (desc instanceof ExprNodeGenericFuncDesc) { ExprNodeDesc evaluatedFn = foldExpr((ExprNodeGenericFuncDesc) desc); if (null == evaluatedFn || !(evaluatedFn instanceof ExprNodeConstantDesc)) { return null; } ExprNodeConstantDesc constant = (ExprNodeConstantDesc) evaluatedFn; if (constant.getTypeInfo().getCategory() != Category.PRIMITIVE) { // nested complex types cannot be folded cleanly return null; } Object writableValue = PrimitiveObjectInspectorFactory .getPrimitiveJavaObjectInspector((PrimitiveTypeInfo) constant.getTypeInfo()) .getPrimitiveWritableObject(constant.getValue()); arguments[i] = new DeferredJavaObject(writableValue); argois[i] = ObjectInspectorUtils.getConstantObjectInspector(constant.getWritableObjectInspector(), writableValue); } else { return null; } } try { ObjectInspector oi = udf.initialize(argois); Object o = udf.evaluate(arguments); if (LOG.isDebugEnabled()) { LOG.debug(udf.getClass().getName() + "(" + exprs + ")=" + o); } if (o == null) { return new ExprNodeConstantDesc(TypeInfoUtils.getTypeInfoFromObjectInspector(oi), o); } Class<?> clz = o.getClass(); if (PrimitiveObjectInspectorUtils.isPrimitiveWritableClass(clz)) { PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; TypeInfo typeInfo = poi.getTypeInfo(); o = poi.getPrimitiveJavaObject(o); if (typeInfo.getTypeName().contains(serdeConstants.DECIMAL_TYPE_NAME) || typeInfo.getTypeName().contains(serdeConstants.VARCHAR_TYPE_NAME) || typeInfo.getTypeName().contains(serdeConstants.CHAR_TYPE_NAME)) { return new ExprNodeConstantDesc(typeInfo, o); } } else if (udf instanceof GenericUDFStruct && oi instanceof StandardConstantStructObjectInspector) { // do not fold named_struct, only struct() ConstantObjectInspector coi = (ConstantObjectInspector) oi; TypeInfo structType = TypeInfoUtils.getTypeInfoFromObjectInspector(coi); return new ExprNodeConstantDesc(structType, ObjectInspectorUtils.copyToStandardJavaObject(o, coi)); } else if (!PrimitiveObjectInspectorUtils.isPrimitiveJavaClass(clz)) { if (LOG.isErrorEnabled()) { LOG.error("Unable to evaluate " + udf + ". Return value unrecoginizable."); } return null; } else { // fall through } String constStr = null; if (arguments.length == 1 && FunctionRegistry.isOpCast(udf)) { // remember original string representation of constant. constStr = arguments[0].get().toString(); } return new ExprNodeConstantDesc(o).setFoldedFromVal(constStr); } catch (HiveException e) { LOG.error("Evaluation function " + udf.getClass() + " failed in Constant Propagation Optimizer."); throw new RuntimeException(e); } } /** * Change operator row schema, replace column with constant if it is. * * @param op * @param constants * @throws SemanticException */ private static void foldOperator(Operator<? extends Serializable> op, ConstantPropagateProcCtx cppCtx) throws SemanticException { RowSchema schema = op.getSchema(); Map<ColumnInfo, ExprNodeDesc> constants = cppCtx.getOpToConstantExprs().get(op); if (schema != null && schema.getSignature() != null) { for (ColumnInfo col : schema.getSignature()) { ExprNodeDesc constant = constants.get(col); if (constant != null) { if (LOG.isDebugEnabled()) { LOG.debug("Replacing column " + col + " with constant " + constant + " in " + op); } if (!col.getType().equals(constant.getTypeInfo())) { constant = typeCast(constant, col.getType()); } if (constant != null) { col.setObjectinspector(constant.getWritableObjectInspector()); } } } } Map<String, ExprNodeDesc> colExprMap = op.getColumnExprMap(); if (colExprMap != null) { for (Entry<ColumnInfo, ExprNodeDesc> e : constants.entrySet()) { String internalName = e.getKey().getInternalName(); if (colExprMap.containsKey(internalName)) { colExprMap.put(internalName, e.getValue()); } } } } /** * Node Processor for Constant Propagation on Filter Operators. The processor is to fold * conditional expressions and extract assignment expressions and propagate them. */ public static class ConstantPropagateFilterProc implements NodeProcessor { @Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx, Object... nodeOutputs) throws SemanticException { FilterOperator op = (FilterOperator) nd; ConstantPropagateProcCtx cppCtx = (ConstantPropagateProcCtx) ctx; Map<ColumnInfo, ExprNodeDesc> constants = cppCtx.getPropagatedConstants(op); cppCtx.getOpToConstantExprs().put(op, constants); ExprNodeDesc condn = op.getConf().getPredicate(); if (LOG.isDebugEnabled()) { LOG.debug("Old filter FIL[" + op.getIdentifier() + "] conditions:" + condn.getExprString()); } ExprNodeDesc newCondn = foldExpr(condn, constants, cppCtx, op, 0, true); if (newCondn instanceof ExprNodeConstantDesc) { ExprNodeConstantDesc c = (ExprNodeConstantDesc) newCondn; if (Boolean.TRUE.equals(c.getValue())) { cppCtx.addOpToDelete(op); if (LOG.isDebugEnabled()) { LOG.debug("Filter expression " + condn + " holds true. Will delete it."); } } else if (Boolean.FALSE.equals(c.getValue())) { if (LOG.isWarnEnabled()) { LOG.warn("Filter expression " + condn + " holds false!"); } } } if (newCondn instanceof ExprNodeConstantDesc && ((ExprNodeConstantDesc) newCondn).getValue() == null) { // where null is same as where false newCondn = new ExprNodeConstantDesc(Boolean.FALSE); } if (LOG.isDebugEnabled()) { LOG.debug("New filter FIL[" + op.getIdentifier() + "] conditions:" + newCondn.getExprString()); } // merge it with the downstream col list op.getConf().setPredicate(newCondn); foldOperator(op, cppCtx); return null; } } /** * Factory method to get the ConstantPropagateFilterProc class. * * @return ConstantPropagateFilterProc */ public static ConstantPropagateFilterProc getFilterProc() { return new ConstantPropagateFilterProc(); } /** * Node Processor for Constant Propagate for Group By Operators. */ public static class ConstantPropagateGroupByProc implements NodeProcessor { @Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx, Object... nodeOutputs) throws SemanticException { GroupByOperator op = (GroupByOperator) nd; ConstantPropagateProcCtx cppCtx = (ConstantPropagateProcCtx) ctx; Map<ColumnInfo, ExprNodeDesc> colToConstants = cppCtx.getPropagatedConstants(op); cppCtx.getOpToConstantExprs().put(op, colToConstants); RowSchema rs = op.getSchema(); if (op.getColumnExprMap() != null && rs != null) { for (ColumnInfo colInfo : rs.getSignature()) { if (!VirtualColumn.isVirtualColumnBasedOnAlias(colInfo)) { ExprNodeDesc expr = op.getColumnExprMap().get(colInfo.getInternalName()); if (expr instanceof ExprNodeConstantDesc) { colToConstants.put(colInfo, expr); } } } } if (colToConstants.isEmpty()) { return null; } GroupByDesc conf = op.getConf(); ArrayList<ExprNodeDesc> keys = conf.getKeys(); for (int i = 0; i < keys.size(); i++) { ExprNodeDesc key = keys.get(i); ExprNodeDesc newkey = foldExpr(key, colToConstants, cppCtx, op, 0, false); keys.set(i, newkey); } foldOperator(op, cppCtx); return null; } } /** * Factory method to get the ConstantPropagateGroupByProc class. * * @return ConstantPropagateGroupByProc */ public static ConstantPropagateGroupByProc getGroupByProc() { return new ConstantPropagateGroupByProc(); } /** * The Default Node Processor for Constant Propagation. */ public static class ConstantPropagateDefaultProc implements NodeProcessor { @Override @SuppressWarnings("unchecked") public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx, Object... nodeOutputs) throws SemanticException { ConstantPropagateProcCtx cppCtx = (ConstantPropagateProcCtx) ctx; Operator<? extends Serializable> op = (Operator<? extends Serializable>) nd; Map<ColumnInfo, ExprNodeDesc> constants = cppCtx.getPropagatedConstants(op); cppCtx.getOpToConstantExprs().put(op, constants); RowSchema rs = op.getSchema(); if (op.getColumnExprMap() != null && rs != null) { for (ColumnInfo colInfo : rs.getSignature()) { if (!VirtualColumn.isVirtualColumnBasedOnAlias(colInfo)) { ExprNodeDesc expr = op.getColumnExprMap().get(colInfo.getInternalName()); if (expr instanceof ExprNodeConstantDesc) { constants.put(colInfo, expr); } } } } if (constants.isEmpty()) { return null; } foldOperator(op, cppCtx); return null; } } /** * Factory method to get the ConstantPropagateDefaultProc class. * * @return ConstantPropagateDefaultProc */ public static ConstantPropagateDefaultProc getDefaultProc() { return new ConstantPropagateDefaultProc(); } /** * The Node Processor for Constant Propagation for Select Operators. */ public static class ConstantPropagateSelectProc implements NodeProcessor { @Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx, Object... nodeOutputs) throws SemanticException { SelectOperator op = (SelectOperator) nd; ConstantPropagateProcCtx cppCtx = (ConstantPropagateProcCtx) ctx; Map<ColumnInfo, ExprNodeDesc> constants = cppCtx.getPropagatedConstants(op); cppCtx.getOpToConstantExprs().put(op, constants); foldOperator(op, cppCtx); List<ExprNodeDesc> colList = op.getConf().getColList(); List<String> columnNames = op.getConf().getOutputColumnNames(); Map<String, ExprNodeDesc> columnExprMap = op.getColumnExprMap(); if (colList != null) { for (int i = 0; i < colList.size(); i++) { ExprNodeDesc newCol = foldExpr(colList.get(i), constants, cppCtx, op, 0, false); if (!(colList.get(i) instanceof ExprNodeConstantDesc) && newCol instanceof ExprNodeConstantDesc) { // Lets try to store original column name, if this column got folded // This is useful for optimizations like GroupByOptimizer String colName = colList.get(i).getExprString(); if (HiveConf.getPositionFromInternalName(colName) == -1) { // if its not an internal name, this is what we want. ((ExprNodeConstantDesc) newCol).setFoldedFromCol(colName); } else { // If it was internal column, lets try to get name from columnExprMap ExprNodeDesc desc = columnExprMap.get(colName); if (desc instanceof ExprNodeConstantDesc) { ((ExprNodeConstantDesc) newCol) .setFoldedFromCol(((ExprNodeConstantDesc) desc).getFoldedFromCol()); } } } colList.set(i, newCol); if (newCol instanceof ExprNodeConstantDesc && op.getSchema() != null) { ColumnInfo colInfo = op.getSchema().getSignature().get(i); if (!VirtualColumn.isVirtualColumnBasedOnAlias(colInfo)) { constants.put(colInfo, newCol); } } if (columnExprMap != null) { columnExprMap.put(columnNames.get(i), newCol); } } if (LOG.isDebugEnabled()) { LOG.debug("New column list:(" + StringUtils.join(colList, " ") + ")"); } } return null; } } /** * The Factory method to get the ConstantPropagateSelectProc class. * * @return ConstantPropagateSelectProc */ public static ConstantPropagateSelectProc getSelectProc() { return new ConstantPropagateSelectProc(); } /** * The Node Processor for constant propagation for FileSink Operators. In addition to constant * propagation, this processor also prunes dynamic partitions to static partitions if possible. */ public static class ConstantPropagateFileSinkProc implements NodeProcessor { @Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx, Object... nodeOutputs) throws SemanticException { FileSinkOperator op = (FileSinkOperator) nd; ConstantPropagateProcCtx cppCtx = (ConstantPropagateProcCtx) ctx; Map<ColumnInfo, ExprNodeDesc> constants = cppCtx.getPropagatedConstants(op); cppCtx.getOpToConstantExprs().put(op, constants); if (constants.isEmpty()) { return null; } FileSinkDesc fsdesc = op.getConf(); DynamicPartitionCtx dpCtx = fsdesc.getDynPartCtx(); if (dpCtx != null) { // Assume only 1 parent for FS operator Operator<? extends Serializable> parent = op.getParentOperators().get(0); Map<ColumnInfo, ExprNodeDesc> parentConstants = cppCtx.getPropagatedConstants(parent); RowSchema rs = parent.getSchema(); boolean allConstant = true; int dpColStartIdx = Utilities.getDPColOffset(fsdesc); List<ColumnInfo> colInfos = rs.getSignature(); for (int i = dpColStartIdx; i < colInfos.size(); i++) { ColumnInfo ci = colInfos.get(i); if (parentConstants.get(ci) == null) { allConstant = false; break; } } if (allConstant) { pruneDP(fsdesc); } } foldOperator(op, cppCtx); return null; } private void pruneDP(FileSinkDesc fsdesc) { // FIXME: Support pruning dynamic partitioning. LOG.info("DP can be rewritten to SP!"); } } public static NodeProcessor getFileSinkProc() { return new ConstantPropagateFileSinkProc(); } /** * The Node Processor for Constant Propagation for Operators which is designed to stop propagate. * Currently these kinds of Operators include UnionOperator and ScriptOperator. */ public static class ConstantPropagateStopProc implements NodeProcessor { @Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx, Object... nodeOutputs) throws SemanticException { Operator<?> op = (Operator<?>) nd; ConstantPropagateProcCtx cppCtx = (ConstantPropagateProcCtx) ctx; cppCtx.getOpToConstantExprs().put(op, new HashMap<ColumnInfo, ExprNodeDesc>()); if (LOG.isDebugEnabled()) { LOG.debug("Stop propagate constants on op " + op.getOperatorId()); } return null; } } public static NodeProcessor getStopProc() { return new ConstantPropagateStopProc(); } /** * The Node Processor for Constant Propagation for ReduceSink Operators. If the RS Operator is for * a join, then only those constants from inner join tables, or from the 'inner side' of a outer * join (left table for left outer join and vice versa) can be propagated. */ public static class ConstantPropagateReduceSinkProc implements NodeProcessor { @Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx, Object... nodeOutputs) throws SemanticException { ReduceSinkOperator op = (ReduceSinkOperator) nd; ReduceSinkDesc rsDesc = op.getConf(); ConstantPropagateProcCtx cppCtx = (ConstantPropagateProcCtx) ctx; Map<ColumnInfo, ExprNodeDesc> constants = cppCtx.getPropagatedConstants(op); cppCtx.getOpToConstantExprs().put(op, constants); RowSchema rs = op.getSchema(); if (op.getColumnExprMap() != null && rs != null) { for (ColumnInfo colInfo : rs.getSignature()) { if (!VirtualColumn.isVirtualColumnBasedOnAlias(colInfo)) { ExprNodeDesc expr = op.getColumnExprMap().get(colInfo.getInternalName()); if (expr instanceof ExprNodeConstantDesc) { constants.put(colInfo, expr); } } } } if (constants.isEmpty()) { return null; } if (op.getChildOperators().size() == 1 && op.getChildOperators().get(0) instanceof JoinOperator) { JoinOperator joinOp = (JoinOperator) op.getChildOperators().get(0); if (skipFolding(joinOp.getConf())) { if (LOG.isDebugEnabled()) { LOG.debug("Skip folding in outer join " + op); } cppCtx.getOpToConstantExprs().put(op, new HashMap<ColumnInfo, ExprNodeDesc>()); return null; } } if (rsDesc.getDistinctColumnIndices() != null && !rsDesc.getDistinctColumnIndices().isEmpty()) { if (LOG.isDebugEnabled()) { LOG.debug("Skip folding in distinct subqueries " + op); } cppCtx.getOpToConstantExprs().put(op, new HashMap<ColumnInfo, ExprNodeDesc>()); return null; } // key columns ArrayList<ExprNodeDesc> newKeyEpxrs = new ArrayList<ExprNodeDesc>(); for (ExprNodeDesc desc : rsDesc.getKeyCols()) { ExprNodeDesc newDesc = foldExpr(desc, constants, cppCtx, op, 0, false); if (newDesc != desc && desc instanceof ExprNodeColumnDesc && newDesc instanceof ExprNodeConstantDesc) { ((ExprNodeConstantDesc) newDesc).setFoldedFromCol(((ExprNodeColumnDesc) desc).getColumn()); } newKeyEpxrs.add(newDesc); } rsDesc.setKeyCols(newKeyEpxrs); // partition columns ArrayList<ExprNodeDesc> newPartExprs = new ArrayList<ExprNodeDesc>(); for (ExprNodeDesc desc : rsDesc.getPartitionCols()) { ExprNodeDesc expr = foldExpr(desc, constants, cppCtx, op, 0, false); if (expr != desc && desc instanceof ExprNodeColumnDesc && expr instanceof ExprNodeConstantDesc) { ((ExprNodeConstantDesc) expr).setFoldedFromCol(((ExprNodeColumnDesc) desc).getColumn()); } newPartExprs.add(expr); } rsDesc.setPartitionCols(newPartExprs); // value columns ArrayList<ExprNodeDesc> newValExprs = new ArrayList<ExprNodeDesc>(); for (ExprNodeDesc desc : rsDesc.getValueCols()) { newValExprs.add(foldExpr(desc, constants, cppCtx, op, 0, false)); } rsDesc.setValueCols(newValExprs); foldOperator(op, cppCtx); return null; } /** * Skip folding constants if there is outer join in join tree. * @param joinDesc * @return true if to skip. */ private boolean skipFolding(JoinDesc joinDesc) { for (JoinCondDesc cond : joinDesc.getConds()) { if (cond.getType() == JoinDesc.INNER_JOIN || cond.getType() == JoinDesc.UNIQUE_JOIN || cond.getType() == JoinDesc.LEFT_SEMI_JOIN) { continue; } return true; } return false; } } public static NodeProcessor getReduceSinkProc() { return new ConstantPropagateReduceSinkProc(); } /** * The Node Processor for Constant Propagation for Join Operators. */ public static class ConstantPropagateJoinProc implements NodeProcessor { @Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx, Object... nodeOutputs) throws SemanticException { JoinOperator op = (JoinOperator) nd; JoinDesc conf = op.getConf(); ConstantPropagateProcCtx cppCtx = (ConstantPropagateProcCtx) ctx; Map<ColumnInfo, ExprNodeDesc> constants = cppCtx.getPropagatedConstants(op); cppCtx.getOpToConstantExprs().put(op, constants); if (constants.isEmpty()) { return null; } // Note: the following code (removing folded constants in exprs) is deeply coupled with // ColumnPruner optimizer. // Assuming ColumnPrunner will remove constant columns so we don't deal with output columns. // Except one case that the join operator is followed by a redistribution (RS operator). if (op.getChildOperators().size() == 1 && op.getChildOperators().get(0) instanceof ReduceSinkOperator) { LOG.debug("Skip JOIN-RS structure."); return null; } if (LOG.isInfoEnabled()) { LOG.info("Old exprs " + conf.getExprs()); } Iterator<Entry<Byte, List<ExprNodeDesc>>> itr = conf.getExprs().entrySet().iterator(); while (itr.hasNext()) { Entry<Byte, List<ExprNodeDesc>> e = itr.next(); int tag = e.getKey(); List<ExprNodeDesc> exprs = e.getValue(); if (exprs == null) { continue; } List<ExprNodeDesc> newExprs = new ArrayList<ExprNodeDesc>(); for (ExprNodeDesc expr : exprs) { ExprNodeDesc newExpr = foldExpr(expr, constants, cppCtx, op, tag, false); if (newExpr instanceof ExprNodeConstantDesc) { if (LOG.isInfoEnabled()) { LOG.info("expr " + newExpr + " fold from " + expr + " is removed."); } continue; } newExprs.add(newExpr); } e.setValue(newExprs); } if (LOG.isInfoEnabled()) { LOG.info("New exprs " + conf.getExprs()); } for (List<ExprNodeDesc> v : conf.getFilters().values()) { for (int i = 0; i < v.size(); i++) { ExprNodeDesc expr = foldExpr(v.get(i), constants, cppCtx, op, 0, false); v.set(i, expr); } } foldOperator(op, cppCtx); return null; } } public static NodeProcessor getJoinProc() { return new ConstantPropagateJoinProc(); } /** * The Node Processor for Constant Propagation for Table Scan Operators. */ public static class ConstantPropagateTableScanProc implements NodeProcessor { @Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx, Object... nodeOutputs) throws SemanticException { TableScanOperator op = (TableScanOperator) nd; TableScanDesc conf = op.getConf(); ConstantPropagateProcCtx cppCtx = (ConstantPropagateProcCtx) ctx; Map<ColumnInfo, ExprNodeDesc> constants = cppCtx.getPropagatedConstants(op); cppCtx.getOpToConstantExprs().put(op, constants); ExprNodeGenericFuncDesc pred = conf.getFilterExpr(); if (pred == null) { return null; } ExprNodeDesc constant = foldExpr(pred, constants, cppCtx, op, 0, false); if (constant instanceof ExprNodeGenericFuncDesc) { conf.setFilterExpr((ExprNodeGenericFuncDesc) constant); } else { conf.setFilterExpr(null); } return null; } } public static NodeProcessor getTableScanProc() { return new ConstantPropagateTableScanProc(); } }