Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.hive.ql.exec; import java.io.Serializable; import java.lang.management.ManagementFactory; import java.lang.management.MemoryMXBean; import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.OpParseContext; import org.apache.hadoop.hive.ql.plan.AggregationDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.GroupByDesc; import org.apache.hadoop.hive.ql.plan.api.OperatorType; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.AggregationBuffer; import org.apache.hadoop.hive.serde2.lazy.LazyPrimitive; import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyStringObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.UnionObject; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.apache.hadoop.io.Text; /** * GroupBy operator implementation. */ public class GroupByOperator extends Operator<GroupByDesc> implements Serializable { private static final Log LOG = LogFactory.getLog(GroupByOperator.class.getName()); private static final long serialVersionUID = 1L; private static final int NUMROWSESTIMATESIZE = 1000; protected transient ExprNodeEvaluator[] keyFields; protected transient ObjectInspector[] keyObjectInspectors; protected transient ExprNodeEvaluator[][] aggregationParameterFields; protected transient ObjectInspector[][] aggregationParameterObjectInspectors; protected transient ObjectInspector[][] aggregationParameterStandardObjectInspectors; protected transient Object[][] aggregationParameterObjects; // In the future, we may allow both count(DISTINCT a) and sum(DISTINCT a) in // the same SQL clause, // so aggregationIsDistinct is a boolean array instead of a single number. protected transient boolean[] aggregationIsDistinct; // Map from integer tag to distinct aggrs transient protected Map<Integer, Set<Integer>> distinctKeyAggrs = new HashMap<Integer, Set<Integer>>(); // Map from integer tag to non-distinct aggrs with key parameters. transient protected Map<Integer, Set<Integer>> nonDistinctKeyAggrs = new HashMap<Integer, Set<Integer>>(); // List of non-distinct aggrs. transient protected List<Integer> nonDistinctAggrs = new ArrayList<Integer>(); // Union expr for distinct keys transient ExprNodeEvaluator unionExprEval = null; transient GenericUDAFEvaluator[] aggregationEvaluators; protected transient ArrayList<ObjectInspector> objectInspectors; transient ArrayList<String> fieldNames; transient KeyWrapperFactory keyWrapperFactory; // Used by sort-based GroupBy: Mode = COMPLETE, PARTIAL1, PARTIAL2, // MERGEPARTIAL protected transient KeyWrapper currentKeys; protected transient KeyWrapper newKeys; protected transient AggregationBuffer[] aggregations; protected transient Object[][] aggregationsParametersLastInvoke; // Used by hash-based GroupBy: Mode = HASH, PARTIALS protected transient HashMap<KeyWrapper, AggregationBuffer[]> hashAggregations; // Used by hash distinct aggregations when hashGrpKeyNotRedKey is true protected transient HashSet<KeyWrapper> keysCurrentGroup; transient boolean bucketGroup; transient boolean firstRow; transient long totalMemory; transient boolean hashAggr; // The reduction is happening on the reducer, and the grouping key and // reduction keys are different. // For example: select a, count(distinct b) from T group by a // The data is sprayed by 'b' and the reducer is grouping it by 'a' transient boolean groupKeyIsNotReduceKey; transient boolean firstRowInGroup; transient long numRowsInput; transient long numRowsHashTbl; transient int groupbyMapAggrInterval; transient long numRowsCompareHashAggr; transient float minReductionHashAggr; // current Key ObjectInspectors are standard ObjectInspectors protected transient ObjectInspector[] currentKeyObjectInspectors; // new Key ObjectInspectors are objectInspectors from the parent transient StructObjectInspector newKeyObjectInspector; transient StructObjectInspector currentKeyObjectInspector; public static MemoryMXBean memoryMXBean; private long maxMemory; private float memoryThreshold; /** * This is used to store the position and field names for variable length * fields. **/ class varLenFields { int aggrPos; List<Field> fields; varLenFields(int aggrPos, List<Field> fields) { this.aggrPos = aggrPos; this.fields = fields; } int getAggrPos() { return aggrPos; } List<Field> getFields() { return fields; } }; // for these positions, some variable primitive type (String) is used, so size // cannot be estimated. sample it at runtime. transient List<Integer> keyPositionsSize; // for these positions, some variable primitive type (String) is used for the // aggregation classes transient List<varLenFields> aggrPositions; transient int fixedRowSize; transient long maxHashTblMemory; transient int totalVariableSize; transient int numEntriesVarSize; transient int numEntriesHashTable; transient int countAfterReport; transient int heartbeatInterval; @Override protected void initializeOp(Configuration hconf) throws HiveException { totalMemory = Runtime.getRuntime().totalMemory(); numRowsInput = 0; numRowsHashTbl = 0; heartbeatInterval = HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVESENDHEARTBEAT); countAfterReport = 0; ObjectInspector rowInspector = inputObjInspectors[0]; // init keyFields keyFields = new ExprNodeEvaluator[conf.getKeys().size()]; keyObjectInspectors = new ObjectInspector[conf.getKeys().size()]; currentKeyObjectInspectors = new ObjectInspector[conf.getKeys().size()]; for (int i = 0; i < keyFields.length; i++) { keyFields[i] = ExprNodeEvaluatorFactory.get(conf.getKeys().get(i)); keyObjectInspectors[i] = keyFields[i].initialize(rowInspector); currentKeyObjectInspectors[i] = ObjectInspectorUtils.getStandardObjectInspector(keyObjectInspectors[i], ObjectInspectorCopyOption.WRITABLE); } // initialize unionExpr for reduce-side // reduce KEY has union field as the last field if there are distinct // aggregates in group-by. List<? extends StructField> sfs = ((StandardStructObjectInspector) rowInspector).getAllStructFieldRefs(); if (sfs.size() > 0) { StructField keyField = sfs.get(0); if (keyField.getFieldName().toUpperCase().equals(Utilities.ReduceField.KEY.name())) { ObjectInspector keyObjInspector = keyField.getFieldObjectInspector(); if (keyObjInspector instanceof StandardStructObjectInspector) { List<? extends StructField> keysfs = ((StandardStructObjectInspector) keyObjInspector) .getAllStructFieldRefs(); if (keysfs.size() > 0) { // the last field is the union field, if any StructField sf = keysfs.get(keysfs.size() - 1); if (sf.getFieldObjectInspector().getCategory().equals(ObjectInspector.Category.UNION)) { unionExprEval = ExprNodeEvaluatorFactory.get(new ExprNodeColumnDesc( TypeInfoUtils.getTypeInfoFromObjectInspector(sf.getFieldObjectInspector()), keyField.getFieldName() + "." + sf.getFieldName(), null, false)); unionExprEval.initialize(rowInspector); } } } } } // init aggregationParameterFields ArrayList<AggregationDesc> aggrs = conf.getAggregators(); aggregationParameterFields = new ExprNodeEvaluator[aggrs.size()][]; aggregationParameterObjectInspectors = new ObjectInspector[aggrs.size()][]; aggregationParameterStandardObjectInspectors = new ObjectInspector[aggrs.size()][]; aggregationParameterObjects = new Object[aggrs.size()][]; aggregationIsDistinct = new boolean[aggrs.size()]; for (int i = 0; i < aggrs.size(); i++) { AggregationDesc aggr = aggrs.get(i); ArrayList<ExprNodeDesc> parameters = aggr.getParameters(); aggregationParameterFields[i] = new ExprNodeEvaluator[parameters.size()]; aggregationParameterObjectInspectors[i] = new ObjectInspector[parameters.size()]; aggregationParameterStandardObjectInspectors[i] = new ObjectInspector[parameters.size()]; aggregationParameterObjects[i] = new Object[parameters.size()]; for (int j = 0; j < parameters.size(); j++) { aggregationParameterFields[i][j] = ExprNodeEvaluatorFactory.get(parameters.get(j)); aggregationParameterObjectInspectors[i][j] = aggregationParameterFields[i][j] .initialize(rowInspector); if (unionExprEval != null) { String[] names = parameters.get(j).getExprString().split("\\."); // parameters of the form : KEY.colx:t.coly if (Utilities.ReduceField.KEY.name().equals(names[0])) { String name = names[names.length - 2]; int tag = Integer.parseInt(name.split("\\:")[1]); if (aggr.getDistinct()) { // is distinct Set<Integer> set = distinctKeyAggrs.get(tag); if (null == set) { set = new HashSet<Integer>(); distinctKeyAggrs.put(tag, set); } if (!set.contains(i)) { set.add(i); } } else { Set<Integer> set = nonDistinctKeyAggrs.get(tag); if (null == set) { set = new HashSet<Integer>(); nonDistinctKeyAggrs.put(tag, set); } if (!set.contains(i)) { set.add(i); } } } else { // will be VALUE._COLx if (!nonDistinctAggrs.contains(i)) { nonDistinctAggrs.add(i); } } } else { if (aggr.getDistinct()) { aggregationIsDistinct[i] = true; } } aggregationParameterStandardObjectInspectors[i][j] = ObjectInspectorUtils .getStandardObjectInspector(aggregationParameterObjectInspectors[i][j], ObjectInspectorCopyOption.WRITABLE); aggregationParameterObjects[i][j] = null; } if (parameters.size() == 0) { // for ex: count(*) if (!nonDistinctAggrs.contains(i)) { nonDistinctAggrs.add(i); } } } // init aggregationClasses aggregationEvaluators = new GenericUDAFEvaluator[conf.getAggregators().size()]; for (int i = 0; i < aggregationEvaluators.length; i++) { AggregationDesc agg = conf.getAggregators().get(i); aggregationEvaluators[i] = agg.getGenericUDAFEvaluator(); } // init objectInspectors int totalFields = keyFields.length + aggregationEvaluators.length; objectInspectors = new ArrayList<ObjectInspector>(totalFields); for (ExprNodeEvaluator keyField : keyFields) { objectInspectors.add(null); } for (int i = 0; i < aggregationEvaluators.length; i++) { ObjectInspector roi = aggregationEvaluators[i].init(conf.getAggregators().get(i).getMode(), aggregationParameterObjectInspectors[i]); objectInspectors.add(roi); } bucketGroup = conf.getBucketGroup(); aggregationsParametersLastInvoke = new Object[conf.getAggregators().size()][]; if (conf.getMode() != GroupByDesc.Mode.HASH || bucketGroup) { aggregations = newAggregations(); hashAggr = false; } else { hashAggregations = new HashMap<KeyWrapper, AggregationBuffer[]>(256); aggregations = newAggregations(); hashAggr = true; keyPositionsSize = new ArrayList<Integer>(); aggrPositions = new ArrayList<varLenFields>(); groupbyMapAggrInterval = HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEGROUPBYMAPINTERVAL); // compare every groupbyMapAggrInterval rows numRowsCompareHashAggr = groupbyMapAggrInterval; minReductionHashAggr = HiveConf.getFloatVar(hconf, HiveConf.ConfVars.HIVEMAPAGGRHASHMINREDUCTION); groupKeyIsNotReduceKey = conf.getGroupKeyNotReductionKey(); if (groupKeyIsNotReduceKey) { keysCurrentGroup = new HashSet<KeyWrapper>(); } } fieldNames = conf.getOutputColumnNames(); for (int i = 0; i < keyFields.length; i++) { objectInspectors.set(i, currentKeyObjectInspectors[i]); } // Generate key names ArrayList<String> keyNames = new ArrayList<String>(keyFields.length); for (int i = 0; i < keyFields.length; i++) { keyNames.add(fieldNames.get(i)); } newKeyObjectInspector = ObjectInspectorFactory.getStandardStructObjectInspector(keyNames, Arrays.asList(keyObjectInspectors)); currentKeyObjectInspector = ObjectInspectorFactory.getStandardStructObjectInspector(keyNames, Arrays.asList(currentKeyObjectInspectors)); outputObjInspector = ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, objectInspectors); keyWrapperFactory = new KeyWrapperFactory(keyFields, keyObjectInspectors, currentKeyObjectInspectors); newKeys = keyWrapperFactory.getKeyWrapper(); firstRow = true; // estimate the number of hash table entries based on the size of each // entry. Since the size of a entry // is not known, estimate that based on the number of entries if (hashAggr) { computeMaxEntriesHashAggr(hconf); } memoryMXBean = ManagementFactory.getMemoryMXBean(); maxMemory = memoryMXBean.getHeapMemoryUsage().getMax(); memoryThreshold = this.getConf().getMemoryThreshold(); initializeChildren(hconf); } /** * Estimate the number of entries in map-side hash table. The user can specify * the total amount of memory to be used by the map-side hash. By default, all * available memory is used. The size of each row is estimated, rather * crudely, and the number of entries are figure out based on that. * * @return number of entries that can fit in hash table - useful for map-side * aggregation only **/ private void computeMaxEntriesHashAggr(Configuration hconf) throws HiveException { float memoryPercentage = this.getConf().getGroupByMemoryUsage(); maxHashTblMemory = (long) (memoryPercentage * Runtime.getRuntime().maxMemory()); estimateRowSize(); } private static final int javaObjectOverHead = 64; private static final int javaHashEntryOverHead = 64; private static final int javaSizePrimitiveType = 16; private static final int javaSizeUnknownType = 256; /** * The size of the element at position 'pos' is returned, if possible. If the * datatype is of variable length, STRING, a list of such key positions is * maintained, and the size for such positions is then actually calculated at * runtime. * * @param pos * the position of the key * @param c * the type of the key * @return the size of this datatype **/ private int getSize(int pos, PrimitiveCategory category) { switch (category) { case VOID: case BOOLEAN: case BYTE: case SHORT: case INT: case LONG: case FLOAT: case DOUBLE: return javaSizePrimitiveType; case STRING: keyPositionsSize.add(new Integer(pos)); return javaObjectOverHead; default: return javaSizeUnknownType; } } /** * The size of the element at position 'pos' is returned, if possible. If the * field is of variable length, STRING, a list of such field names for the * field position is maintained, and the size for such positions is then * actually calculated at runtime. * * @param pos * the position of the key * @param c * the type of the key * @param f * the field to be added * @return the size of this datatype **/ private int getSize(int pos, Class<?> c, Field f) { if (c.isPrimitive() || c.isInstance(Boolean.valueOf(true)) || c.isInstance(Byte.valueOf((byte) 0)) || c.isInstance(Short.valueOf((short) 0)) || c.isInstance(Integer.valueOf(0)) || c.isInstance(Long.valueOf(0)) || c.isInstance(new Float(0)) || c.isInstance(new Double(0))) { return javaSizePrimitiveType; } if (c.isInstance(new String())) { int idx = 0; varLenFields v = null; for (idx = 0; idx < aggrPositions.size(); idx++) { v = aggrPositions.get(idx); if (v.getAggrPos() == pos) { break; } } if (idx == aggrPositions.size()) { v = new varLenFields(pos, new ArrayList<Field>()); aggrPositions.add(v); } v.getFields().add(f); return javaObjectOverHead; } return javaSizeUnknownType; } /** * @param pos * position of the key * @param typeinfo * type of the input * @return the size of this datatype **/ private int getSize(int pos, TypeInfo typeInfo) { if (typeInfo instanceof PrimitiveTypeInfo) { return getSize(pos, ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory()); } return javaSizeUnknownType; } /** * @return the size of each row **/ private void estimateRowSize() throws HiveException { // estimate the size of each entry - // a datatype with unknown size (String/Struct etc. - is assumed to be 256 // bytes for now). // 64 bytes is the overhead for a reference fixedRowSize = javaHashEntryOverHead; ArrayList<ExprNodeDesc> keys = conf.getKeys(); // Go over all the keys and get the size of the fields of fixed length. Keep // track of the variable length keys for (int pos = 0; pos < keys.size(); pos++) { fixedRowSize += getSize(pos, keys.get(pos).getTypeInfo()); } // Go over all the aggregation classes and and get the size of the fields of // fixed length. Keep track of the variable length // fields in these aggregation classes. for (int i = 0; i < aggregationEvaluators.length; i++) { fixedRowSize += javaObjectOverHead; Class<? extends AggregationBuffer> agg = aggregationEvaluators[i].getNewAggregationBuffer().getClass(); Field[] fArr = ObjectInspectorUtils.getDeclaredNonStaticFields(agg); for (Field f : fArr) { fixedRowSize += getSize(i, f.getType(), f); } } } protected AggregationBuffer[] newAggregations() throws HiveException { AggregationBuffer[] aggs = new AggregationBuffer[aggregationEvaluators.length]; for (int i = 0; i < aggregationEvaluators.length; i++) { aggs[i] = aggregationEvaluators[i].getNewAggregationBuffer(); // aggregationClasses[i].reset(aggs[i]); } return aggs; } protected void resetAggregations(AggregationBuffer[] aggs) throws HiveException { for (int i = 0; i < aggs.length; i++) { aggregationEvaluators[i].reset(aggs[i]); } } /* * Update aggregations. If the aggregation is for distinct, in case of hash * aggregation, the client tells us whether it is a new entry. For sort-based * aggregations, the last row is compared with the current one to figure out * whether it has changed. As a cleanup, the lastInvoke logic can be pushed in * the caller, and this function can be independent of that. The client should * always notify whether it is a different row or not. * * @param aggs the aggregations to be evaluated * * @param row the row being processed * * @param rowInspector the inspector for the row * * @param hashAggr whether hash aggregation is being performed or not * * @param newEntryForHashAggr only valid if it is a hash aggregation, whether * it is a new entry or not */ protected void updateAggregations(AggregationBuffer[] aggs, Object row, ObjectInspector rowInspector, boolean hashAggr, boolean newEntryForHashAggr, Object[][] lastInvoke) throws HiveException { if (unionExprEval == null) { for (int ai = 0; ai < aggs.length; ai++) { // Calculate the parameters Object[] o = new Object[aggregationParameterFields[ai].length]; for (int pi = 0; pi < aggregationParameterFields[ai].length; pi++) { o[pi] = aggregationParameterFields[ai][pi].evaluate(row); } // Update the aggregations. if (aggregationIsDistinct[ai]) { if (hashAggr) { if (newEntryForHashAggr) { aggregationEvaluators[ai].aggregate(aggs[ai], o); } } else { if (lastInvoke[ai] == null) { lastInvoke[ai] = new Object[o.length]; } if (ObjectInspectorUtils.compare(o, aggregationParameterObjectInspectors[ai], lastInvoke[ai], aggregationParameterStandardObjectInspectors[ai]) != 0) { aggregationEvaluators[ai].aggregate(aggs[ai], o); for (int pi = 0; pi < o.length; pi++) { lastInvoke[ai][pi] = ObjectInspectorUtils.copyToStandardObject(o[pi], aggregationParameterObjectInspectors[ai][pi], ObjectInspectorCopyOption.WRITABLE); } } } } else { aggregationEvaluators[ai].aggregate(aggs[ai], o); } } return; } if (distinctKeyAggrs.size() > 0) { // evaluate union object UnionObject uo = (UnionObject) (unionExprEval.evaluate(row)); int unionTag = uo.getTag(); // update non-distinct key aggregations : "KEY._colx:t._coly" if (nonDistinctKeyAggrs.get(unionTag) != null) { for (int pos : nonDistinctKeyAggrs.get(unionTag)) { Object[] o = new Object[aggregationParameterFields[pos].length]; for (int pi = 0; pi < aggregationParameterFields[pos].length; pi++) { o[pi] = aggregationParameterFields[pos][pi].evaluate(row); } aggregationEvaluators[pos].aggregate(aggs[pos], o); } } // there may be multi distinct clauses for one column // update them all. if (distinctKeyAggrs.get(unionTag) != null) { for (int i : distinctKeyAggrs.get(unionTag)) { Object[] o = new Object[aggregationParameterFields[i].length]; for (int pi = 0; pi < aggregationParameterFields[i].length; pi++) { o[pi] = aggregationParameterFields[i][pi].evaluate(row); } if (hashAggr) { if (newEntryForHashAggr) { aggregationEvaluators[i].aggregate(aggs[i], o); } } else { if (lastInvoke[i] == null) { lastInvoke[i] = new Object[o.length]; } if (ObjectInspectorUtils.compare(o, aggregationParameterObjectInspectors[i], lastInvoke[i], aggregationParameterStandardObjectInspectors[i]) != 0) { aggregationEvaluators[i].aggregate(aggs[i], o); for (int pi = 0; pi < o.length; pi++) { lastInvoke[i][pi] = ObjectInspectorUtils.copyToStandardObject(o[pi], aggregationParameterObjectInspectors[i][pi], ObjectInspectorCopyOption.WRITABLE); } } } } } // update non-distinct value aggregations: 'VALUE._colx' // these aggregations should be updated only once. if (unionTag == 0) { for (int pos : nonDistinctAggrs) { Object[] o = new Object[aggregationParameterFields[pos].length]; for (int pi = 0; pi < aggregationParameterFields[pos].length; pi++) { o[pi] = aggregationParameterFields[pos][pi].evaluate(row); } aggregationEvaluators[pos].aggregate(aggs[pos], o); } } } else { for (int ai = 0; ai < aggs.length; ai++) { // there is no distinct aggregation, // update all aggregations Object[] o = new Object[aggregationParameterFields[ai].length]; for (int pi = 0; pi < aggregationParameterFields[ai].length; pi++) { o[pi] = aggregationParameterFields[ai][pi].evaluate(row); } aggregationEvaluators[ai].aggregate(aggs[ai], o); } } } @Override public void startGroup() throws HiveException { firstRowInGroup = true; } @Override public void endGroup() throws HiveException { if (groupKeyIsNotReduceKey) { keysCurrentGroup.clear(); } } @Override public void processOp(Object row, int tag) throws HiveException { firstRow = false; ObjectInspector rowInspector = inputObjInspectors[tag]; // Total number of input rows is needed for hash aggregation only if (hashAggr && !groupKeyIsNotReduceKey) { numRowsInput++; // if hash aggregation is not behaving properly, disable it if (numRowsInput == numRowsCompareHashAggr) { numRowsCompareHashAggr += groupbyMapAggrInterval; // map-side aggregation should reduce the entries by at-least half if (numRowsHashTbl > numRowsInput * minReductionHashAggr) { LOG.warn("Disable Hash Aggr: #hash table = " + numRowsHashTbl + " #total = " + numRowsInput + " reduction = " + 1.0 * (numRowsHashTbl / numRowsInput) + " minReduction = " + minReductionHashAggr); flush(true); hashAggr = false; } else { LOG.trace("Hash Aggr Enabled: #hash table = " + numRowsHashTbl + " #total = " + numRowsInput + " reduction = " + 1.0 * (numRowsHashTbl / numRowsInput) + " minReduction = " + minReductionHashAggr); } } } try { countAfterReport++; newKeys.getNewKey(row, rowInspector); if (hashAggr) { newKeys.setHashKey(); processHashAggr(row, rowInspector, newKeys); } else { processAggr(row, rowInspector, newKeys); } firstRowInGroup = false; if (countAfterReport != 0 && (countAfterReport % heartbeatInterval) == 0 && (reporter != null)) { reporter.progress(); countAfterReport = 0; } } catch (HiveException e) { throw e; } catch (Exception e) { throw new HiveException(e); } } private void processHashAggr(Object row, ObjectInspector rowInspector, KeyWrapper newKeys) throws HiveException { // Prepare aggs for updating AggregationBuffer[] aggs = null; boolean newEntryForHashAggr = false; // hash-based aggregations aggs = hashAggregations.get(newKeys); if (aggs == null) { KeyWrapper newKeyProber = newKeys.copyKey(); aggs = newAggregations(); hashAggregations.put(newKeyProber, aggs); newEntryForHashAggr = true; numRowsHashTbl++; // new entry in the hash table } // If the grouping key and the reduction key are different, a set of // grouping keys for the current reduction key are maintained in // keysCurrentGroup // Peek into the set to find out if a new grouping key is seen for the given // reduction key if (groupKeyIsNotReduceKey) { newEntryForHashAggr = keysCurrentGroup.add(newKeys.copyKey()); } // Update the aggs updateAggregations(aggs, row, rowInspector, true, newEntryForHashAggr, null); // We can only flush after the updateAggregations is done, or the // potentially new entry "aggs" // can be flushed out of the hash table. // Based on user-specified parameters, check if the hash table needs to be // flushed. // If the grouping key is not the same as reduction key, flushing can only // happen at boundaries if ((!groupKeyIsNotReduceKey || firstRowInGroup) && shouldBeFlushed(newKeys)) { flush(false); } } // Non-hash aggregation private void processAggr(Object row, ObjectInspector rowInspector, KeyWrapper newKeys) throws HiveException { // Prepare aggs for updating AggregationBuffer[] aggs = null; Object[][] lastInvoke = null; //boolean keysAreEqual = (currentKeys != null && newKeys != null)? // newKeyStructEqualComparer.areEqual(currentKeys, newKeys) : false; boolean keysAreEqual = (currentKeys != null && newKeys != null) ? newKeys.equals(currentKeys) : false; // Forward the current keys if needed for sort-based aggregation if (currentKeys != null && !keysAreEqual) { forward(currentKeys.getKeyArray(), aggregations); countAfterReport = 0; } // Need to update the keys? if (currentKeys == null || !keysAreEqual) { if (currentKeys == null) { currentKeys = newKeys.copyKey(); } else { currentKeys.copyKey(newKeys); } // Reset the aggregations resetAggregations(aggregations); // clear parameters in last-invoke for (int i = 0; i < aggregationsParametersLastInvoke.length; i++) { aggregationsParametersLastInvoke[i] = null; } } aggs = aggregations; lastInvoke = aggregationsParametersLastInvoke; // Update the aggs updateAggregations(aggs, row, rowInspector, false, false, lastInvoke); } /** * Based on user-parameters, should the hash table be flushed. * * @param newKeys * keys for the row under consideration **/ private boolean shouldBeFlushed(KeyWrapper newKeys) { int numEntries = hashAggregations.size(); long usedMemory; float rate; // The fixed size for the aggregation class is already known. Get the // variable portion of the size every NUMROWSESTIMATESIZE rows. if ((numEntriesHashTable == 0) || ((numEntries % NUMROWSESTIMATESIZE) == 0)) { //check how much memory left memory usedMemory = memoryMXBean.getHeapMemoryUsage().getUsed(); rate = (float) usedMemory / (float) maxMemory; if (rate > memoryThreshold) { return true; } for (Integer pos : keyPositionsSize) { Object key = newKeys.getKeyArray()[pos.intValue()]; // Ignore nulls if (key != null) { if (key instanceof LazyPrimitive) { totalVariableSize += ((LazyPrimitive<LazyStringObjectInspector, Text>) key) .getWritableObject().getLength(); } else if (key instanceof String) { totalVariableSize += ((String) key).length(); } else if (key instanceof Text) { totalVariableSize += ((Text) key).getLength(); } } } AggregationBuffer[] aggs = null; if (aggrPositions.size() > 0) { KeyWrapper newKeyProber = newKeys.copyKey(); aggs = hashAggregations.get(newKeyProber); } for (varLenFields v : aggrPositions) { int aggrPos = v.getAggrPos(); List<Field> fieldsVarLen = v.getFields(); AggregationBuffer agg = aggs[aggrPos]; try { for (Field f : fieldsVarLen) { totalVariableSize += ((String) f.get(agg)).length(); } } catch (IllegalAccessException e) { assert false; } } numEntriesVarSize++; // Update the number of entries that can fit in the hash table numEntriesHashTable = (int) (maxHashTblMemory / (fixedRowSize + (totalVariableSize / numEntriesVarSize))); LOG.trace("Hash Aggr: #hash table = " + numEntries + " #max in hash table = " + numEntriesHashTable); } // flush if necessary if (numEntries >= numEntriesHashTable) { return true; } return false; } private void flush(boolean complete) throws HiveException { countAfterReport = 0; // Currently, the algorithm flushes 10% of the entries - this can be // changed in the future if (complete) { Iterator<Map.Entry<KeyWrapper, AggregationBuffer[]>> iter = hashAggregations.entrySet().iterator(); while (iter.hasNext()) { Map.Entry<KeyWrapper, AggregationBuffer[]> m = iter.next(); forward(m.getKey().getKeyArray(), m.getValue()); } hashAggregations.clear(); hashAggregations = null; LOG.info("Hash Table completed flushed"); return; } int oldSize = hashAggregations.size(); LOG.info("Hash Tbl flush: #hash table = " + oldSize); Iterator<Map.Entry<KeyWrapper, AggregationBuffer[]>> iter = hashAggregations.entrySet().iterator(); int numDel = 0; while (iter.hasNext()) { Map.Entry<KeyWrapper, AggregationBuffer[]> m = iter.next(); forward(m.getKey().getKeyArray(), m.getValue()); iter.remove(); numDel++; if (numDel * 10 >= oldSize) { LOG.info("Hash Table flushed: new size = " + hashAggregations.size()); return; } } } transient Object[] forwardCache; /** * Forward a record of keys and aggregation results. * * @param keys * The keys in the record * @throws HiveException */ protected void forward(Object[] keys, AggregationBuffer[] aggs) throws HiveException { int totalFields = keys.length + aggs.length; if (forwardCache == null) { forwardCache = new Object[totalFields]; } for (int i = 0; i < keys.length; i++) { forwardCache[i] = keys[i]; } for (int i = 0; i < aggs.length; i++) { forwardCache[keys.length + i] = aggregationEvaluators[i].evaluate(aggs[i]); } forward(forwardCache, outputObjInspector); } /** * We need to forward all the aggregations to children. * */ @Override public void closeOp(boolean abort) throws HiveException { if (!abort) { try { // If there is no grouping key and no row came to this operator if (firstRow && (keyFields.length == 0)) { firstRow = false; // There is no grouping key - simulate a null row // This is based on the assumption that a null row is ignored by // aggregation functions for (int ai = 0; ai < aggregations.length; ai++) { // o is set to NULL in order to distinguish no rows at all Object[] o; if (aggregationParameterFields[ai].length > 0) { o = new Object[aggregationParameterFields[ai].length]; } else { o = null; } // Calculate the parameters for (int pi = 0; pi < aggregationParameterFields[ai].length; pi++) { o[pi] = null; } aggregationEvaluators[ai].aggregate(aggregations[ai], o); } // create dummy keys - size 0 forward(new Object[0], aggregations); } else { if (hashAggregations != null) { LOG.info("Begin Hash Table flush at close: size = " + hashAggregations.size()); Iterator iter = hashAggregations.entrySet().iterator(); while (iter.hasNext()) { Map.Entry<KeyWrapper, AggregationBuffer[]> m = (Map.Entry) iter.next(); forward(m.getKey().getKeyArray(), m.getValue()); iter.remove(); } hashAggregations.clear(); } else if (aggregations != null) { // sort-based aggregations if (currentKeys != null) { forward(currentKeys.getKeyArray(), aggregations); } currentKeys = null; } else { // The GroupByOperator is not initialized, which means there is no // data // (since we initialize the operators when we see the first record). // Just do nothing here. } } } catch (Exception e) { e.printStackTrace(); throw new HiveException(e); } } } // Group by contains the columns needed - no need to aggregate from children public List<String> genColLists(HashMap<Operator<? extends Serializable>, OpParseContext> opParseCtx) { List<String> colLists = new ArrayList<String>(); ArrayList<ExprNodeDesc> keys = conf.getKeys(); for (ExprNodeDesc key : keys) { colLists = Utilities.mergeUniqElems(colLists, key.getCols()); } ArrayList<AggregationDesc> aggrs = conf.getAggregators(); for (AggregationDesc aggr : aggrs) { ArrayList<ExprNodeDesc> params = aggr.getParameters(); for (ExprNodeDesc param : params) { colLists = Utilities.mergeUniqElems(colLists, param.getCols()); } } return colLists; } /** * @return the name of the operator */ @Override public String getName() { return "GBY"; } @Override public OperatorType getType() { return OperatorType.GROUPBY; } }