Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.drill.exec.store.hive; import java.util.List; import java.util.Properties; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import com.google.common.util.concurrent.ListenableFuture; import io.netty.buffer.DrillBuf; import org.apache.drill.common.exceptions.ExecutionSetupException; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.types.TypeProtos.MajorType; import org.apache.drill.exec.exception.SchemaChangeException; import org.apache.drill.exec.expr.TypeHelper; import org.apache.drill.exec.ops.FragmentContext; import org.apache.drill.exec.ops.OperatorContext; import org.apache.drill.exec.physical.impl.OutputMutator; import org.apache.drill.exec.record.MaterializedField; import org.apache.drill.exec.server.options.OptionManager; import org.apache.drill.exec.store.AbstractRecordReader; import org.apache.drill.exec.vector.AllocationHelper; import org.apache.drill.exec.vector.ValueVector; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; import org.apache.hadoop.hive.serde2.SerDe; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.RecordReader; import org.apache.hadoop.mapred.Reporter; import com.google.common.collect.Lists; import org.apache.hadoop.security.UserGroupInformation; public abstract class HiveAbstractReader extends AbstractRecordReader { protected static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HiveAbstractReader.class); protected final DrillBuf managedBuffer; protected HiveTableWithColumnCache table; protected HivePartition partition; protected InputSplit inputSplit; protected List<String> selectedColumnNames; protected List<StructField> selectedStructFieldRefs = Lists.newArrayList(); protected List<TypeInfo> selectedColumnTypes = Lists.newArrayList(); protected List<ObjectInspector> selectedColumnObjInspectors = Lists.newArrayList(); protected List<HiveFieldConverter> selectedColumnFieldConverters = Lists.newArrayList(); protected List<String> selectedPartitionNames = Lists.newArrayList(); protected List<TypeInfo> selectedPartitionTypes = Lists.newArrayList(); protected List<Object> selectedPartitionValues = Lists.newArrayList(); // SerDe of the reading partition (or table if the table is non-partitioned) protected SerDe partitionSerDe; // ObjectInspector to read data from partitionSerDe (for a non-partitioned table this is same as the table // ObjectInspector). protected StructObjectInspector partitionOI; // Final ObjectInspector. We may not use the partitionOI directly if there are schema changes between the table and // partition. If there are no schema changes then this is same as the partitionOI. protected StructObjectInspector finalOI; // Converter which converts data from partition schema to table schema. protected Converter partTblObjectInspectorConverter; protected Object key; protected RecordReader<Object, Object> reader; protected List<ValueVector> vectors = Lists.newArrayList(); protected List<ValueVector> pVectors = Lists.newArrayList(); protected boolean empty; protected HiveConf hiveConf; protected FragmentContext fragmentContext; protected String defaultPartitionValue; protected final UserGroupInformation proxyUgi; protected static final int TARGET_RECORD_COUNT = 4000; public HiveAbstractReader(HiveTableWithColumnCache table, HivePartition partition, InputSplit inputSplit, List<SchemaPath> projectedColumns, FragmentContext context, final HiveConf hiveConf, UserGroupInformation proxyUgi) throws ExecutionSetupException { this.table = table; this.partition = partition; this.inputSplit = inputSplit; this.empty = (inputSplit == null && partition == null); this.hiveConf = hiveConf; this.fragmentContext = context; this.proxyUgi = proxyUgi; this.managedBuffer = fragmentContext.getManagedBuffer().reallocIfNeeded(256); setColumns(projectedColumns); } public abstract void internalInit(Properties tableProperties, RecordReader<Object, Object> reader); private void init() throws ExecutionSetupException { final JobConf job = new JobConf(hiveConf); // Get the configured default val defaultPartitionValue = hiveConf.get(ConfVars.DEFAULTPARTITIONNAME.varname); Properties tableProperties; try { tableProperties = HiveUtilities.getTableMetadata(table); final Properties partitionProperties = (partition == null) ? tableProperties : HiveUtilities.getPartitionMetadata(partition, table); HiveUtilities.addConfToJob(job, partitionProperties); final SerDe tableSerDe = createSerDe(job, table.getSd().getSerdeInfo().getSerializationLib(), tableProperties); final StructObjectInspector tableOI = getStructOI(tableSerDe); if (partition != null) { partitionSerDe = createSerDe(job, partition.getSd().getSerdeInfo().getSerializationLib(), partitionProperties); partitionOI = getStructOI(partitionSerDe); finalOI = (StructObjectInspector) ObjectInspectorConverters.getConvertedOI(partitionOI, tableOI); partTblObjectInspectorConverter = ObjectInspectorConverters.getConverter(partitionOI, finalOI); job.setInputFormat(HiveUtilities.getInputFormatClass(job, partition.getSd(), table)); } else { // For non-partitioned tables, there is no need to create converter as there are no schema changes expected. partitionSerDe = tableSerDe; partitionOI = tableOI; partTblObjectInspectorConverter = null; finalOI = tableOI; job.setInputFormat(HiveUtilities.getInputFormatClass(job, table.getSd(), table)); } if (logger.isTraceEnabled()) { for (StructField field : finalOI.getAllStructFieldRefs()) { logger.trace("field in finalOI: {}", field.getClass().getName()); } logger.trace("partitionSerDe class is {} {}", partitionSerDe.getClass().getName()); } // Get list of partition column names final List<String> partitionNames = Lists.newArrayList(); for (FieldSchema field : table.getPartitionKeys()) { partitionNames.add(field.getName()); } // We should always get the columns names from ObjectInspector. For some of the tables (ex. avro) metastore // may not contain the schema, instead it is derived from other sources such as table properties or external file. // SerDe object knows how to get the schema with all the config and table properties passed in initialization. // ObjectInspector created from the SerDe object has the schema. final StructTypeInfo sTypeInfo = (StructTypeInfo) TypeInfoUtils.getTypeInfoFromObjectInspector(finalOI); final List<String> tableColumnNames = sTypeInfo.getAllStructFieldNames(); // Select list of columns for project pushdown into Hive SerDe readers. final List<Integer> columnIds = Lists.newArrayList(); if (isStarQuery()) { selectedColumnNames = tableColumnNames; for (int i = 0; i < selectedColumnNames.size(); i++) { columnIds.add(i); } selectedPartitionNames = partitionNames; } else { selectedColumnNames = Lists.newArrayList(); for (SchemaPath field : getColumns()) { String columnName = field.getRootSegment().getPath(); if (partitionNames.contains(columnName)) { selectedPartitionNames.add(columnName); } else { columnIds.add(tableColumnNames.indexOf(columnName)); selectedColumnNames.add(columnName); } } } ColumnProjectionUtils.appendReadColumns(job, columnIds, selectedColumnNames); for (String columnName : selectedColumnNames) { StructField fieldRef = finalOI.getStructFieldRef(columnName); selectedStructFieldRefs.add(fieldRef); ObjectInspector fieldOI = fieldRef.getFieldObjectInspector(); TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(fieldOI.getTypeName()); selectedColumnObjInspectors.add(fieldOI); selectedColumnTypes.add(typeInfo); selectedColumnFieldConverters.add(HiveFieldConverter.create(typeInfo, fragmentContext)); } for (int i = 0; i < selectedColumnNames.size(); ++i) { logger.trace("inspector:typeName={}, className={}, TypeInfo: {}, converter:{}", selectedColumnObjInspectors.get(i).getTypeName(), selectedColumnObjInspectors.get(i).getClass().getName(), selectedColumnTypes.get(i).toString(), selectedColumnFieldConverters.get(i).getClass().getName()); } for (int i = 0; i < table.getPartitionKeys().size(); i++) { FieldSchema field = table.getPartitionKeys().get(i); if (selectedPartitionNames.contains(field.getName())) { TypeInfo pType = TypeInfoUtils.getTypeInfoFromTypeString(field.getType()); selectedPartitionTypes.add(pType); if (partition != null) { selectedPartitionValues.add(HiveUtilities.convertPartitionType(pType, partition.getValues().get(i), defaultPartitionValue)); } } } } catch (Exception e) { throw new ExecutionSetupException("Failure while initializing Hive Reader " + this.getClass().getName(), e); } if (!empty) { try { reader = (org.apache.hadoop.mapred.RecordReader<Object, Object>) job.getInputFormat() .getRecordReader(inputSplit, job, Reporter.NULL); logger.trace("hive reader created: {} for inputSplit {}", reader.getClass().getName(), inputSplit.toString()); } catch (Exception e) { throw new ExecutionSetupException( "Failed to get o.a.hadoop.mapred.RecordReader from Hive InputFormat", e); } internalInit(tableProperties, reader); } } /** * Utility method which creates a SerDe object for given SerDe class name and properties. */ private static SerDe createSerDe(final JobConf job, final String sLib, final Properties properties) throws Exception { final Class<? extends SerDe> c = Class.forName(sLib).asSubclass(SerDe.class); final SerDe serde = c.getConstructor().newInstance(); serde.initialize(job, properties); return serde; } private static StructObjectInspector getStructOI(final SerDe serDe) throws Exception { ObjectInspector oi = serDe.getObjectInspector(); if (oi.getCategory() != ObjectInspector.Category.STRUCT) { throw new UnsupportedOperationException(String.format("%s category not supported", oi.getCategory())); } return (StructObjectInspector) oi; } @Override public void setup(OperatorContext context, OutputMutator output) throws ExecutionSetupException { // initializes "reader" final Callable<Void> readerInitializer = new Callable<Void>() { @Override public Void call() throws Exception { init(); return null; } }; final ListenableFuture<Void> result = context.runCallableAs(proxyUgi, readerInitializer); try { result.get(); } catch (InterruptedException e) { result.cancel(true); // Preserve evidence that the interruption occurred so that code higher up on the call stack can learn of the // interruption and respond to it if it wants to. Thread.currentThread().interrupt(); } catch (ExecutionException e) { throw ExecutionSetupException.fromThrowable(e.getMessage(), e); } try { final OptionManager options = fragmentContext.getOptions(); for (int i = 0; i < selectedColumnNames.size(); i++) { MajorType type = HiveUtilities.getMajorTypeFromHiveTypeInfo(selectedColumnTypes.get(i), options); MaterializedField field = MaterializedField.create(selectedColumnNames.get(i), type); Class<? extends ValueVector> vvClass = TypeHelper.getValueVectorClass(type.getMinorType(), type.getMode()); vectors.add(output.addField(field, vvClass)); } for (int i = 0; i < selectedPartitionNames.size(); i++) { MajorType type = HiveUtilities.getMajorTypeFromHiveTypeInfo(selectedPartitionTypes.get(i), options); MaterializedField field = MaterializedField.create(selectedPartitionNames.get(i), type); Class<? extends ValueVector> vvClass = TypeHelper .getValueVectorClass(field.getType().getMinorType(), field.getDataMode()); pVectors.add(output.addField(field, vvClass)); } } catch (SchemaChangeException e) { throw new ExecutionSetupException(e); } } /** * To take into account Hive "skip.header.lines.count" property first N values from file are skipped. * Since file can be read in batches (depends on TARGET_RECORD_COUNT), additional checks are made * to determine if it's new file or continuance. * * To take into account Hive "skip.footer.lines.count" property values are buffered in queue * until queue size exceeds number of footer lines to skip, then first value in queue is retrieved. * Buffer of value objects is used to re-use value objects in order to reduce number of created value objects. * For each new file queue is cleared to drop footer lines from previous file. */ @Override public abstract int next(); protected void setValueCountAndPopulatePartitionVectors(int recordCount) { for (ValueVector v : vectors) { v.getMutator().setValueCount(recordCount); } if (partition != null) { populatePartitionVectors(recordCount); } } @Override public void close() { try { if (reader != null) { reader.close(); reader = null; } } catch (Exception e) { logger.warn("Failure while closing Hive Record reader.", e); } } protected void populatePartitionVectors(int recordCount) { for (int i = 0; i < pVectors.size(); i++) { final ValueVector vector = pVectors.get(i); final Object val = selectedPartitionValues.get(i); AllocationHelper.allocateNew(vector, recordCount); if (val != null) { HiveUtilities.populateVector(vector, managedBuffer, val, 0, recordCount); } vector.getMutator().setValueCount(recordCount); } } }