Java tutorial
/* * Copyright 2014 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of * the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations under * the License. */ package co.cask.cdap.hive.datasets; import co.cask.cdap.api.data.batch.RecordScannable; import co.cask.cdap.api.data.batch.RecordWritable; import co.cask.cdap.api.data.schema.Schema; import co.cask.cdap.api.data.schema.UnsupportedTypeException; import co.cask.cdap.api.dataset.Dataset; import co.cask.cdap.api.dataset.DatasetManagementException; import co.cask.cdap.api.dataset.DatasetSpecification; import co.cask.cdap.common.ServiceUnavailableException; import co.cask.cdap.common.conf.Constants; import co.cask.cdap.data.dataset.SystemDatasetInstantiator; import co.cask.cdap.hive.context.ContextManager; import co.cask.cdap.hive.serde.ObjectDeserializer; import co.cask.cdap.hive.serde.ObjectSerializer; import co.cask.cdap.internal.io.ReflectionSchemaGenerator; import co.cask.cdap.internal.io.SchemaGenerator; import co.cask.cdap.proto.Id; import com.google.common.collect.Lists; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.serde2.SerDe; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.SerDeStats; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.io.ObjectWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.lang.reflect.Type; import java.util.ArrayList; import java.util.Properties; /** * SerDe to serialize Dataset Objects. It MUST implement the deprecated SerDe interface instead of extending the * abstract SerDe class, otherwise we get ClassNotFound exceptions on cdh4.x. */ public class DatasetSerDe implements SerDe { private static final Logger LOG = LoggerFactory.getLogger(DatasetSerDe.class); private static final SchemaGenerator schemaGenerator = new ReflectionSchemaGenerator(); private ObjectInspector objectInspector; private ObjectDeserializer deserializer; private ObjectSerializer serializer; private Schema schema; @Override public void initialize(Configuration conf, Properties properties) throws SerDeException { // The column names are saved as the given inspector to #serialize doesn't preserves them // - maybe because it's an external table // The columns property comes from the Hive metastore, which has it from the create table statement // It is then important that this schema be accurate and in the right order - the same order as // object inspectors will reflect them. String datasetName = properties.getProperty(Constants.Explore.DATASET_NAME); String namespace = properties.getProperty(Constants.Explore.DATASET_NAMESPACE); // no namespace SHOULD be an exception but... Hive calls initialize in several places, one of which is // when you try and drop a table. // When updating from CDAP 2.6, old tables will not have namespace as a serde property. So in order // to avoid a null pointer exception that prevents dropping a table, we handle the null namespace case here. if (namespace == null) { // we also still need an ObjectInspector as Hive uses it to check what columns the table has. this.objectInspector = new ObjectDeserializer(properties, null).getInspector(); return; } if (datasetName == null || datasetName.isEmpty()) { throw new SerDeException("Dataset name not found in serde properties."); } // Hive may call initialize a bunch of times... so remember the schema so we don't instantiate the dataset // a bunch of times. if (schema == null) { Id.DatasetInstance datasetId = Id.DatasetInstance.from(namespace, datasetName); getDatasetSchema(conf, datasetId); } this.deserializer = new ObjectDeserializer(properties, schema); ArrayList<String> columnNames = Lists .newArrayList(StringUtils.split(properties.getProperty("columns"), ",")); this.serializer = new ObjectSerializer(columnNames); this.objectInspector = deserializer.getInspector(); } private void getDatasetSchema(Configuration conf, Id.DatasetInstance datasetId) throws SerDeException { try (ContextManager.Context hiveContext = ContextManager.getContext(conf)) { // apparently the conf can be null in some versions of Hive? // Because it calls initialize just to get the object inspector if (hiveContext == null) { LOG.info("Hive provided a null conf, will not be able to get dataset schema."); return; } // some datasets like Table and ObjectMappedTable have schema in the dataset properties try { DatasetSpecification datasetSpec = hiveContext.getDatasetSpec(datasetId); String schemaStr = datasetSpec.getProperty("schema"); if (schemaStr != null) { schema = Schema.parseJson(schemaStr); return; } } catch (DatasetManagementException | ServiceUnavailableException e) { throw new SerDeException("Could not instantiate dataset " + datasetId, e); } catch (IOException e) { throw new SerDeException("Exception getting schema for dataset " + datasetId, e); } // other datasets must be instantiated to get their schema // conf is null if this is a query that writes to a dataset ClassLoader parentClassLoader = conf == null ? null : conf.getClassLoader(); try (SystemDatasetInstantiator datasetInstantiator = hiveContext .createDatasetInstantiator(parentClassLoader)) { Dataset dataset = datasetInstantiator.getDataset(datasetId); if (dataset == null) { throw new SerDeException("Could not find dataset " + datasetId); } Type recordType; if (dataset instanceof RecordScannable) { recordType = ((RecordScannable) dataset).getRecordType(); } else if (dataset instanceof RecordWritable) { recordType = ((RecordWritable) dataset).getRecordType(); } else { throw new SerDeException("Dataset " + datasetId + " is not explorable."); } schema = schemaGenerator.generate(recordType); } catch (UnsupportedTypeException e) { throw new SerDeException("Dataset " + datasetId + " has an unsupported schema.", e); } catch (IOException e) { throw new SerDeException("Exception while trying to instantiate dataset " + datasetId, e); } } catch (IOException e) { throw new SerDeException("Could not get hive context from configuration.", e); } } @Override public Class<? extends Writable> getSerializedClass() { return Text.class; } @Override public Writable serialize(Object o, ObjectInspector objectInspector) throws SerDeException { // NOTE: the object inspector here is not one that we build. It's a default one that Hive built, // that contains generic names for columns. The object is a list of objects, each element // representing one attribute of the Record type. // The object and the objectInspector represent one row of a query result to write into a dataset. // Therefore, it is not guaranteed that the object exactly matches the schema of the dataset // we want to write into. if (!(objectInspector instanceof StructObjectInspector)) { throw new SerDeException("Trying to serialize with unknown object inspector type " + objectInspector.getClass().getName() + ". Expected StructObjectInspector."); } return serializer.serialize(o, objectInspector); } @Override public SerDeStats getSerDeStats() { // TODO: add real Dataset stats - CDAP-12 return new SerDeStats(); } @Override public Object deserialize(Writable writable) throws SerDeException { ObjectWritable objectWritable = (ObjectWritable) writable; Object obj = objectWritable.get(); try { return deserializer.deserialize(obj); } catch (Throwable t) { LOG.error("Unable to deserialize object {}.", obj, t); throw new SerDeException("Unable to deserialize an object.", t); } } @Override public ObjectInspector getObjectInspector() throws SerDeException { return objectInspector; } }