com.uber.hoodie.common.HoodieJsonPayload.java Source code

Introduction

Here is the source code for com.uber.hoodie.common.HoodieJsonPayload.java

Source

/*
 * Copyright (c) 2016 Uber Technologies, Inc. (hoodie-dev-group@uber.com)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *          http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.uber.hoodie.common;

import com.uber.hoodie.avro.MercifulJsonConverter;
import com.uber.hoodie.common.model.HoodieRecordPayload;
import com.uber.hoodie.exception.HoodieException;

import org.apache.avro.Schema;
import org.apache.avro.generic.IndexedRecord;
import org.apache.commons.io.IOUtils;
import org.codehaus.jackson.JsonNode;
import org.codehaus.jackson.map.ObjectMapper;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.StringWriter;
import java.util.Optional;
import java.util.zip.Deflater;
import java.util.zip.DeflaterOutputStream;
import java.util.zip.InflaterInputStream;

public class HoodieJsonPayload implements HoodieRecordPayload<HoodieJsonPayload> {
    private byte[] jsonDataCompressed;
    private int dataSize;

    public HoodieJsonPayload(String json) throws IOException {
        this.jsonDataCompressed = compressData(json);
        this.dataSize = json.length();
    }

    @Override
    public HoodieJsonPayload preCombine(HoodieJsonPayload another) {
        return this;
    }

    @Override
    public Optional<IndexedRecord> combineAndGetUpdateValue(IndexedRecord oldRec, Schema schema)
            throws IOException {
        return getInsertValue(schema);
    }

    @Override
    public Optional<IndexedRecord> getInsertValue(Schema schema) throws IOException {
        MercifulJsonConverter jsonConverter = new MercifulJsonConverter(schema);
        return Optional.of(jsonConverter.convert(getJsonData()));
    }

    private String getJsonData() throws IOException {
        return unCompressData(jsonDataCompressed);
    }

    private byte[] compressData(String jsonData) throws IOException {
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        Deflater deflater = new Deflater(Deflater.BEST_COMPRESSION);
        DeflaterOutputStream dos = new DeflaterOutputStream(baos, deflater, true);
        try {
            dos.write(jsonData.getBytes());
        } finally {
            dos.flush();
            dos.close();
            // Its important to call this.
            // Deflater takes off-heap native memory and does not release until GC kicks in
            deflater.end();
        }
        return baos.toByteArray();
    }

    private String unCompressData(byte[] data) throws IOException {
        InflaterInputStream iis = new InflaterInputStream(new ByteArrayInputStream(data));
        try {
            StringWriter sw = new StringWriter(dataSize);
            IOUtils.copy(iis, sw);
            return sw.toString();
        } finally {
            iis.close();
        }
    }

    private String getFieldFromJsonOrFail(String field) throws IOException {
        JsonNode node = new ObjectMapper().readTree(getJsonData());
        if (!node.has(field)) {
            throw new HoodieException("Field :" + field + " not found in payload => " + node.toString());
        }
        return node.get(field).getTextValue();
    }

    public String getRowKey(String keyColumnField) throws IOException {
        return getFieldFromJsonOrFail(keyColumnField);
    }

    public String getPartitionPath(String partitionPathField) throws IOException {
        return getFieldFromJsonOrFail(partitionPathField);
    }
}