com.aliyun.odps.ogg.handler.datahub.DatahubHandler.java Source code

Java tutorial

Introduction

Here is the source code for com.aliyun.odps.ogg.handler.datahub.DatahubHandler.java

Source

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 * <p/>
 * http://www.apache.org/licenses/LICENSE-2.0
 * <p/>
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.aliyun.odps.ogg.handler.datahub;

import com.aliyun.odps.Odps;
import com.aliyun.odps.Table;
import com.aliyun.odps.account.Account;
import com.aliyun.odps.account.AliyunAccount;
import com.aliyun.odps.ogg.handler.datahub.alarm.LogAlarm;
import com.aliyun.odps.ogg.handler.datahub.alarm.OggAlarm;
import com.aliyun.odps.ogg.handler.datahub.operations.OperationHandler;
import com.aliyun.odps.ogg.handler.datahub.operations.OperationTypes;
import com.aliyun.odps.tunnel.TableTunnel;
import com.goldengate.atg.datasource.*;
import com.goldengate.atg.datasource.GGDataSource.Status;
import com.goldengate.atg.datasource.adapt.Op;
import com.goldengate.atg.datasource.meta.DsMetaData;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.*;
import java.text.SimpleDateFormat;
import java.util.*;

public class DatahubHandler extends AbstractHandler {
    private final static Logger logger = LoggerFactory.getLogger(DatahubHandler.class);

    private String alarmImplement = LogAlarm.class.getName();

    // Focus cols, comma seperated
    private String focusFields;
    private String keyFields;

    private String tableMap;

    // Table related
    private String project;
    private String tableName;
    private String accessID;
    private String accessKey;
    private String endPoint = "http://service.odps.aliyun.com/api";
    private String datahubEndPoint = "http://dh.odps.aliyun.com";
    private long shardNumber = 1;
    private int shardTimeout = 60;
    private int hubLifeCycle = 7;
    private String timestampField;

    private String dbDateFormat = "yyyy-MM-dd HH:mm:ss";
    private String timestampDateFormat = "yyyy-MM-dd HH:mm:ss.SSSSSS";
    private String partitionValues;
    private String partitionFields;

    private int batchSize = 1000;
    private int retryCount = 3;
    private int passFailedOperationCount = 0;
    private String failedOperationFileName;

    private HandlerProperties handlerProperties;
    private Odps odps;
    private OggAlarm oggAlarm;

    private String handlerInfoFileName;
    private int badRecordCount;

    @Override
    public void init(DsConfiguration dsConf, DsMetaData dsMeta) {
        super.init(dsConf, dsMeta);

        if (StringUtils.isEmpty(handlerInfoFileName)) {
            handlerInfoFileName = "dirdat/" + getName() + "_handler_info";
        }
        if (StringUtils.isEmpty(failedOperationFileName)) {
            failedOperationFileName = "dirrpt/" + getName() + "_bad_operations.data";
        }

        badRecordCount = 0;
        buildOdps();
        initProperty();
        // For recovery
        handlerProperties.setSkipSendTimes(getSkipSendTimes(handlerInfoFileName, oggAlarm));
    }

    public static int getSkipSendTimes(String fileName, OggAlarm oggAlarm) {
        File handlerInfoFile = new File(fileName);
        if (handlerInfoFile.exists() && !handlerInfoFile.isDirectory()) {
            DataInputStream in = null;
            try {
                in = new DataInputStream(new FileInputStream(handlerInfoFile));
                return in.readInt();
            } catch (IOException e) {
                logger.warn("Error reading handler info file, may cause duplication ", e);
                oggAlarm.warn("Error reading handler info file, may cause duplication ", e);
            } finally {
                if (in != null) {
                    try {
                        in.close();
                    } catch (IOException e) {
                        logger.warn("Close handler info file failed. ", e);
                        oggAlarm.warn("Close handler info file failed. ", e);
                    }
                }
            }
        }
        return 0;
    }

    private void buildOdps() {
        Account account = new AliyunAccount(accessID, accessKey);
        odps = new Odps(account);
        odps.setDefaultProject(project);
        odps.setEndpoint(endPoint);
    }

    private Map<String, String> buildMap(String str) {
        if (StringUtils.isEmpty(str)) {
            return null;
        }
        // "key1/value1,key2/value2,..."
        // all to lower case
        Map<String, String> map = Maps.newHashMap();
        String[] array = str.split(",");
        for (String s : array) {
            String[] pair = s.split("/");
            map.put(pair[0].trim().toLowerCase(), pair[1].trim().toLowerCase());
        }
        return map;
    }

    private Map<String, Set<String>> buildStringSetMap(String str) {
        Map<String, Set<String>> map = Maps.newHashMap();
        if (StringUtils.isEmpty(str)) {
            return map;
        }
        // "table1:name1/type1,name2/type2|table2:name3/type3|...
        // all to lower case

        String[] tableInfos = str.split("\\|");
        for (String tableInfo : tableInfos) {
            String[] nameList = tableInfo.split(":");
            String name = nameList[0].trim().toLowerCase();
            Set<String> valSet = new HashSet<>();
            for (String s : nameList[1].split(",")) {
                valSet.add(s.split("/")[0].trim().toLowerCase());
            }
            map.put(name, valSet);
        }
        return map;
    }

    private void initProperty() {
        handlerProperties = new HandlerProperties();

        handlerProperties.setOracleOdpsTableMap(buildMap(tableMap));
        handlerProperties.setTableKeysMap(buildStringSetMap(keyFields));
        handlerProperties.setTableFocusMap(buildStringSetMap(focusFields));

        // Set custom alarm...
        try {
            oggAlarm = (OggAlarm) Class.forName(alarmImplement).newInstance();
        } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) {
            throw new RuntimeException("Cannot initialize custom OGG alarm. ", e);
        }
        handlerProperties.setOggAlarm(oggAlarm);

        // Set properties...
        handlerProperties.setTimestampField(timestampField);
        handlerProperties.setSimpleDateFormat(new SimpleDateFormat(timestampDateFormat));

        // Build partition...
        List<String> partitionCols = getPropCols(partitionFields);
        List<String> partitionVals = getPropCols(partitionValues);

        // TODO partition map
        if (partitionCols != null) {
            if (partitionVals != null & partitionCols.size() == partitionVals.size()) {
                handlerProperties.setPartitionCols(partitionCols);
                handlerProperties.setPartitionVals(partitionVals);
            } else {
                logger.error("Number of partition cols and vals are not consistent.");
                oggAlarm.error("Number of partition cols and vals are not consistent.");
                throw new RuntimeException("Number of partition cols and vals are not consistent.");
            }
        }

        // Other properties
        handlerProperties.setProject(project);
        handlerProperties.setShardNumber(shardNumber);
        handlerProperties.setOdpsTables(odps.tables());
        handlerProperties.setShardTimeout(shardTimeout);
        handlerProperties.setBatchSize(batchSize);
        handlerProperties.setRetryCount(retryCount);
        handlerProperties.setDbDateFormat(dbDateFormat);
        handlerProperties.setMetaFieldName("meta");
        handlerProperties.setOpTypeFieldName("optype");
        handlerProperties.setReadTimeFieldName("readtime");
        handlerProperties.setHandlerInfoFileName(handlerInfoFileName);

        TableTunnel tunnel = new TableTunnel(odps);
        tunnel.setEndpoint(datahubEndPoint);
        handlerProperties.setTunnel(tunnel);
    }

    private List<String> getPropCols(String propertyString) {
        if (StringUtils.isEmpty(propertyString)) {
            logger.warn("Property is empty. property name:" + propertyString);
            oggAlarm.warn("Property is empty. property name:" + propertyString);
            return null;
        }
        List<String> propList = Lists.newArrayList();
        String[] propCols = propertyString.split(",");
        for (int i = 0; i < propCols.length; i++) {
            propList.add(propCols[i].split("/")[0].trim().toLowerCase());
        }
        return propList;
    }

    @Override
    public Status metaDataChanged(DsEvent e, DsMetaData meta) {
        return super.metaDataChanged(e, meta);
    }

    @Override
    public Status transactionBegin(DsEvent e, DsTransaction tx) {
        handlerProperties.setSendTimesInTx(0);
        return super.transactionBegin(e, tx);
    }

    @Override
    public Status operationAdded(DsEvent e, DsTransaction tx, DsOperation dsOperation) {
        Status status = Status.OK;
        super.operationAdded(e, tx, dsOperation);

        Op op = new Op(dsOperation, getMetaData().getTableMetaData(dsOperation.getTableName()), getConfig());
        // Get corresponding operationHandler
        OperationTypes operationType = OperationTypes.valueOf(dsOperation.getOperationType().toString());
        OperationHandler operationHandler = operationType.getOperationHandler();
        if (operationHandler != null) {
            try {
                operationHandler.process(op, handlerProperties);
                handlerProperties.totalOperations++;
            } catch (Exception e1) {
                if (e1 instanceof OperationHandler.OdpsTableNotSetException) {
                    status = Status.OK;
                    logger.warn(e1.getMessage());
                    oggAlarm.warn(e1.getMessage());
                } else if (e1 instanceof OdpsWriter.WriteHubRetryFailedException) {
                    status = Status.ABEND;
                    oggAlarm.error("Failed after retry", e1);
                    logger.error("Failed after retry", e1);
                } else {
                    String msg = "Unable to process operation.";
                    if (badRecordCount < passFailedOperationCount) {
                        status = Status.OK;
                        badRecordCount++;
                        logBadOperation(op, e1.getMessage());
                        oggAlarm.warn(msg, e1);
                        logger.warn(msg, e1);
                    } else {
                        oggAlarm.error(msg, e1);
                        logger.error(msg, e1);
                        status = Status.ABEND;
                    }
                }
            }
        } else {
            String msg = "Unable to instantiate operation handler. Transaction ID: " + tx.getTranID()
                    + ". Operation type: " + dsOperation.getOperationType().toString();
            if (badRecordCount < passFailedOperationCount) {
                status = Status.OK;
                badRecordCount++;
                logBadOperation(op, msg);
                oggAlarm.warn(msg);
                logger.warn(msg);
            } else {
                oggAlarm.error(msg);
                logger.error(msg);
                status = Status.ABEND;
            }
        }
        return status;
    }

    private void logBadOperation(Op op, String msg) {
        StringBuilder sb = new StringBuilder();
        String sep = "\n";
        sb.append("Bad Operation...").append("\n");
        sb.append("Operation Read Time: ").append(op.getOperation().getReadTimeAsString()).append(".\n");
        sb.append("Error msg: ").append(msg).append(".\n");
        sb.append("Full table name: ").append(op.getTableName().getFullName()).append(".\n");
        sb.append("Operation type: ").append(op.getOperationType().toString()).append(".\n");
        sb.append("Operation detail: ").append("\n");

        List<DsColumn> cols = op.getColumns();
        for (int i = 0; i < cols.size(); i++) {
            String colName = op.getTableMeta().getColumnName(i).toLowerCase();
            sb.append(sep).append(colName).append(":'").append(cols.get(i).getBeforeValue()).append("' -> '")
                    .append(cols.get(i).getAfterValue()).append("'");
        }
        sb.append("\n\n");
        try {
            BufferedWriter bw = new BufferedWriter(new FileWriter(failedOperationFileName, true));
            bw.write(sb.toString());
            bw.close();
        } catch (IOException e) {
            logger.error("logBadOperation() failed. ", e);
            oggAlarm.error("logBadOperation() failed. ", e);
            throw new RuntimeException("logBadOperation() failed. ", e);
        }
    }

    @Override
    public Status transactionCommit(DsEvent e, DsTransaction tx) {
        Status status = super.transactionCommit(e, tx);
        for (OdpsWriter writer : handlerProperties.getTableWriterMap().values()) {
            try {
                writer.flushAll();
            } catch (Exception e1) {
                status = status.ABEND;
                oggAlarm.error("Unable to deliver records", e1);
                logger.error("Unable to deliver records", e1);
            }
        }
        handlerProperties.setSkipSendTimes(0);
        recordSendTimes(0, handlerProperties);
        handlerProperties.totalTxns++;
        return status;
    }

    @Override
    public String reportStatus() {
        StringBuilder sb = new StringBuilder();
        sb.append(":- Status report: mode=").append(getMode());
        sb.append(", transactions=").append(handlerProperties.totalTxns);
        sb.append(", operations=").append(handlerProperties.totalOperations);
        sb.append(", inserts=").append(handlerProperties.totalInserts);
        sb.append(", updates=").append(handlerProperties.totalUpdates);
        sb.append(", deletes=").append(handlerProperties.totalDeletes);

        return sb.toString();
    }

    @Override
    public void destroy() {
        oggAlarm.warn("Handler destroying...");
        super.destroy();
    }

    public static void recordSendTimes(int times, HandlerProperties handlerProperties) {
        DataOutputStream out = null;
        OggAlarm oggAlarm = handlerProperties.getOggAlarm();
        try {
            out = new DataOutputStream(new FileOutputStream(handlerProperties.getHandlerInfoFileName(), false));
            out.writeInt(handlerProperties.getSkipSendTimes());
        } catch (IOException e) {
            oggAlarm.error("Error writing handler info file. sendTimesInTx=" + times + ".", e);
            logger.error("Error writing handler info file. sendTimesInTx=" + times + ".", e);
        } finally {
            if (out != null) {
                try {
                    out.close();
                } catch (IOException e) {
                    oggAlarm.error("Close handler info file failed. sendTimesInTx=" + times + ".", e);
                    logger.error("Close handler info file failed. sendTimesInTx=" + times + ".", e);
                }
            }
        }
    }

    public String getPartitionValues() {
        return partitionValues;
    }

    public void setPartitionValues(String partitionValues) {
        this.partitionValues = partitionValues;
    }

    public String getFocusFields() {
        return focusFields;
    }

    public void setFocusFields(String focusFields) {
        this.focusFields = focusFields;
    }

    public String getProject() {
        return project;
    }

    public void setProject(String project) {
        this.project = project;
    }

    public String getTableName() {
        return tableName;
    }

    public void setTableName(String tableName) {
        this.tableName = tableName;
    }

    public String getAccessID() {
        return accessID;
    }

    public void setAccessID(String accessID) {
        this.accessID = accessID;
    }

    public String getAccessKey() {
        return accessKey;
    }

    public void setAccessKey(String accessKey) {
        this.accessKey = accessKey;
    }

    public String getEndPoint() {
        return endPoint;
    }

    public void setEndPoint(String endPoint) {
        this.endPoint = endPoint;
    }

    public String getDatahubEndPoint() {
        return datahubEndPoint;
    }

    public void setDatahubEndPoint(String datahubEndPoint) {
        this.datahubEndPoint = datahubEndPoint;
    }

    public long getShardNumber() {
        return shardNumber;
    }

    public void setShardNumber(long shardNumber) {
        this.shardNumber = shardNumber;
    }

    public int getShardTimeout() {
        return shardTimeout;
    }

    public void setShardTimeout(int shardTimeout) {
        this.shardTimeout = shardTimeout;
    }

    public String getPartitionFields() {
        return partitionFields;
    }

    public void setPartitionFields(String partitionFields) {
        this.partitionFields = partitionFields;
    }

    public String getKeyFields() {
        return keyFields;
    }

    public void setKeyFields(String keyFields) {
        this.keyFields = keyFields;
    }

    public int getHubLifeCycle() {
        return hubLifeCycle;
    }

    public void setHubLifeCycle(int hubLifeCycle) {
        this.hubLifeCycle = hubLifeCycle;
    }

    public String getTimestampField() {
        return timestampField;
    }

    public String getTimestampDateFormat() {
        return timestampDateFormat;
    }

    public void setTimestampDateFormat(String timestampDateFormat) {
        this.timestampDateFormat = timestampDateFormat;
    }

    public void setTimestampField(String timestampField) {
        this.timestampField = timestampField;
    }

    public String getDbDateFormat() {
        return dbDateFormat;
    }

    public void setDbDateFormat(String dbDateFormat) {
        this.dbDateFormat = dbDateFormat;
    }

    public String getAlarmImplement() {
        return alarmImplement;
    }

    public void setAlarmImplement(String alarmImplement) {
        this.alarmImplement = alarmImplement;
    }

    public int getBatchSize() {
        return batchSize;
    }

    public void setBatchSize(int batchSize) {
        this.batchSize = batchSize;
    }

    public int getRetryCount() {
        return retryCount;
    }

    public void setRetryCount(int retryCount) {
        this.retryCount = retryCount;
    }

    public int getPassFailedOperationCount() {
        return passFailedOperationCount;
    }

    public void setPassFailedOperationCount(int passFailedOperationCount) {
        this.passFailedOperationCount = passFailedOperationCount;
    }

    public String getFailedOperationFileName() {
        return failedOperationFileName;
    }

    public void setFailedOperationFileName(String failedOperationFileName) {
        this.failedOperationFileName = failedOperationFileName;
    }

    public String getHandlerInfoFileName() {
        return handlerInfoFileName;
    }

    public void setHandlerInfoFileName(String handlerInfoFileName) {
        this.handlerInfoFileName = handlerInfoFileName;
    }

    public String getTableMap() {
        return tableMap;
    }

    public void setTableMap(String tableMap) {
        this.tableMap = tableMap;
    }
}