com.alibaba.otter.node.etl.select.selector.canal.CanalEmbedSelector.java Source code

Java tutorial

Introduction

Here is the source code for com.alibaba.otter.node.etl.select.selector.canal.CanalEmbedSelector.java

Source

/*
 * Copyright (C) 2010-2101 Alibaba Group Holding Limited.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.alibaba.otter.node.etl.select.selector.canal;

import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.LockSupport;

import org.apache.commons.lang.SystemUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
import org.springframework.util.CollectionUtils;

import com.alibaba.otter.canal.common.CanalException;
import com.alibaba.otter.canal.extend.communication.CanalConfigClient;
import com.alibaba.otter.canal.extend.ha.MediaHAController;
import com.alibaba.otter.canal.instance.core.CanalInstance;
import com.alibaba.otter.canal.instance.core.CanalInstanceGenerator;
import com.alibaba.otter.canal.instance.manager.CanalInstanceWithManager;
import com.alibaba.otter.canal.instance.manager.model.Canal;
import com.alibaba.otter.canal.instance.manager.model.CanalParameter.HAMode;
import com.alibaba.otter.canal.parse.CanalEventParser;
import com.alibaba.otter.canal.parse.ha.CanalHAController;
import com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser;
import com.alibaba.otter.canal.parse.support.AuthenticationInfo;
import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded;
import com.alibaba.otter.canal.sink.AbstractCanalEventSink;
import com.alibaba.otter.canal.sink.CanalEventSink;
import com.alibaba.otter.node.common.config.ConfigClientService;
import com.alibaba.otter.node.etl.OtterConstants;
import com.alibaba.otter.node.etl.OtterContextLocator;
import com.alibaba.otter.node.etl.select.exceptions.SelectException;
import com.alibaba.otter.node.etl.select.selector.Message;
import com.alibaba.otter.node.etl.select.selector.MessageDumper;
import com.alibaba.otter.node.etl.select.selector.MessageParser;
import com.alibaba.otter.node.etl.select.selector.OtterSelector;
import com.alibaba.otter.shared.common.model.config.data.DataMedia.ModeValue;
import com.alibaba.otter.shared.common.model.config.data.DataMediaPair;
import com.alibaba.otter.shared.common.model.config.pipeline.Pipeline;
import com.alibaba.otter.shared.etl.model.EventData;

/**
 * canal embed???
 * 
 * @author jianghang 2012-7-31 ?02:45:15
 * @version 4.1.0
 */
public class CanalEmbedSelector implements OtterSelector {

    private static final Logger logger = LoggerFactory.getLogger(CanalEmbedSelector.class);
    private static final String SEP = SystemUtils.LINE_SEPARATOR;
    private static final String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss";
    private static final int maxEmptyTimes = 10;
    private int logSplitSize = 50;
    private boolean dump = true;
    private boolean dumpDetail = true;
    private Long pipelineId;
    private CanalServerWithEmbedded canalServer;
    private ClientIdentity clientIdentity;
    private MessageParser messageParser;
    private ConfigClientService configClientService;
    private OtterDownStreamHandler handler;

    private String destination;
    private String filter;
    private int batchSize = 10000;
    private long batchTimeout = -1L;
    private boolean ddlSync = true;
    private boolean filterTableError = false;

    private CanalConfigClient canalConfigClient;
    private volatile boolean running = false; // ??
    private volatile long lastEntryTime = 0;

    public CanalEmbedSelector(Long pipelineId) {
        this.pipelineId = pipelineId;
        canalServer = new CanalServerWithEmbedded();
    }

    public boolean isStart() {
        return running;
    }

    public void start() {
        if (running) {
            return;
        }
        // ?destination/filter?
        Pipeline pipeline = configClientService.findPipeline(pipelineId);
        filter = makeFilterExpression(pipeline);
        destination = pipeline.getParameters().getDestinationName();
        batchSize = pipeline.getParameters().getMainstemBatchsize();
        batchTimeout = pipeline.getParameters().getBatchTimeout();
        ddlSync = pipeline.getParameters().getDdlSync();
        final boolean syncFull = pipeline.getParameters().getSyncMode().isRow()
                || pipeline.getParameters().isEnableRemedy();
        // skip load
        filterTableError = pipeline.getParameters().getSkipSelectException();
        if (pipeline.getParameters().getDumpSelector() != null) {
            dump = pipeline.getParameters().getDumpSelector();
        }

        if (pipeline.getParameters().getDumpSelectorDetail() != null) {
            dumpDetail = pipeline.getParameters().getDumpSelectorDetail();
        }

        canalServer.setCanalInstanceGenerator(new CanalInstanceGenerator() {

            public CanalInstance generate(String destination) {
                Canal canal = canalConfigClient.findCanal(destination);
                final OtterAlarmHandler otterAlarmHandler = new OtterAlarmHandler();
                otterAlarmHandler.setPipelineId(pipelineId);
                OtterContextLocator.autowire(otterAlarmHandler); // spring?
                // slaveId??piplineId?????
                long slaveId = 10000;// 
                if (canal.getCanalParameter().getSlaveId() != null) {
                    slaveId = canal.getCanalParameter().getSlaveId();
                }
                canal.getCanalParameter().setSlaveId(slaveId + pipelineId);
                canal.getCanalParameter().setDdlIsolation(ddlSync);
                canal.getCanalParameter().setFilterTableError(filterTableError);

                CanalInstanceWithManager instance = new CanalInstanceWithManager(canal, filter) {

                    protected CanalHAController initHaController() {
                        HAMode haMode = parameters.getHaMode();
                        if (haMode.isMedia()) {
                            return new MediaHAController(parameters.getMediaGroup(), parameters.getDbUsername(),
                                    parameters.getDbPassword(), parameters.getDefaultDatabaseName());
                        } else {
                            return super.initHaController();
                        }
                    }

                    protected void startEventParserInternal(CanalEventParser parser, boolean isGroup) {
                        super.startEventParserInternal(parser, isGroup);

                        if (eventParser instanceof MysqlEventParser) {
                            // ?
                            ((MysqlEventParser) eventParser).setSupportBinlogFormats("ROW");
                            if (syncFull) {
                                ((MysqlEventParser) eventParser).setSupportBinlogImages("FULL");
                            } else {
                                ((MysqlEventParser) eventParser).setSupportBinlogImages("FULL,MINIMAL");
                            }

                            MysqlEventParser mysqlEventParser = (MysqlEventParser) eventParser;
                            CanalHAController haController = mysqlEventParser.getHaController();

                            if (haController instanceof MediaHAController) {
                                if (isGroup) {
                                    throw new CanalException("not support group database use media HA");
                                }

                                ((MediaHAController) haController).setCanalHASwitchable(mysqlEventParser);
                            }

                            if (!haController.isStart()) {
                                haController.start();
                            }

                            // mediaHatddl???
                            if (haController instanceof MediaHAController) {
                                AuthenticationInfo authenticationInfo = ((MediaHAController) haController)
                                        .getAvailableAuthenticationInfo();
                                ((MysqlEventParser) eventParser).setMasterInfo(authenticationInfo);
                            }
                        }
                    }

                };
                instance.setAlarmHandler(otterAlarmHandler);

                CanalEventSink eventSink = instance.getEventSink();
                if (eventSink instanceof AbstractCanalEventSink) {
                    handler = new OtterDownStreamHandler();
                    handler.setPipelineId(pipelineId);
                    handler.setDetectingIntervalInSeconds(
                            canal.getCanalParameter().getDetectingIntervalInSeconds());
                    OtterContextLocator.autowire(handler); // spring?
                    ((AbstractCanalEventSink) eventSink).addHandler(handler, 0); // 
                    handler.start();
                }

                return instance;
            }
        });
        canalServer.start();

        canalServer.start(destination);
        this.clientIdentity = new ClientIdentity(destination, pipeline.getParameters().getMainstemClientId(),
                filter);
        canalServer.subscribe(clientIdentity);// ?

        running = true;
    }

    public void stop() {
        if (!running) {
            return;
        }
        running = false;
        try {
            handler.stop();
        } catch (Exception e) {
            logger.warn("failed destory handler", e);
        }

        handler = null;
        canalServer.stop(destination);
        canalServer.stop();
    }

    public Message<EventData> selector() throws InterruptedException {
        int emptyTimes = 0;
        com.alibaba.otter.canal.protocol.Message message = null;
        if (batchTimeout < 0) {// ?
            while (running) {
                message = canalServer.getWithoutAck(clientIdentity, batchSize);
                if (message == null || message.getId() == -1L) { // ?
                    applyWait(emptyTimes++);
                } else {
                    break;
                }
            }
            if (!running) {
                throw new InterruptedException();
            }
        } else { // 
            while (running) {
                message = canalServer.getWithoutAck(clientIdentity, batchSize, batchTimeout, TimeUnit.MILLISECONDS);
                if (message == null || message.getId() == -1L) { // ?
                    continue;
                } else {
                    break;
                }
            }
            if (!running) {
                throw new InterruptedException();
            }
        }

        List<EventData> eventDatas = messageParser.parse(pipelineId, message.getEntries()); // /?
        Message<EventData> result = new Message<EventData>(message.getId(), eventDatas);
        // ?entry?
        if (!CollectionUtils.isEmpty(message.getEntries())) {
            long lastEntryTime = message.getEntries().get(message.getEntries().size() - 1).getHeader()
                    .getExecuteTime();
            if (lastEntryTime > 0) {// oracle?0
                this.lastEntryTime = lastEntryTime;
            }
        }

        if (dump && logger.isInfoEnabled()) {
            String startPosition = null;
            String endPosition = null;
            if (!CollectionUtils.isEmpty(message.getEntries())) {
                startPosition = buildPositionForDump(message.getEntries().get(0));
                endPosition = buildPositionForDump(message.getEntries().get(message.getEntries().size() - 1));
            }

            dumpMessages(result, startPosition, endPosition, message.getEntries().size());// 
        }
        return result;
    }

    public void rollback(Long batchId) {
        canalServer.rollback(clientIdentity, batchId);
    }

    public void rollback() {
        canalServer.rollback(clientIdentity);
    }

    public void ack(Long batchId) {
        canalServer.ack(clientIdentity, batchId);
    }

    public List<Long> unAckBatchs() {
        return canalServer.listBatchIds(clientIdentity);
    }

    public Long lastEntryTime() {
        return lastEntryTime;
    }

    /**
     * message
     */
    private synchronized void dumpMessages(Message message, String startPosition, String endPosition, int total) {
        try {
            MDC.put(OtterConstants.splitPipelineSelectLogFileKey, String.valueOf(pipelineId));
            logger.info(SEP + "****************************************************" + SEP);
            logger.info(MessageDumper.dumpMessageInfo(message, startPosition, endPosition, total));
            logger.info("****************************************************" + SEP);
            if (dumpDetail) {// ????
                dumpEventDatas(message.getDatas());
                logger.info("****************************************************" + SEP);
            }
        } finally {
            MDC.remove(OtterConstants.splitPipelineSelectLogFileKey);
        }
    }

    /**
     * ?
     */
    private void dumpEventDatas(List<EventData> eventDatas) {
        int size = eventDatas.size();
        // ??
        int index = 0;
        do {
            if (index + logSplitSize >= size) {
                logger.info(MessageDumper.dumpEventDatas(eventDatas.subList(index, size)));
            } else {
                logger.info(MessageDumper.dumpEventDatas(eventDatas.subList(index, index + logSplitSize)));
            }
            index += logSplitSize;
        } while (index < size);
    }

    /**
     * filter ?
     */
    private String makeFilterExpression(Pipeline pipeline) {
        List<DataMediaPair> dataMediaPairs = pipeline.getPairs();
        if (dataMediaPairs.isEmpty()) {
            throw new SelectException("ERROR ## the pair is empty,the pipeline id = " + pipeline.getId());
        }

        Set<String> mediaNames = new HashSet<String>();
        for (DataMediaPair dataMediaPair : dataMediaPairs) {
            ModeValue namespaceMode = dataMediaPair.getSource().getNamespaceMode();
            ModeValue nameMode = dataMediaPair.getSource().getNameMode();

            if (namespaceMode.getMode().isSingle()) {
                buildFilter(mediaNames, namespaceMode.getSingleValue(), nameMode, false);
            } else if (namespaceMode.getMode().isMulti()) {
                for (String namespace : namespaceMode.getMultiValue()) {
                    buildFilter(mediaNames, namespace, nameMode, false);
                }
            } else if (namespaceMode.getMode().isWildCard()) {
                buildFilter(mediaNames, namespaceMode.getSingleValue(), nameMode, true);
            }
        }

        StringBuilder result = new StringBuilder();
        Iterator<String> iter = mediaNames.iterator();
        int i = -1;
        while (iter.hasNext()) {
            i++;
            if (i == 0) {
                result.append(iter.next());
            } else {
                result.append(",").append(iter.next());
            }
        }

        String markTable = pipeline.getParameters().getSystemSchema() + "."
                + pipeline.getParameters().getSystemMarkTable();
        String bufferTable = pipeline.getParameters().getSystemSchema() + "."
                + pipeline.getParameters().getSystemBufferTable();
        String dualTable = pipeline.getParameters().getSystemSchema() + "."
                + pipeline.getParameters().getSystemDualTable();

        if (!mediaNames.contains(markTable)) {
            result.append(",").append(markTable);
        }

        if (!mediaNames.contains(bufferTable)) {
            result.append(",").append(bufferTable);
        }

        if (!mediaNames.contains(dualTable)) {
            result.append(",").append(dualTable);
        }

        // String otterTable = pipeline.getParameters().getSystemSchema() +
        // "\\..*";
        // if (!mediaNames.contains(otterTable)) {
        // result.append(",").append(otterTable);
        // }

        return result.toString();
    }

    private void buildFilter(Set<String> mediaNames, String namespace, ModeValue nameMode, boolean wildcard) {
        String splitChar = ".";
        if (wildcard) {
            splitChar = "\\.";
        }

        if (nameMode.getMode().isSingle()) {
            mediaNames.add(namespace + splitChar + nameMode.getSingleValue());
        } else if (nameMode.getMode().isMulti()) {
            for (String name : nameMode.getMultiValue()) {
                mediaNames.add(namespace + splitChar + name);
            }
        } else if (nameMode.getMode().isWildCard()) {
            mediaNames.add(namespace + "\\." + nameMode.getSingleValue());
        }
    }

    // ????
    private void applyWait(int emptyTimes) {
        int newEmptyTimes = emptyTimes > maxEmptyTimes ? maxEmptyTimes : emptyTimes;
        if (emptyTimes <= 3) { // 3
            Thread.yield();
        } else { // 3?sleep 10ms
            LockSupport.parkNanos(1000 * 1000L * newEmptyTimes);
        }
    }

    private String buildPositionForDump(Entry entry) {
        long time = entry.getHeader().getExecuteTime();
        Date date = new Date(time);
        SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
        return entry.getHeader().getLogfileName() + ":" + entry.getHeader().getLogfileOffset() + ":"
                + entry.getHeader().getExecuteTime() + "(" + format.format(date) + ")";
    }

    // ================== setter / getter ==================
    public void setMessageParser(MessageParser messageParser) {
        this.messageParser = messageParser;
    }

    public void setConfigClientService(ConfigClientService configClientService) {
        this.configClientService = configClientService;
    }

    public void setCanalConfigClient(CanalConfigClient canalConfigClient) {
        this.canalConfigClient = canalConfigClient;
    }

    public void setDump(boolean dump) {
        this.dump = dump;
    }

}