Java tutorial
/* * Copyright (C) 2010-2101 Alibaba Group Holding Limited. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.alibaba.otter.node.etl.select.selector; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Date; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import org.apache.commons.lang.StringUtils; import org.apache.ddlutils.model.Table; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.util.CollectionUtils; import com.alibaba.otter.canal.protocol.CanalEntry; import com.alibaba.otter.canal.protocol.CanalEntry.Column; import com.alibaba.otter.canal.protocol.CanalEntry.Entry; import com.alibaba.otter.canal.protocol.CanalEntry.RowChange; import com.alibaba.otter.canal.protocol.CanalEntry.RowData; import com.alibaba.otter.node.common.config.ConfigClientService; import com.alibaba.otter.node.etl.common.db.dialect.DbDialect; import com.alibaba.otter.node.etl.common.db.dialect.DbDialectFactory; import com.alibaba.otter.node.etl.common.db.dialect.mysql.MysqlDialect; import com.alibaba.otter.node.etl.common.db.dialect.oracle.OracleDialect; import com.alibaba.otter.node.etl.select.exceptions.SelectException; import com.alibaba.otter.shared.common.model.config.ConfigHelper; import com.alibaba.otter.shared.common.model.config.channel.ChannelParameter.SyncConsistency; import com.alibaba.otter.shared.common.model.config.data.DataMedia; import com.alibaba.otter.shared.common.model.config.data.DataMediaPair; import com.alibaba.otter.shared.common.model.config.data.db.DbMediaSource; import com.alibaba.otter.shared.common.model.config.pipeline.Pipeline; import com.alibaba.otter.shared.common.model.config.pipeline.PipelineParameter; import com.alibaba.otter.shared.etl.model.EventColumn; import com.alibaba.otter.shared.etl.model.EventColumnIndexComparable; import com.alibaba.otter.shared.etl.model.EventData; import com.alibaba.otter.shared.etl.model.EventType; /** * ?? * * @author jianghang 2012-10-25 ?02:31:06 * @version 4.1.2 */ public class MessageParser { private static final Logger logger = LoggerFactory.getLogger(MessageParser.class); private ConfigClientService configClientService; private DbDialectFactory dbDialectFactory; private static final String RETL_CLIENT_FLAG = "_SYNC"; private static final String compatibleMarkTable = "retl_client"; private static final String compatibleMarkInfoColumn = "client_info"; private static final String compatibleMarkIdentifierColumn = "client_identifier"; /** * canal??Entry?otter * * <pre> * ??? * 1. Transaction Begin/End * 2. retl.retl_client/retl.retl_mark ???? * 3. retl.xdual canal? * </pre> */ public List<EventData> parse(Long pipelineId, List<Entry> datas) throws SelectException { List<EventData> eventDatas = new ArrayList<EventData>(); Pipeline pipeline = configClientService.findPipeline(pipelineId); List<Entry> transactionDataBuffer = new ArrayList<Entry>(); // hzus->hz???us?? PipelineParameter pipelineParameter = pipeline.getParameters(); boolean enableLoopbackRemedy = pipelineParameter.isEnableRemedy() && pipelineParameter.isHome() && pipelineParameter.getRemedyAlgorithm().isLoopback(); boolean isLoopback = false; boolean needLoopback = false; // ??loopback???otter??????? long now = new Date().getTime(); try { for (Entry entry : datas) { switch (entry.getEntryType()) { case TRANSACTIONBEGIN: isLoopback = false; break; case ROWDATA: String tableName = entry.getHeader().getTableName(); // ?retl_mark boolean isMarkTable = tableName.equalsIgnoreCase(pipeline.getParameters().getSystemMarkTable()); if (isMarkTable) { RowChange rowChange = RowChange.parseFrom(entry.getStoreValue()); if (!rowChange.getIsDdl()) { int loopback = checkLoopback(pipeline, rowChange.getRowDatas(0)); if (loopback == 2) { needLoopback |= true; // ???? } isLoopback |= loopback > 0; } } // otter3.0schmea???? boolean isCompatibleLoopback = tableName.equalsIgnoreCase(compatibleMarkTable); if (isCompatibleLoopback) { RowChange rowChange = RowChange.parseFrom(entry.getStoreValue()); if (!rowChange.getIsDdl()) { int loopback = checkCompatibleLoopback(pipeline, rowChange.getRowDatas(0)); if (loopback == 2) { needLoopback |= true; // ???? } isLoopback |= loopback > 0; } } if ((!isLoopback || (enableLoopbackRemedy && needLoopback)) && !isMarkTable && !isCompatibleLoopback) { transactionDataBuffer.add(entry); } break; case TRANSACTIONEND: if (!isLoopback || (enableLoopbackRemedy && needLoopback)) { // ?? for (Entry bufferEntry : transactionDataBuffer) { List<EventData> parseDatas = internParse(pipeline, bufferEntry); if (CollectionUtils.isEmpty(parseDatas)) {// ?ddlnull continue; } // ?? long totalSize = bufferEntry.getHeader().getEventLength(); long eachSize = totalSize / parseDatas.size(); for (EventData eventData : parseDatas) { if (eventData == null) { continue; } eventData.setSize(eachSize);// ? if (needLoopback) {// ?? // ???db if (now - eventData.getExecuteTime() > 1000 * pipeline.getParameters().getRemedyDelayThresoldForMedia()) { eventData.setSyncConsistency(SyncConsistency.MEDIA); } else { eventData.setSyncConsistency(SyncConsistency.BASE); } eventData.setRemedy(true); } eventDatas.add(eventData); } } } isLoopback = false; needLoopback = false; transactionDataBuffer.clear(); break; default: break; } } // ???TRANSACTIONEND if (!isLoopback || (enableLoopbackRemedy && needLoopback)) { // ?? for (Entry bufferEntry : transactionDataBuffer) { List<EventData> parseDatas = internParse(pipeline, bufferEntry); if (CollectionUtils.isEmpty(parseDatas)) {// ?ddlnull continue; } // ?? long totalSize = bufferEntry.getHeader().getEventLength(); long eachSize = totalSize / parseDatas.size(); for (EventData eventData : parseDatas) { if (eventData == null) { continue; } eventData.setSize(eachSize);// ? if (needLoopback) {// ?? // ???db if (now - eventData.getExecuteTime() > 1000 * pipeline.getParameters().getRemedyDelayThresoldForMedia()) { eventData.setSyncConsistency(SyncConsistency.MEDIA); } else { eventData.setSyncConsistency(SyncConsistency.BASE); } } eventDatas.add(eventData); } } } } catch (Exception e) { throw new SelectException(e); } return eventDatas; } /** * <pre> * the table def: * channel_info varchar * channel_id varchar * ???? retl_mark channel_info channel_id ? * a. channel_info '_SYNC' ?? * b. ?? * i. channel_id = "xx"channel_id???channelId * ii. ??? * </pre> */ private int checkLoopback(Pipeline pipeline, RowData rowData) { // channel_info // after? Column infokColumn = getColumnIgnoreCase(rowData.getAfterColumnsList(), pipeline.getParameters().getSystemMarkTableInfo()); // ?channelInfo_SYNC? if (infokColumn != null && StringUtils.endsWithIgnoreCase(infokColumn.getValue(), RETL_CLIENT_FLAG)) { return 1; } // ?channelInfo??2?check?otter if (infokColumn != null && StringUtils.equalsIgnoreCase(infokColumn.getValue(), pipeline.getParameters().getChannelInfo())) { return 2; } infokColumn = getColumnIgnoreCase(rowData.getBeforeColumnsList(), pipeline.getParameters().getSystemMarkTableInfo()); // ?channelInfo_SYNC? if (infokColumn != null && StringUtils.endsWithIgnoreCase(infokColumn.getValue(), RETL_CLIENT_FLAG)) { return 1; } // ?channelInfo??2?check?otter if (infokColumn != null && StringUtils.equalsIgnoreCase(infokColumn.getValue(), pipeline.getParameters().getChannelInfo())) { return 2; } // channel_id Column markColumn = getColumnIgnoreCase(rowData.getAfterColumnsList(), pipeline.getParameters().getSystemMarkTableColumn()); // ?channel id if (markColumn != null && pipeline.getChannelId().equals(Long.parseLong(markColumn.getValue()))) { return 2; } markColumn = getColumnIgnoreCase(rowData.getBeforeColumnsList(), pipeline.getParameters().getSystemMarkTableColumn()); if (markColumn != null && pipeline.getChannelId().equals(Long.parseLong(markColumn.getValue()))) { return 2; } return 0; } /** * otter3.0??3.0???? */ private int checkCompatibleLoopback(Pipeline pipeline, RowData rowData) { // _info // after? Column infokColumn = getColumnIgnoreCase(rowData.getAfterColumnsList(), compatibleMarkInfoColumn); // ?channel id if (infokColumn != null && infokColumn.getValue().toUpperCase().endsWith(RETL_CLIENT_FLAG)) { return 1; } infokColumn = getColumnIgnoreCase(rowData.getBeforeColumnsList(), compatibleMarkInfoColumn); if (infokColumn != null && infokColumn.getValue().toUpperCase().endsWith(RETL_CLIENT_FLAG)) { return 1; } // _id Column markColumn = getColumnIgnoreCase(rowData.getAfterColumnsList(), compatibleMarkIdentifierColumn); // ?channel id if (markColumn != null && pipeline.getChannelId().equals(Long.parseLong(markColumn.getValue()))) { return 2; } markColumn = getColumnIgnoreCase(rowData.getBeforeColumnsList(), compatibleMarkIdentifierColumn); if (markColumn != null && pipeline.getChannelId().equals(Long.parseLong(markColumn.getValue()))) { return 2; } return 0; } private Column getColumnIgnoreCase(List<Column> columns, String columName) { for (Column column : columns) { if (column.getName().equalsIgnoreCase(columName)) { return column; } } return null; } private List<EventData> internParse(Pipeline pipeline, Entry entry) { RowChange rowChange = null; try { rowChange = RowChange.parseFrom(entry.getStoreValue()); } catch (Exception e) { throw new SelectException("parser of canal-event has an error , data:" + entry.toString(), e); } if (rowChange == null) { return null; } String schemaName = entry.getHeader().getSchemaName(); String tableName = entry.getHeader().getTableName(); EventType eventType = EventType.valueOf(rowChange.getEventType().name()); // ?DDL? if (eventType.isQuery()) { // query return null; } // ? if (StringUtils.equalsIgnoreCase(pipeline.getParameters().getSystemSchema(), schemaName)) { // do noting if (eventType.isDdl()) { return null; } if (StringUtils.equalsIgnoreCase(pipeline.getParameters().getSystemDualTable(), tableName)) { // ? return null; } } else { if (eventType.isDdl()) { boolean notExistReturnNull = false; if (eventType.isRename()) { notExistReturnNull = true; } DataMedia dataMedia = ConfigHelper.findSourceDataMedia(pipeline, schemaName, tableName, notExistReturnNull); // EventTypeCREATE/ALTER?reload // DataMediaInfo;CREATE/ALTER. if (dataMedia != null && (eventType.isCreate() || eventType.isAlter() || eventType.isRename())) { DbDialect dbDialect = dbDialectFactory.getDbDialect(pipeline.getId(), (DbMediaSource) dataMedia.getSource()); dbDialect.reloadTable(schemaName, tableName);// meta? } boolean ddlSync = pipeline.getParameters().getDdlSync(); if (ddlSync) { // ?ddl? EventData eventData = new EventData(); eventData.setSchemaName(schemaName); eventData.setTableName(tableName); eventData.setEventType(eventType); eventData.setExecuteTime(entry.getHeader().getExecuteTime()); eventData.setSql(rowChange.getSql()); eventData.setDdlSchemaName(rowChange.getDdlSchemaName()); eventData.setTableId(dataMedia.getId()); return Arrays.asList(eventData); } else { return null; } } } List<EventData> eventDatas = new ArrayList<EventData>(); for (RowData rowData : rowChange.getRowDatasList()) { EventData eventData = internParse(pipeline, entry, rowChange, rowData); if (eventData != null) { eventDatas.add(eventData); } } return eventDatas; } /** * ?canal?Event<br> * Oracle:?. <br> * <i>insert:afterColumns???<br> * <i>delete:beforeColumns???<br> * <i>update:before???after???,????<br> * Mysql:??????.<br> * <i>insert:afterColumns???<br> * <i>delete:beforeColumns???<br> * <i>update:beforeColumns???,afterColumns???<br> */ private EventData internParse(Pipeline pipeline, Entry entry, RowChange rowChange, RowData rowData) { EventData eventData = new EventData(); eventData.setTableName(entry.getHeader().getTableName()); eventData.setSchemaName(entry.getHeader().getSchemaName()); eventData.setEventType(EventType.valueOf(rowChange.getEventType().name())); eventData.setExecuteTime(entry.getHeader().getExecuteTime()); EventType eventType = eventData.getEventType(); TableInfoHolder tableHolder = null; if (!StringUtils.equalsIgnoreCase(pipeline.getParameters().getSystemSchema(), eventData.getSchemaName())) { boolean useTableTransform = pipeline.getParameters().getUseTableTransform(); Table table = null; DataMedia dataMedia = ConfigHelper.findSourceDataMedia(pipeline, eventData.getSchemaName(), eventData.getTableName()); eventData.setTableId(dataMedia.getId()); if (useTableTransform || dataMedia.getSource().getType().isOracle()) {// oracle???meta // ?table meta??table? // oracle erosa?????erosa? DbDialect dbDialect = dbDialectFactory.getDbDialect(pipeline.getId(), (DbMediaSource) dataMedia.getSource()); table = dbDialect.findTable(eventData.getSchemaName(), eventData.getTableName());// meta? if (table == null) { logger.warn("find table[{}.{}] is null , may be drop table.", eventData.getSchemaName(), eventData.getTableName()); } tableHolder = new TableInfoHolder(dbDialect, table, useTableTransform); } } List<Column> beforeColumns = rowData.getBeforeColumnsList(); List<Column> afterColumns = rowData.getAfterColumnsList(); String tableName = eventData.getSchemaName() + "." + eventData.getTableName(); // ??all columns boolean isRowMode = pipeline.getParameters().getSyncMode().isRow(); // rowMode??updated boolean needAllColumns = isRowMode || checkNeedAllColumns(pipeline); // ?? Map<String, EventColumn> keyColumns = new LinkedHashMap<String, EventColumn>(); // ?? Map<String, EventColumn> oldKeyColumns = new LinkedHashMap<String, EventColumn>(); // ?? Map<String, EventColumn> notKeyColumns = new LinkedHashMap<String, EventColumn>(); if (eventType.isInsert()) { for (Column column : afterColumns) { if (isKey(tableHolder, tableName, column)) { keyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder)); } else { // mysql notKeyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder)); } } } else if (eventType.isDelete()) { for (Column column : beforeColumns) { if (isKey(tableHolder, tableName, column)) { keyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder)); } else { // mysql notKeyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder)); } } } else if (eventType.isUpdate()) { // ???. for (Column column : beforeColumns) { if (isKey(tableHolder, tableName, column)) { oldKeyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder)); // ?new // key,mysql5.6?minimal?,after?,?before keyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder)); } else { if (needAllColumns && entry.getHeader().getSourceType() == CanalEntry.Type.ORACLE) { // ?oracle?update?aftercolume notKeyColumns.put(column.getName(), copyEventColumn(column, isRowMode, tableHolder)); } } } for (Column column : afterColumns) { if (isKey(tableHolder, tableName, column)) { // ??? keyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder)); } else if (needAllColumns || entry.getHeader().getSourceType() == CanalEntry.Type.ORACLE || column.getUpdated()) { // update?oraclemysql?????,oracle??; // mysql??????????. // oracleafter? boolean isUpdate = true; if (entry.getHeader().getSourceType() == CanalEntry.Type.MYSQL) { // mysqlafter??,oracleafter? isUpdate = column.getUpdated(); } notKeyColumns.put(column.getName(), copyEventColumn(column, isRowMode || isUpdate, tableHolder));// rowModeupdated } } if (entry.getHeader().getSourceType() == CanalEntry.Type.ORACLE) { // oracle? checkUpdateKeyColumns(oldKeyColumns, keyColumns); } } List<EventColumn> keys = new ArrayList<EventColumn>(keyColumns.values()); List<EventColumn> oldKeys = new ArrayList<EventColumn>(oldKeyColumns.values()); List<EventColumn> columns = new ArrayList<EventColumn>(notKeyColumns.values()); Collections.sort(keys, new EventColumnIndexComparable()); Collections.sort(oldKeys, new EventColumnIndexComparable()); Collections.sort(columns, new EventColumnIndexComparable()); if (!keyColumns.isEmpty()) { eventData.setKeys(keys); if (eventData.getEventType().isUpdate() && !oldKeys.equals(keys)) { // update??,old // keys?? eventData.setOldKeys(oldKeys); } eventData.setColumns(columns); // } else if (CanalEntry.Type.MYSQL == // entry.getHeader().getSourceType()) { // // ??mysql? // if (eventType.isUpdate()) { // List<EventColumn> oldColumns = new ArrayList<EventColumn>(); // List<EventColumn> newColumns = new ArrayList<EventColumn>(); // for (Column column : beforeColumns) { // oldColumns.add(copyEventColumn(column, true, tableHolder)); // } // // for (Column column : afterColumns) { // newColumns.add(copyEventColumn(column, true, tableHolder)); // } // Collections.sort(oldColumns, new EventColumnIndexComparable()); // Collections.sort(newColumns, new EventColumnIndexComparable()); // eventData.setOldKeys(oldColumns);// ?? // eventData.setKeys(newColumns);// ?????? // } else { // // ?? // eventData.setKeys(columns); // } } else { throw new SelectException( "this rowdata has no pks , entry: " + entry.toString() + " and rowData: " + rowData); } return eventData; } private boolean checkNeedAllColumns(Pipeline pipeline) { boolean needAllColumns = false; // ??filter/resolver??? for (DataMediaPair pair : pipeline.getPairs()) { needAllColumns |= pair.isExistFilter(); if (pair.getResolverData() != null && pair.getResolverData().getExtensionDataType() != null) { if (pair.getResolverData().getExtensionDataType().isClazz()) { needAllColumns |= StringUtils.isNotEmpty(pair.getResolverData().getClazzPath()); } else { needAllColumns |= StringUtils.isNotEmpty(pair.getResolverData().getSourceText()); } } else { needAllColumns |= Boolean.FALSE; } } return needAllColumns; } /** * oracle?<br> * ??old?<br> * ????oldnew??new. * * @param oldKeys * @param newKeys */ private void checkUpdateKeyColumns(Map<String, EventColumn> oldKeyColumns, Map<String, EventColumn> keyColumns) { // ?? if (oldKeyColumns.size() == 0) { return; } // ??????? if (keyColumns.size() > oldKeyColumns.size()) { return; } // ??????. if (keyColumns.size() == 0) { keyColumns.putAll(oldKeyColumns); return; } // oldnew??new if (oldKeyColumns.size() != keyColumns.size()) { for (String oldKey : oldKeyColumns.keySet()) { if (keyColumns.get(oldKey) == null) { keyColumns.put(oldKey, oldKeyColumns.get(oldKey)); } } } } /** * erosa-protocol's Column ? otter's model EventColumn. * * @param column * @return */ private EventColumn copyEventColumn(Column column, boolean isUpdate, TableInfoHolder tableHolder) { EventColumn eventColumn = new EventColumn(); eventColumn.setIndex(column.getIndex()); eventColumn.setKey(column.getIsKey()); eventColumn.setNull(column.getIsNull()); eventColumn.setColumnName(column.getName()); eventColumn.setColumnValue(column.getValue()); eventColumn.setUpdate(isUpdate); eventColumn.setColumnType(column.getSqlType()); if (tableHolder != null && tableHolder.getTable() != null && (tableHolder.isUseTableTransform() || tableHolder.isOracle())) { org.apache.ddlutils.model.Column dbColumn = tableHolder.getTable().findColumn(column.getName(), false); if (dbColumn == null) { // ?ddl?reloadtable tableHolder.reload(); dbColumn = tableHolder.getTable().findColumn(column.getName(), false); } if (dbColumn != null) { int sqlType = dbColumn.getTypeCode(); if (sqlType != column.getSqlType()) { // oracleerosa?jdbc???? eventColumn.setColumnType(sqlType); logger.info("table [{}] column [{}] is not match , MeType: {}, EType {}", new Object[] { tableHolder.getTable().getName(), column.getName(), sqlType, column.getSqlType() }); } } } return eventColumn; } private boolean isKey(TableInfoHolder tableHolder, String tableName, Column column) { boolean isEKey = column.getIsKey(); if (tableHolder == null || tableHolder.getTable() == null || !tableHolder.isUseTableTransform()) { return isEKey; } org.apache.ddlutils.model.Column dbColumn = tableHolder.getTable().findColumn(column.getName(), false); if (dbColumn == null) { // ?ddl?reloadtable tableHolder.reload(); dbColumn = tableHolder.getTable().findColumn(column.getName(), false); if (dbColumn == null) { throw new SelectException(String.format("not found column[%s] in table[%s]", column.getName(), tableHolder.getTable().toVerboseString())); } } boolean isMKey = dbColumn.isPrimaryKey(); if (isMKey != isEKey) { logger.info("table [{}] column [{}] is not match , isMeky: {}, isEkey {}", new Object[] { tableName, column.getName(), isMKey, isEKey }); } return isMKey; } // ======================== setter / getter ============================= public void setDbDialectFactory(DbDialectFactory dbDialectFactory) { this.dbDialectFactory = dbDialectFactory; } public void setConfigClientService(ConfigClientService configClientService) { this.configClientService = configClientService; } /** * ?reloadtable meta??table. * * @author jianghang 2012-5-16 ?04:34:18 * @version 4.0.2 */ static class TableInfoHolder { private DbDialect dbDialect; private Table table; private boolean useTableTransform; public TableInfoHolder(DbDialect dbDialect, Table table, boolean useTableTransform) { this.dbDialect = dbDialect; this.table = table; this.useTableTransform = useTableTransform; } public Table getTable() { return table; } public void setTable(Table table) { this.table = table; } public DbDialect getDbDialect() { return dbDialect; } public void setDbDialect(DbDialect dbDialect) { this.dbDialect = dbDialect; } public boolean isUseTableTransform() { return useTableTransform; } public void setUseTableTransform(boolean useTableTransform) { this.useTableTransform = useTableTransform; } public boolean isOracle() { return (dbDialect != null && dbDialect instanceof OracleDialect); } public boolean isMysql() { return (dbDialect != null && dbDialect instanceof MysqlDialect); } public void reload() { if (table != null) { String schemaName = StringUtils.isEmpty(table.getCatalog()) ? table.getSchema() : table.getCatalog(); this.table = dbDialect.findTable(schemaName, table.getName(), false); } } } }