Java tutorial
/* * Copyright (C) 2010-2101 Alibaba Group Holding Limited. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.alibaba.otter.node.etl.transform.transformer; import java.util.ArrayList; import java.util.Collection; import java.util.List; import org.apache.commons.lang.StringUtils; import org.apache.ddlutils.model.Column; import org.apache.ddlutils.model.Table; import org.springframework.util.CollectionUtils; import com.alibaba.otter.node.etl.common.db.dialect.DbDialect; import com.alibaba.otter.node.etl.common.db.dialect.DbDialectFactory; import com.alibaba.otter.node.etl.transform.exception.TransformException; import com.alibaba.otter.shared.common.model.config.ConfigHelper; import com.alibaba.otter.shared.common.model.config.data.ColumnPair; import com.alibaba.otter.shared.common.model.config.data.DataMedia; import com.alibaba.otter.shared.common.model.config.data.DataMedia.ModeValue; import com.alibaba.otter.shared.common.model.config.data.DataMediaPair; import com.alibaba.otter.shared.common.model.config.data.db.DbMediaSource; import com.alibaba.otter.shared.etl.model.EventColumn; import com.alibaba.otter.shared.etl.model.EventData; import com.alibaba.otter.shared.etl.model.EventType; import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; /** * RowData -> RowData?? * * @author jianghang 2011-10-27 ?04:11:45 * @version 4.0.0 */ public class RowDataTransformer extends AbstractOtterTransformer<EventData, EventData> { private DbDialectFactory dbDialectFactory; public EventData transform(EventData data, OtterTransformerContext context) { EventData result = new EventData(); // ?Table DataMedia dataMedia = context.getDataMediaPair().getTarget(); result.setPairId(context.getDataMediaPair().getId()); result.setTableId(dataMedia.getId()); // ??multi buildName(data, result, context.getDataMediaPair()); result.setEventType(data.getEventType()); result.setExecuteTime(data.getExecuteTime()); result.setSyncConsistency(data.getSyncConsistency()); result.setRemedy(data.isRemedy()); result.setSyncMode(data.getSyncMode()); result.setSize(data.getSize()); result.setHint(data.getHint()); result.setWithoutSchema(data.isWithoutSchema()); if (data.getEventType().isDdl()) { // ddl??? if (StringUtils.equalsIgnoreCase(result.getSchemaName(), data.getSchemaName()) && StringUtils.equalsIgnoreCase(result.getTableName(), data.getTableName())) { // ??ddl sql???????? result.setDdlSchemaName(data.getDdlSchemaName()); result.setSql(data.getSql()); return result; } else { throw new TransformException("no support ddl for [" + data.getSchemaName() + "." + data.getTableName() + "] to [" + result.getSchemaName() + "." + result.getTableName() + "] , sql :" + data.getSql()); } } Multimap<String, String> translateColumnNames = HashMultimap.create(); if (context.getDataMediaPair().getColumnPairMode().isInclude()) { // ?????exclude??? List<ColumnPair> columnPairs = context.getDataMediaPair().getColumnPairs(); for (ColumnPair columnPair : columnPairs) { translateColumnNames.put(columnPair.getSourceColumn().getName(), columnPair.getTargetColumn().getName()); } } // table meta DataMediaPair dataMediaPair = context.getDataMediaPair(); boolean useTableTransform = context.getPipeline().getParameters().getUseTableTransform(); boolean enableCompatibleMissColumn = context.getPipeline().getParameters().getEnableCompatibleMissColumn(); TableInfoHolder tableHolder = null; if (useTableTransform || enableCompatibleMissColumn) {// ????table // meta?????? // ?? DbDialect dbDialect = dbDialectFactory.getDbDialect(dataMediaPair.getPipelineId(), (DbMediaSource) dataMedia.getSource()); Table table = dbDialect.findTable(result.getSchemaName(), result.getTableName()); tableHolder = new TableInfoHolder(table, useTableTransform, enableCompatibleMissColumn); } // ?column List<EventColumn> otherColumns = translateColumns(result, data.getColumns(), context.getDataMediaPair(), translateColumnNames, tableHolder); translatePkColumn(result, data.getKeys(), data.getOldKeys(), otherColumns, context.getDataMediaPair(), translateColumnNames, tableHolder); result.setColumns(otherColumns); return result; } /** * schema.name?mutl? * * <pre> * case: * 1. ?:offer , offer * 2. ?:offer[1-128] , offer * 3. ?:offer[1-128] , offer[1-128] * 4. ?:offer , offer[1-128] ?? */ private void buildName(EventData data, EventData result, DataMediaPair pair) { DataMedia targetDataMedia = pair.getTarget(); DataMedia sourceDataMedia = pair.getSource(); String schemaName = buildName(data.getSchemaName(), sourceDataMedia.getNamespaceMode(), targetDataMedia.getNamespaceMode()); String tableName = buildName(data.getTableName(), sourceDataMedia.getNameMode(), targetDataMedia.getNameMode()); result.setSchemaName(schemaName); result.setTableName(tableName); } private String buildName(String name, ModeValue sourceModeValue, ModeValue targetModeValue) { if (targetModeValue.getMode().isWildCard()) { return name; // ?? } else if (targetModeValue.getMode().isMulti()) { int index = ConfigHelper.indexIgnoreCase(sourceModeValue.getMultiValue(), name); if (index == -1) { throw new TransformException( "can not found namespace or name in media:" + sourceModeValue.toString()); } return targetModeValue.getMultiValue().get(index); } else { return targetModeValue.getSingleValue(); } } // ? private List<EventColumn> translateColumns(EventData data, List<EventColumn> columns, DataMediaPair dataMediaPair, Multimap<String, String> translateColumnNames, TableInfoHolder tableHolder) { List<EventColumn> tcolumns = new ArrayList<EventColumn>(); for (EventColumn scolumn : columns) { EventColumn tcolumn = translateColumn(data, scolumn, tableHolder, dataMediaPair, translateColumnNames); if (tcolumn != null) { tcolumns.add(tcolumn); } } return tcolumns; } private void translatePkColumn(EventData data, List<EventColumn> pks, List<EventColumn> oldPks, List<EventColumn> columns, DataMediaPair dataMediaPair, Multimap<String, String> translateColumnNames, TableInfoHolder tableHolder) { if (CollectionUtils.isEmpty(oldPks)) { // ?? List<EventColumn> tpks = new ArrayList<EventColumn>(); for (EventColumn scolumn : pks) { EventColumn tcolumn = translateColumn(data, scolumn, tableHolder, dataMediaPair, translateColumnNames); if (tcolumn != null) { tpks.add(tcolumn); } } data.setKeys(tpks); } else { // ? // modify by ljh at 2012-11-07 , ??view???update table xxx // set pk = newPK where pk = oldPk? List<EventColumn> tnewPks = new ArrayList<EventColumn>(); List<EventColumn> toldPks = new ArrayList<EventColumn>(); for (int i = 0; i < pks.size(); i++) { EventColumn newPk = pks.get(i); EventColumn oldPk = oldPks.get(i); // new pk EventColumn tnewPk = translateColumn(data, newPk, tableHolder, dataMediaPair, translateColumnNames); if (tnewPk != null) { tnewPks.add(tnewPk); // old pk??translateColumnNamesnew // pk?removeview name toldPks.add(translateColumn(tnewPk, oldPk.getColumnValue(), dataMediaPair)); } } data.setKeys(tnewPks); data.setOldKeys(toldPks); // ?sql // update table xxx set pk = newPK where pk = oldPk; // for (int i = 0; i < pks.size(); i++) { // EventColumn scolumn = pks.get(i); // EventColumn oldPk = oldPks.get(i); // // EventColumn updatePk = translateColumn(scolumn, tableHolder, // dataMediaPair, translateColumnNames); // if (scolumn.getColumnValue().equals(oldPk.getColumnValue())) {// // ? // tcolumns.add(updatePk); // } else { // columns.add(updatePk);// ?, set pk = newPK // // where pk = oldPk? // tcolumns.add(translateColumn(updatePk, oldPk.getColumnValue(), // dataMediaPair)); // } // } } } private EventColumn translateColumn(EventData data, EventColumn scolumn, TableInfoHolder tableHolder, DataMediaPair dataMediaPair, Multimap<String, String> translateColumnNames) { EventType type = data.getEventType(); EventColumn tcolumn = new EventColumn(); tcolumn.setNull(scolumn.getColumnValue() == null); tcolumn.setKey(scolumn.isKey());// ?????? tcolumn.setIndex(scolumn.getIndex()); tcolumn.setUpdate(scolumn.isUpdate()); String columnName = translateColumnName(scolumn.getColumnName(), dataMediaPair, translateColumnNames); if (StringUtils.isBlank(columnName)) { throw new TransformException("can't translate column name:" + scolumn.getColumnName() + "in pair:" + dataMediaPair.toString()); } // ? // columnName = StringUtils.remove(columnName, "`"); // // ?eromanga?? tcolumn.setColumnName(columnName); tcolumn.setColumnType(scolumn.getColumnType());// ???? if (tableHolder != null) { // modify by ljh at 2013-01-23 // ??????T? // ?ps. mysql? // ???????null?(??columndefaultValue??) boolean canColumnsNotExist = tableHolder.isEnableCompatibleMissColumn(); if (type == EventType.UPDATE) { // ???null canColumnsNotExist &= !scolumn.isUpdate() && scolumn.isNull(); } else if (type == EventType.INSERT) { // ?null canColumnsNotExist &= scolumn.isNull(); } else if (type == EventType.DELETE) { canColumnsNotExist &= !scolumn.isKey(); // ??? } Column matchDbColumn = getMatchColumn(tableHolder.getTable().getColumns(), tcolumn.getColumnName()); // ????DDL??meta? if (matchDbColumn == null) { // ?reloadtable meta // ?? DbMediaSource dbMediaSource = (DbMediaSource) dataMediaPair.getTarget().getSource(); DbDialect dbDialect = dbDialectFactory.getDbDialect(dataMediaPair.getPipelineId(), dbMediaSource); String schemaName = tableHolder.getTable().getSchema(); if (StringUtils.isEmpty(schemaName)) { schemaName = tableHolder.getTable().getCatalog(); } Table table = dbDialect.findTable(schemaName, tableHolder.getTable().getName(), false); // ??cache tableHolder.setTable(table); matchDbColumn = getMatchColumn(tableHolder.getTable().getColumns(), tcolumn.getColumnName()); if (matchDbColumn == null) { if (canColumnsNotExist) { return null; } else { throw new TransformException(scolumn.getColumnName() + " is not found in " + table.toString() + " and source : " + dataMediaPair.getTarget().getNamespace() + "." + dataMediaPair.getTarget().getName()); } } } if (tableHolder.isUseTableTransform()) { int sqlType = matchDbColumn.getTypeCode(); tcolumn.setColumnType(sqlType); } } // if (dataMediaPair.getTarget().getSource().getType().isOracle()) { // // ?oracle? // String encodeValue = SqlUtils.encoding(scolumn.getColumnValue(), // scolumn.getColumnType(), // dataMediaPair.getSource().getSource().getEncode(), // dataMediaPair.getTarget().getSource().getEncode()); // tcolumn.setColumnValue(encodeValue); // } else { // mysql???? tcolumn.setColumnValue(scolumn.getColumnValue()); // } translateColumnNames.remove(scolumn.getColumnName(), columnName);// ????? return tcolumn; } // ?pk + oldPkvalue?columnwhere pk = oldValue private EventColumn translateColumn(EventColumn scolumn, String newValue, DataMediaPair dataMediaPair) { EventColumn tcolumn = new EventColumn(); tcolumn.setNull(newValue == null); tcolumn.setKey(scolumn.isKey());// ?????? tcolumn.setIndex(scolumn.getIndex()); tcolumn.setColumnName(scolumn.getColumnName()); tcolumn.setColumnType(scolumn.getColumnType()); tcolumn.setUpdate(scolumn.isUpdate()); // if (dataMediaPair.getTarget().getSource().getType().isOracle()) { // // ?oracle? // String encodeValue = SqlUtils.encoding(newValue, // scolumn.getColumnType(), dataMediaPair.getSource() // .getSource() // .getEncode(), dataMediaPair.getTarget().getSource().getEncode()); // tcolumn.setColumnValue(encodeValue); // } else { tcolumn.setColumnValue(newValue); // } return tcolumn; } // ============ helper method ============ /** * ???manager??? */ private String translateColumnName(String srcColumnName, DataMediaPair dataMediaPair, Multimap<String, String> translateDict) { if (dataMediaPair.getColumnPairMode().isExclude() || CollectionUtils.isEmpty(dataMediaPair.getColumnPairs())) { return srcColumnName; // ??? } Collection<String> tColumnNames = translateDict.get(srcColumnName); if (CollectionUtils.isEmpty(tColumnNames)) { throw new TransformException( srcColumnName + " is not found in column pairs: " + translateDict.toString()); } String columnName = tColumnNames.iterator().next(); return columnName; } private Column getMatchColumn(Column[] columns, String columnName) { // for (Column column : columns) { if (column.getName().equalsIgnoreCase(columnName)) { return column; } } return null; } // =============== setter / getter ============= public void setDbDialectFactory(DbDialectFactory dbDialectFactory) { this.dbDialectFactory = dbDialectFactory; } /** * ?reloadtable meta??table. * * @author jianghang 2012-5-16 ?04:34:18 * @version 4.0.2 */ static class TableInfoHolder { private Table table; private boolean useTableTransform = true; private boolean enableCompatibleMissColumn = true; public TableInfoHolder(Table table, boolean useTableTransform, boolean enableCompatibleMissColumn) { this.useTableTransform = useTableTransform; this.enableCompatibleMissColumn = enableCompatibleMissColumn; this.table = table; } public Table getTable() { return table; } public void setTable(Table table) { this.table = table; } public boolean isUseTableTransform() { return useTableTransform; } public void setUseTableTransform(boolean useTableTransform) { this.useTableTransform = useTableTransform; } public boolean isEnableCompatibleMissColumn() { return enableCompatibleMissColumn; } public void setEnableCompatibleMissColumn(boolean enableCompatibleMissColumn) { this.enableCompatibleMissColumn = enableCompatibleMissColumn; } } }