Java tutorial
/* * Copyright (C) 2010-2101 Alibaba Group Holding Limited. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.alibaba.otter.node.etl.extract.extractor; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; import org.apache.commons.lang.StringUtils; import org.apache.ddlutils.model.Column; import org.apache.ddlutils.model.Table; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.util.Assert; import org.springframework.util.CollectionUtils; import com.alibaba.otter.node.etl.common.db.dialect.DbDialect; import com.alibaba.otter.node.etl.extract.exceptions.ExtractException; import com.alibaba.otter.shared.common.model.config.ConfigException; import com.alibaba.otter.shared.common.model.config.ConfigHelper; import com.alibaba.otter.shared.common.model.config.channel.ChannelParameter.SyncConsistency; import com.alibaba.otter.shared.common.model.config.channel.ChannelParameter.SyncMode; import com.alibaba.otter.shared.common.model.config.data.DataMedia; import com.alibaba.otter.shared.common.model.config.data.db.DbMediaSource; import com.alibaba.otter.shared.common.model.config.pipeline.Pipeline; import com.alibaba.otter.shared.etl.model.DbBatch; import com.alibaba.otter.shared.etl.model.EventColumn; import com.alibaba.otter.shared.etl.model.EventData; import com.alibaba.otter.shared.etl.model.EventType; /** * ???? * * <pre> * buffer * id , table_id , type , pk_data , gmt_create , gmt_modified * * pk_datachar(1) * </pre> * * @author jianghang 2012-4-25 ?04:41:33 * @version 4.0.2 */ public class FreedomExtractor extends AbstractExtractor<DbBatch> { private static final Logger logger = LoggerFactory.getLogger(FreedomExtractor.class); private static final char PK_SPLIT = (char) 1; // private static final String ID = "id"; private static final String TABLE_ID = "table_id"; private static final String FULL_NAME = "full_name"; private static final String TYPE = "type"; private static final String PK_DATA = "pk_data"; public void extract(DbBatch dbBatch) throws ExtractException { Assert.notNull(dbBatch); // ?? Pipeline pipeline = getPipeline(dbBatch.getRowBatch().getIdentity().getPipelineId()); boolean skipFreedom = pipeline.getParameters().getSkipFreedom(); String bufferSchema = pipeline.getParameters().getSystemSchema(); String bufferTable = pipeline.getParameters().getSystemBufferTable(); List<EventData> eventDatas = dbBatch.getRowBatch().getDatas(); Set<EventData> removeDatas = new HashSet<EventData>();// set???remove for (EventData eventData : eventDatas) { if (StringUtils.equalsIgnoreCase(bufferSchema, eventData.getSchemaName()) && StringUtils.equalsIgnoreCase(bufferTable, eventData.getTableName())) { if (eventData.getEventType().isDdl()) { continue; } if (skipFreedom) {// ?? removeDatas.add(eventData); continue; } // ??insert / update if (eventData.getEventType().isInsert() || eventData.getEventType().isUpdate()) { // ?EventData?? EventColumn tableIdColumn = getMatchColumn(eventData.getColumns(), TABLE_ID); // ?tableIdmedia? try { DataMedia dataMedia = null; Long tableId = Long.valueOf(tableIdColumn.getColumnValue()); eventData.setTableId(tableId); if (tableId <= 0) { // full_name // ??schema+table name EventColumn fullNameColumn = getMatchColumn(eventData.getColumns(), FULL_NAME); if (fullNameColumn != null) { String[] names = StringUtils.split(fullNameColumn.getColumnValue(), "."); if (names.length >= 2) { dataMedia = ConfigHelper.findSourceDataMedia(pipeline, names[0], names[1]); eventData.setTableId(dataMedia.getId()); } else { throw new ConfigException("no such DataMedia " + names); } } } else { // tableId?tableId??? dataMedia = ConfigHelper.findDataMedia(pipeline, Long.valueOf(tableIdColumn.getColumnValue())); } DbDialect dbDialect = dbDialectFactory.getDbDialect(pipeline.getId(), (DbMediaSource) dataMedia.getSource()); // offer[1-128]?? if (!dataMedia.getNameMode().getMode().isSingle() || !dataMedia.getNamespaceMode().getMode().isSingle()) { boolean hasError = true; EventColumn fullNameColumn = getMatchColumn(eventData.getColumns(), FULL_NAME); if (fullNameColumn != null) { String[] names = StringUtils.split(fullNameColumn.getColumnValue(), "."); if (names.length >= 2) { eventData.setSchemaName(names[0]); eventData.setTableName(names[1]); hasError = false; } } if (hasError) { // ? logger.warn("dataMedia mode:{} , fullname:{} ", dataMedia.getMode(), fullNameColumn == null ? null : fullNameColumn.getColumnValue()); removeDatas.add(eventData); // ? continue; } } else { eventData.setSchemaName(dataMedia.getNamespace()); eventData.setTableName(dataMedia.getName()); } // EventColumn typeColumn = getMatchColumn(eventData.getColumns(), TYPE); EventType eventType = EventType.valuesOf(typeColumn.getColumnValue()); eventData.setEventType(eventType); if (eventType.isUpdate()) {// updateinsert?merge // sql eventData.setEventType(EventType.INSERT); } else if (eventType.isDdl()) { dbDialect.reloadTable(eventData.getSchemaName(), eventData.getTableName()); removeDatas.add(eventData);// ? continue; } // ? EventColumn pkDataColumn = getMatchColumn(eventData.getColumns(), PK_DATA); String pkData = pkDataColumn.getColumnValue(); String[] pks = StringUtils.split(pkData, PK_SPLIT); Table table = dbDialect.findTable(eventData.getSchemaName(), eventData.getTableName()); List<EventColumn> newColumns = new ArrayList<EventColumn>(); Column[] primaryKeyColumns = table.getPrimaryKeyColumns(); if (primaryKeyColumns.length > pks.length) { throw new ExtractException( "data pk column size not match , data:" + eventData.toString()); } // Column[] allColumns = table.getColumns(); int pkIndex = 0; for (int i = 0; i < allColumns.length; i++) { Column column = allColumns[i]; if (column.isPrimaryKey()) { EventColumn newColumn = new EventColumn(); newColumn.setIndex(i); // newColumn.setColumnName(column.getName()); newColumn.setColumnType(column.getTypeCode()); newColumn.setColumnValue(pks[pkIndex]); newColumn.setKey(true); newColumn.setNull(pks[pkIndex] == null); newColumn.setUpdate(true); // newColumns.add(newColumn); pkIndex++; } } // ? eventData.setKeys(newColumns); eventData.setOldKeys(new ArrayList<EventColumn>()); eventData.setColumns(new ArrayList<EventColumn>()); // +?? eventData.setSyncMode(SyncMode.ROW); eventData.setSyncConsistency(SyncConsistency.MEDIA); eventData.setRemedy(true); eventData.setSize(1024);// 1kbbinlog???rpc? } catch (ConfigException e) { // ????? logger.info("find DataMedia error " + eventData.toString(), e); removeDatas.add(eventData); continue; } catch (Throwable e) { // logger.warn("process freedom data error " + eventData.toString(), e); removeDatas.add(eventData); continue; } } else { removeDatas.add(eventData);// } } } if (!CollectionUtils.isEmpty(removeDatas)) { eventDatas.removeAll(removeDatas); } } private EventColumn getMatchColumn(List<EventColumn> columns, String columnName) { for (EventColumn column : columns) { if (StringUtils.equalsIgnoreCase(column.getColumnName(), columnName)) { return column; } } return null; } }