com.alibaba.otter.node.etl.extract.extractor.GroupExtractor.java Source code

Introduction

Here is the source code for com.alibaba.otter.node.etl.extract.extractor.GroupExtractor.java

Source

/*
 * Copyright (C) 2010-2101 Alibaba Group Holding Limited.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.alibaba.otter.node.etl.extract.extractor;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.commons.lang.StringUtils;
import org.springframework.util.Assert;
import org.springframework.util.CollectionUtils;

import com.alibaba.otter.node.etl.extract.exceptions.ExtractException;
import com.alibaba.otter.shared.common.model.config.channel.ChannelParameter.SyncConsistency;
import com.alibaba.otter.shared.common.model.config.data.ColumnGroup;
import com.alibaba.otter.shared.common.model.config.data.ColumnPair;
import com.alibaba.otter.shared.common.model.config.data.DataMediaPair;
import com.alibaba.otter.shared.common.model.config.pipeline.Pipeline;
import com.alibaba.otter.shared.etl.model.DbBatch;
import com.alibaba.otter.shared.etl.model.EventColumn;
import com.alibaba.otter.shared.etl.model.EventData;

/**
 * ?+? group?????group??
 * 
 * @author simon 2012-4-26 ?5:04:51
 */
public class GroupExtractor extends AbstractExtractor<DbBatch> {

    @Override
    public void extract(DbBatch dbBatch) throws ExtractException {
        Assert.notNull(dbBatch);
        Assert.notNull(dbBatch.getRowBatch());

        Pipeline pipeline = getPipeline(dbBatch.getRowBatch().getIdentity().getPipelineId());
        List<DataMediaPair> dataMediaPairs = pipeline.getPairs();

        /**
         * Key = TableId<br>
         * Value = a List of this tableId's column need to sync<br>
         */
        Map<Long, List<ColumnGroup>> groupColumns = new HashMap<Long, List<ColumnGroup>>();

        for (DataMediaPair dataMediaPair : dataMediaPairs) {
            List<ColumnGroup> columnGroups = dataMediaPair.getColumnGroups();
            if (!CollectionUtils.isEmpty(columnGroups)) {
                groupColumns.put(dataMediaPair.getSource().getId(), columnGroups);
            }
        }

        List<EventData> eventDatas = dbBatch.getRowBatch().getDatas();
        for (EventData eventData : eventDatas) {
            if (eventData.getEventType().isDdl()) {
                continue;
            }

            List<ColumnGroup> columnGroups = groupColumns.get(eventData.getTableId());
            if (!CollectionUtils.isEmpty(columnGroups)) {
                for (ColumnGroup columnGroup : columnGroups) {
                    if (columnGroup != null && !CollectionUtils.isEmpty(columnGroup.getColumnPairs())) {
                        groupFilter(eventData, columnGroup);
                    }
                }
            }
        }
    }

    private void groupFilter(EventData eventData, ColumnGroup columnGroup) {
        List<EventColumn> addColumns = new ArrayList<EventColumn>();

        // ??
        Set<String> updatedColumns = new HashSet<String>();
        Set<String> pks = new HashSet<String>();

        // ?????
        for (EventColumn column : eventData.getUpdatedColumns()) {
            updatedColumns.add(column.getColumnName());
        }
        for (EventColumn pk : eventData.getKeys()) {
            pks.add(pk.getColumnName());
        }

        if (!CollectionUtils.isEmpty(eventData.getOldKeys())) {// ??
            int i = 0;
            for (EventColumn pk : eventData.getKeys()) {
                if (!StringUtils.equals(pk.getColumnValue(), eventData.getOldKeys().get(i).getColumnValue())) {
                    updatedColumns.add(pk.getColumnName());
                }
                i++;
            }
        }

        if (containsInGroupColumn(updatedColumns, columnGroup.getColumnPairs())) {// 
            // ?+?  group?
            for (ColumnPair columnPair : columnGroup.getColumnPairs()) {
                boolean groupColumnHasInChangedColunms = false;// ????

                // add by ljh at 2012-11-04
                // ?
                // 1. select??FileResolver?
                // 2. groupupdate=trueGroup?before???????

                // columns??updateColumns
                // for (String columnName : updatedColumns) {
                for (EventColumn column : eventData.getColumns()) {
                    if (StringUtils.equalsIgnoreCase(columnPair.getSourceColumn().getName(),
                            column.getColumnName())) {
                        groupColumnHasInChangedColunms = true;
                        if (!column.isUpdate()) {// ??update=true??
                            column.setUpdate(true);
                        }
                        break;
                    }
                }

                if (!groupColumnHasInChangedColunms) {// ??
                    String columnName = columnPair.getSourceColumn().getName();
                    if (!pks.contains(columnName)) { // ????column????
                        EventColumn addColumn = new EventColumn();
                        addColumn.setColumnName(columnPair.getSourceColumn().getName());
                        addColumn.setUpdate(true);
                        addColumns.add(addColumn);
                    }
                }
            }

            if (!CollectionUtils.isEmpty(addColumns)) {
                // ?
                eventData.getColumns().addAll(addColumns);// ?
                eventData.setSyncConsistency(SyncConsistency.MEDIA);
                return;
            }
        }
    }

    /**
     * ?
     */
    private boolean containsInGroupColumn(Set<String> columns, List<ColumnPair> columnPairs) {
        for (ColumnPair columnPair : columnPairs) {
            for (String columnName : columns) {
                if (StringUtils.equalsIgnoreCase(columnPair.getSourceColumn().getName(), columnName)) {
                    return true;
                }
            }
        }

        return false;
    }
}