com.alibaba.otter.node.etl.common.pipe.HttpPipeIntegration.java Source code

Java tutorial

Introduction

Here is the source code for com.alibaba.otter.node.etl.common.pipe.HttpPipeIntegration.java

Source

/*
 * Copyright (C) 2010-2101 Alibaba Group Holding Limited.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.alibaba.otter.node.etl.common.pipe;

import java.io.File;
import java.io.IOException;
import java.util.Date;

import mockit.Mock;
import mockit.Mocked;
import mockit.Mockit;

import org.apache.commons.lang.math.RandomUtils;
import org.jtester.annotations.SpringBeanByName;
import org.jtester.annotations.SpringBeanFrom;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;

import com.alibaba.otter.node.common.config.ConfigClientService;
import com.alibaba.otter.node.etl.BaseOtterTest;
import com.alibaba.otter.node.etl.common.jetty.JettyEmbedServer;
import com.alibaba.otter.node.etl.common.pipe.impl.http.AttachmentHttpPipe;
import com.alibaba.otter.node.etl.common.pipe.impl.http.HttpPipeKey;
import com.alibaba.otter.node.etl.common.pipe.impl.http.RowDataHttpPipe;
import com.alibaba.otter.shared.common.model.config.node.Node;
import com.alibaba.otter.shared.common.model.config.node.NodeParameter;
import com.alibaba.otter.shared.common.model.config.parameter.SystemParameter.RetrieverType;
import com.alibaba.otter.shared.common.model.config.pipeline.Pipeline;
import com.alibaba.otter.shared.common.utils.NioUtils;
import com.alibaba.otter.shared.etl.model.DbBatch;
import com.alibaba.otter.shared.etl.model.EventColumn;
import com.alibaba.otter.shared.etl.model.EventData;
import com.alibaba.otter.shared.etl.model.EventType;
import com.alibaba.otter.shared.etl.model.FileBatch;
import com.alibaba.otter.shared.etl.model.FileData;
import com.alibaba.otter.shared.etl.model.Identity;
import com.alibaba.otter.shared.etl.model.RowBatch;

public class HttpPipeIntegration extends BaseOtterTest {

    private static final String tmp = System.getProperty("java.io.tmpdir", "/tmp");

    @SpringBeanByName
    private AttachmentHttpPipe attachmentHttpPipe;

    @SpringBeanByName
    private RowDataHttpPipe rowDataHttpPipe;

    @SpringBeanFrom
    @Mocked
    private ConfigClientService configClientService;

    @BeforeClass
    public void initial() {
        Mockit.setUpMock(JettyEmbedServer.class, new Object() {

            @Mock
            private Integer getPort() {
                return null;
            }

        });

    }

    @Test
    public void test_attachment() {
        final Node currentNode = new Node();
        currentNode.setId(1L);
        currentNode.setIp("127.0.0.1");
        currentNode.setParameters(new NodeParameter());
        final Pipeline pipeline = new Pipeline();
        pipeline.getParameters().setRetriever(RetrieverType.ARIA2C);
        // mock
        new NonStrictExpectations() {

            {
                configClientService.currentNode();
                returns(currentNode);

                configClientService.findPipeline(anyLong);
                returns(pipeline);
            }
        };
        Identity identity = new Identity();
        identity.setChannelId(100L);
        identity.setPipelineId(100L);
        identity.setProcessId(100L);

        FileBatch fileBatch = new FileBatch();
        fileBatch.setIdentity(identity);
        File localFile = new File(tmp, "httpPipeTest.jpg");
        FileData localFileData = new FileData();
        localFileData.setEventType(EventType.INSERT);
        localFileData.setPath(localFile.getPath());
        fileBatch.getFiles().add(localFileData);
        try {
            byte[] data = getBlock(10 * 1024);
            NioUtils.write(data, localFile);
            HttpPipeKey key = attachmentHttpPipe.put(fileBatch);
            File target = attachmentHttpPipe.get(key);
            byte[] getbytes = NioUtils.read(new File(target, localFile.getPath()));
            check(data, getbytes);
        } catch (IOException e) {
            want.fail();
        } finally {
            NioUtils.delete(localFile);
        }

    }

    @Test
    public void test_rowData() {
        final Node currentNode = new Node();
        currentNode.setId(1L);
        currentNode.setIp("127.0.0.1");
        currentNode.setParameters(new NodeParameter());
        final Pipeline pipeline = new Pipeline();
        pipeline.getParameters().setRetriever(RetrieverType.ARIA2C);
        // mock
        new NonStrictExpectations() {

            {
                configClientService.currentNode();
                returns(currentNode);

                configClientService.findPipeline(anyLong);
                returns(pipeline);
            }
        };
        Identity identity = new Identity();
        identity.setChannelId(100L);
        identity.setPipelineId(100L);
        identity.setProcessId(100L);

        FileBatch fileBatch = new FileBatch();
        fileBatch.setIdentity(identity);
        File localFile = new File(tmp, "httpPipeTest.jpg");
        FileData localFileData = new FileData();
        localFileData.setPath(localFile.getPath());
        localFileData.setEventType(EventType.INSERT);
        localFileData.setLastModifiedTime(new Date().getTime());
        localFileData.setSize(100L);
        localFileData.setTableId(1L);
        fileBatch.getFiles().add(localFileData);

        RowBatch rowBatch = new RowBatch();
        rowBatch.setIdentity(identity);
        EventData eventData = new EventData();
        eventData.setTableId(1L);
        eventData.setSchemaName("otter");
        eventData.setTableName("test");
        eventData.setEventType(EventType.INSERT);
        eventData.setExecuteTime(100L);

        EventColumn primaryKey = new EventColumn();
        primaryKey.setColumnName("id");
        primaryKey.setColumnType(1);
        primaryKey.setColumnValue("1");
        primaryKey.setKey(true);
        primaryKey.setNull(false);
        eventData.getKeys().add(primaryKey);

        EventColumn column = new EventColumn();
        column.setColumnName("name");
        column.setColumnType(1);
        column.setColumnValue("test");
        column.setKey(false);
        column.setNull(false);
        eventData.getColumns().add(column);

        rowBatch.merge(eventData);

        DbBatch dbBatch = new DbBatch();
        dbBatch.setRowBatch(rowBatch);
        dbBatch.setFileBatch(fileBatch);

        HttpPipeKey key = rowDataHttpPipe.put(dbBatch);
        DbBatch target = rowDataHttpPipe.get(key);

        want.bool(target.getRowBatch().getIdentity().equals(identity));
        want.object(target).notNull();
    }

    private void check(byte[] src, byte[] dest) {
        want.object(src).notNull();
        want.object(dest).notNull();
        want.bool(src.length == dest.length).is(true);

        for (int i = 0; i < src.length; i++) {
            if (src[i] != dest[i]) {
                want.fail();
            }
        }
    }

    private byte[] getBlock(int length) {
        byte[] rawData = new byte[length];
        for (int i = 0; i < rawData.length; i++) {
            rawData[i] = (byte) (' ' + RandomUtils.nextInt(95));

        }
        return rawData;
    }
}