com.taobao.metamorphosis.client.http.SimpleHttpConsumer.java Source code

Java tutorial

Introduction

Here is the source code for com.taobao.metamorphosis.client.http.SimpleHttpConsumer.java

Source

/*
 * (C) 2007-2012 Alibaba Group Holding Limited.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *      http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * Authors:
 *   wuhua <wq163@163.com> , boyan <killme2008@gmail.com>
 */
package com.taobao.metamorphosis.client.http;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;

import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import com.taobao.gecko.core.util.StringUtils;
import com.taobao.metamorphosis.Message;
import com.taobao.metamorphosis.client.MetaClientConfig;
import com.taobao.metamorphosis.client.Shutdownable;
import com.taobao.metamorphosis.client.consumer.FetchManager;
import com.taobao.metamorphosis.client.consumer.FetchRequest;
import com.taobao.metamorphosis.client.consumer.InnerConsumer;
import com.taobao.metamorphosis.client.consumer.MessageIterator;
import com.taobao.metamorphosis.client.consumer.MessageListener;
import com.taobao.metamorphosis.client.consumer.RecoverStorageManager;
import com.taobao.metamorphosis.client.consumer.SimpleFetchManager;
import com.taobao.metamorphosis.client.consumer.SubscribeInfoManager;
import com.taobao.metamorphosis.client.consumer.TopicPartitionRegInfo;
import com.taobao.metamorphosis.client.consumer.storage.MysqlOffsetStorage;
import com.taobao.metamorphosis.client.consumer.storage.OffsetStorage;
import com.taobao.metamorphosis.cluster.Partition;
import com.taobao.metamorphosis.exception.MetaClientException;
import com.taobao.metamorphosis.network.HttpStatus;
import com.taobao.metamorphosis.utils.MetaStatLog;
import com.taobao.metamorphosis.utils.StatConstants;

/**
 * SimpleMessageConsumerHTTPclientLB
 * 
 */
public class SimpleHttpConsumer extends SimpleHttpClient implements Shutdownable, InnerConsumer {
    static final Log logger = LogFactory.getLog(SimpleHttpConsumer.class);

    private final HttpClientConfig config;
    private final ScheduledExecutorService scheduledExecutorService;
    private final OffsetStorage offsetStorage;
    private final SubscribeInfoManager subscribeInfoManager;
    private final RecoverStorageManager recoverStorageManager;

    private final FetchManager fetchManager;

    /**
     * topicpartition,offset,
     * HTTPrerebalancecompleteSubscribe;
     * TopicPartitionRegInfooffsetThread Safe
     */
    private final HashMap<String, HashMap<Partition, TopicPartitionRegInfo>> topicRegistry = new HashMap<String, HashMap<Partition, TopicPartitionRegInfo>>();

    public SimpleHttpConsumer(final HttpClientConfig config) {
        super(config);
        this.config = config;
        this.subscribeInfoManager = new SubscribeInfoManager();
        this.offsetStorage = new MysqlOffsetStorage(config.getDataSource());
        this.recoverStorageManager = new RecoverStorageManager(new MetaClientConfig(), this.subscribeInfoManager);
        this.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
            @Override
            public void run() {
                SimpleHttpConsumer.this.commitOffsets();
            }
        }, config.getCommitOffsetPeriodInMills(), config.getCommitOffsetPeriodInMills(), TimeUnit.MILLISECONDS);
        this.fetchManager = new SimpleFetchManager(config, this);
        this.fetchManager.resetFetchState();
    }

    private void commitOffsets() {
        this.offsetStorage.commitOffset(this.config.getGroup(), this.getTopicPartitionRegInfos());
    }

    private List<TopicPartitionRegInfo> getTopicPartitionRegInfos() {
        final List<TopicPartitionRegInfo> rt = new ArrayList<TopicPartitionRegInfo>();
        for (final HashMap<Partition, TopicPartitionRegInfo> subMap : this.topicRegistry.values()) {
            final Collection<TopicPartitionRegInfo> values = subMap.values();
            if (values != null) {
                rt.addAll(values);
            }
        }
        return rt;
    }

    /**
     * topic
     * 
     * @throws MetaClientException
     */
    public MessageIterator get(final String topic, final Partition partition, final long offset, final int maxsize)
            throws MetaClientException, InterruptedException {
        this.checkRequest(topic, partition, offset, maxsize);
        final FetchRequest request = new FetchRequest(null, 0, new TopicPartitionRegInfo(topic, partition, offset),
                maxsize);
        return this.fetch(request, 10000, TimeUnit.MILLISECONDS);
    }

    final private void checkRequest(final String topic, final Partition partition, final long offset,
            final int maxsize) throws MetaClientException {
        if (topic == null || topic.isEmpty()) {
            throw new MetaClientException("Topic null");
        }

        if (offset < 0) {
            throw new MetaClientException("Offset must be positive");
        }

        if (partition == null) {
            throw new MetaClientException("partition must be positive");
        }

        if (maxsize < 0) {
            throw new MetaClientException("maxsize must be positive");
        }
    }

    /**
     * 
     * MessageListenerMessageListener
     *  complete
     * 
     */
    public void subscribe(final String topic, final Partition partition, final int maxSize,
            final MessageListener messageListener) throws MetaClientException {
        this.checkState();
        if (StringUtils.isBlank(topic)) {
            throw new IllegalArgumentException("Blank topic");
        }
        if (messageListener == null) {
            throw new IllegalArgumentException("Null messageListener");
        }
        // add it to group manager
        this.subscribeInfoManager.subscribe(topic, this.config.getGroup(), maxSize, messageListener);

        HashMap<Partition, TopicPartitionRegInfo> partitionTopicInfo = this.topicRegistry.get(topic);
        if (partitionTopicInfo == null) {
            partitionTopicInfo = new HashMap<Partition, TopicPartitionRegInfo>();
            this.topicRegistry.put(topic, partitionTopicInfo);
        }
        TopicPartitionRegInfo topicPartitionRegInfo = this.offsetStorage.load(topic, this.config.getGroup(),
                partition);
        if (topicPartitionRegInfo == null) {
            // 0,TODO 
            this.offsetStorage.initOffset(topic, this.config.getGroup(), partition, 0);// Long.MAX_VALUE
            topicPartitionRegInfo = new TopicPartitionRegInfo(topic, partition, 0);
        }
        partitionTopicInfo.put(partition, topicPartitionRegInfo);

        this.fetchManager.addFetchRequest(new FetchRequest(null, 0L, topicPartitionRegInfo, maxSize));
    }

    /**
     * topic,,
     */
    public void completeSubscribe() throws MetaClientException {
        this.checkState();
        // this.fetchManager.resetFetchState();
        this.fetchManager.startFetchRunner();
    }

    @Override
    public void shutdown() throws MetaClientException {
        super.shutdown();
        if (this.fetchManager.isShutdown()) {
            return;
        }
        try {
            this.fetchManager.stopFetchRunner();
        } catch (final InterruptedException e) {
            Thread.currentThread().interrupt();
        } finally {
            this.scheduledExecutorService.shutdownNow();
            // offsets
            this.commitOffsets();
            this.offsetStorage.close();
        }
    }

    @Override
    public MessageIterator fetch(final FetchRequest fetchRequest, final long timeout, final TimeUnit timeUnit)
            throws MetaClientException, InterruptedException {
        final long start = System.currentTimeMillis();
        final boolean success = false;

        // URI/get/{brokerID}?topic&partition&group, {brokerID}HTTP
        // LBHTTP clientBroker server
        //  HTTP LBHAProxybrokerIDBorker server
        final String uri = "/get/" + fetchRequest.getPartitionObject().getBrokerId() + "?topic="
                + fetchRequest.getTopic() + "&partition=" + fetchRequest.getPartition() + "&offset="
                + fetchRequest.getOffset() + "&group=" + this.config.getGroup() + "&maxsize="
                + fetchRequest.getMaxSize();
        final GetMethod httpget = new GetMethod(uri);
        try {
            this.httpclient.executeMethod(httpget);
            if (httpget.getStatusCode() == HttpStatus.Success) {
                return new MessageIterator(fetchRequest.getTopic(), httpget.getResponseBody());
            } else if (httpget.getStatusCode() == HttpStatus.NotFound) {
                return null;
            } else {
                throw new MetaClientException(httpget.getResponseBodyAsString());
            }

        } catch (final HttpException e) {
            logger.error(e.getMessage(), e);
            throw new MetaClientException(e);
        } catch (final IOException e) {
            logger.error(e.getMessage(), e);
            throw new MetaClientException(e);
        } finally {
            httpget.releaseConnection();
            final long duration = System.currentTimeMillis() - start;
            if (duration > 200) {
                MetaStatLog.addStatValue2(null, StatConstants.GET_TIME_STAT, fetchRequest.getTopic(), duration);
            }
            if (!success) {
                MetaStatLog.addStat(null, StatConstants.GET_FAILED_STAT, fetchRequest.getTopic());
            }
        }
    }

    private void checkState() {
        if (this.fetchManager.isShutdown()) {
            throw new IllegalStateException("Consumer has been shutdown");
        }
    }

    @Override
    public MessageListener getMessageListener(final String topic) {
        try {
            return this.subscribeInfoManager.getMessageListener(topic, this.config.getGroup());
        } catch (final MetaClientException e) {
            throw new RuntimeException(e);
        }
    }

    @Override
    public void appendCouldNotProcessMessage(final Message message) throws IOException {
        // 
        this.recoverStorageManager.append(this.config.getGroup(), message);

    }

    @Override
    public long offset(final FetchRequest fetchRequest) throws MetaClientException {
        final long start = System.currentTimeMillis();
        final boolean success = false;

        // URI/offset/{brokerID}?topic&partition&group, {brokerID}HTTP
        // LBHTTP clientBroker server
        final String uri = "/offset/" + fetchRequest.getPartitionObject().getBrokerId() + "?topic="
                + fetchRequest.getTopic() + "&partition=" + fetchRequest.getPartition() + "&offset="
                + fetchRequest.getOffset();
        final GetMethod httpget = new GetMethod(uri);
        try {
            this.httpclient.executeMethod(httpget);
            if (httpget.getStatusCode() == HttpStatus.Success) {
                return Long.parseLong(httpget.getResponseBodyAsString());
            } else {
                throw new MetaClientException(httpget.getResponseBodyAsString());
            }
        } catch (final HttpException e) {
            logger.error(e.getMessage(), e);
            throw new MetaClientException(e);
        } catch (final IOException e) {
            logger.error(e.getMessage(), e);
            throw new MetaClientException(e);
        } finally {
            httpget.releaseConnection();
            final long duration = System.currentTimeMillis() - start;
            if (duration > 200) {
                MetaStatLog.addStatValue2(null, StatConstants.GET_TIME_STAT, fetchRequest.getTopic(), duration);
            }
            if (!success) {
                MetaStatLog.addStat(null, StatConstants.GET_FAILED_STAT, fetchRequest.getTopic());
            }
        }
    }

}