qunar.tc.qmq.consumer.pull.AckSendQueue.java Source code

Java tutorial

Introduction

Here is the source code for qunar.tc.qmq.consumer.pull.AckSendQueue.java

Source

/*
 * Copyright 2018 Qunar, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package qunar.tc.qmq.consumer.pull;

import com.google.common.base.Supplier;
import com.google.common.util.concurrent.RateLimiter;
import io.netty.util.Timeout;
import io.netty.util.TimerTask;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import qunar.tc.qmq.base.BaseMessage;
import qunar.tc.qmq.broker.BrokerGroupInfo;
import qunar.tc.qmq.broker.BrokerService;
import qunar.tc.qmq.common.ClientType;
import qunar.tc.qmq.common.TimerUtil;
import qunar.tc.qmq.config.PullSubjectsConfig;
import qunar.tc.qmq.metrics.Metrics;
import qunar.tc.qmq.metrics.QmqCounter;
import qunar.tc.qmq.metrics.QmqMeter;
import qunar.tc.qmq.utils.RetrySubjectUtils;

import java.util.List;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.ReentrantLock;

import static qunar.tc.qmq.metrics.MetricsConstants.SUBJECT_GROUP_ARRAY;

/**
 * @author yiqun.fan create on 17-8-23.
 */
class AckSendQueue implements TimerTask {
    private static final Logger LOGGER = LoggerFactory.getLogger(AckSendQueue.class);
    private static final long DEFAULT_PULL_OFFSET = -1;
    private static final int ACK_INTERVAL_SECONDS = 10;
    private static final long ACK_TRY_SEND_TIMEOUT_MILLIS = 1000;

    private static final int DESTROY_CHECK_WAIT_MILLIS = 50;

    private final String brokerGroupName;
    private final String subject;
    private final String group;

    private final AckService ackService;

    private final String retrySubject;
    private final String deadRetrySubject;

    private final AtomicReference<Integer> pullBatchSize;

    private final ReentrantLock updateLock = new ReentrantLock();

    private final AtomicLong minPullOffset = new AtomicLong(DEFAULT_PULL_OFFSET);
    private final AtomicLong maxPullOffset = new AtomicLong(DEFAULT_PULL_OFFSET);

    private final AtomicInteger toSendNum = new AtomicInteger(0);
    private QmqMeter sendNumQps;
    private QmqCounter appendErrorCount;
    private QmqCounter sendErrorCount;
    private QmqCounter sendFailCount;
    private QmqCounter deadQueueCount;

    private final LinkedBlockingQueue<AckSendEntry> sendEntryQueue = new LinkedBlockingQueue<>();
    private final ReentrantLock sendLock = new ReentrantLock();
    private final AtomicBoolean inSending = new AtomicBoolean(false);

    private final BrokerService brokerService;
    private final SendMessageBack sendMessageBack;
    private final boolean isBroadcast;

    private final RateLimiter ackSendFailLogLimit = RateLimiter.create(0.5);

    private volatile AckEntry head = null;
    private volatile AckEntry tail = null;
    private volatile AckEntry beginScanPosition = null;

    private volatile long lastAppendOffset = -1;
    private volatile long lastSendOkOffset = -1;

    AckSendQueue(String brokerGroupName, String subject, String group, AckService ackService,
            BrokerService brokerService, SendMessageBack sendMessageBack, boolean isBroadcast) {
        this.brokerGroupName = brokerGroupName;
        this.subject = subject;
        this.group = group;
        this.ackService = ackService;
        this.brokerService = brokerService;
        this.sendMessageBack = sendMessageBack;
        this.isBroadcast = isBroadcast;

        String realSubject = RetrySubjectUtils.getRealSubject(subject);
        this.retrySubject = RetrySubjectUtils.buildRetrySubject(realSubject, group);
        this.deadRetrySubject = RetrySubjectUtils.buildDeadRetrySubject(realSubject, group);
        this.pullBatchSize = PullSubjectsConfig.get().getPullBatchSize(realSubject);
    }

    void append(final List<AckEntry> batch) {
        if (batch == null || batch.isEmpty())
            return;

        updateLock.lock();
        try {
            if (lastAppendOffset != -1 && lastAppendOffset + 1 != batch.get(0).pullOffset()) {
                LOGGER.warn("{}/{} append ack entry not continous. last: {}, new: {}", subject, group,
                        lastAppendOffset, batch.get(0).pullOffset());
                appendErrorCount.inc();
            }

            if (head == null) {
                beginScanPosition = head = batch.get(0);
                minPullOffset.set(head.pullOffset());
            }

            if (tail != null) {
                tail.setNext(batch.get(0));
            }

            tail = batch.get(batch.size() - 1);
            lastAppendOffset = tail.pullOffset();
            maxPullOffset.set(tail.pullOffset());
            toSendNum.getAndAdd(batch.size());
        } finally {
            updateLock.unlock();
        }
    }

    void sendBackAndCompleteNack(final int nextRetryCount, final BaseMessage message, final AckEntry ackEntry) {
        final String sendSubject = nextRetryCount > message.getMaxRetryNum() ? deadRetrySubject : retrySubject;
        if (deadRetrySubject.equals(sendSubject)) {
            deadQueueCount.inc();
            LOGGER.warn("process message retry num {} >= {}, and dead retry. subject={}, group={}, msgId={}",
                    nextRetryCount - 1, message.getMaxRetryNum(), subject, group, message.getMessageId());
        }
        message.setSubject(sendSubject);
        sendMessageBack.sendBackAndCompleteNack(nextRetryCount, message, ackEntry);
    }

    void ackCompleted(AckEntry current) {
        if (current == null)
            return;

        updateLock.lock();
        try {
            if (beginScanPosition == null || beginScanPosition.pullOffset() != current.pullOffset()) {
                return;
            }

            AckEntry end = scanCompleted(current);
            beginScanPosition = end.next();
            if (allowSendAck(end)) {
                final AckSendEntry sendEntry = new AckSendEntry(head, end, isBroadcast);
                head = beginScanPosition;
                if (head == null) {
                    tail = null;
                }
                sendEntryQueue.offer(sendEntry);
            } else {
                return;
            }
        } finally {
            updateLock.unlock();
        }
        sendAck();
    }

    private boolean allowSendAck(AckEntry needAck) {
        return needAck.next() == null || needAck.pullOffset() - head.pullOffset() >= pullBatchSize.get() - 1;
    }

    private AckEntry scanCompleted(AckEntry begin) {
        AckEntry needAck = begin;
        while (needAck.next() != null && needAck.next().isDone()) {
            needAck = needAck.next();
        }
        return needAck;
    }

    boolean trySendAck(long timeout) {
        if (!tryLock(timeout))
            return false;

        try {
            if (head == null || !head.isDone()) {
                return sendAck();
            }

            AckEntry end = scanCompleted(head);

            final AckSendEntry sendEntry = new AckSendEntry(head, end, isBroadcast);
            head = beginScanPosition = end.next();

            if (head == null) {
                tail = null;
            }
            sendEntryQueue.offer(sendEntry);
        } finally {
            updateLock.unlock();
        }
        return sendAck();
    }

    private boolean tryLock(long timeout) {
        try {
            if (!updateLock.tryLock(timeout, TimeUnit.MILLISECONDS)) {
                return false;
            }
        } catch (InterruptedException e) {
            return false;
        }
        return true;
    }

    private boolean sendAck() {
        AckSendEntry sendEntry;

        if (inSending.get())
            return false;
        sendLock.lock();
        try {
            if (inSending.get() || sendEntryQueue.isEmpty())
                return false;
            sendEntry = sendEntryQueue.peek();
            if (sendEntry != null) {
                inSending.set(true);
            } else {
                sendEntryQueue.poll();
                LOGGER.error("sendEntry is null");
                return false;
            }
        } finally {
            sendLock.unlock();
        }

        doSendAck(sendEntry);
        return true;
    }

    private void doSendAck(final AckSendEntry sendEntry) {
        BrokerGroupInfo brokerGroup = getBrokerGroup();
        if (brokerGroup == null) {
            LOGGER.debug("lost broker group: {}. subject={}, consumeGroup={}", brokerGroupName, subject, group);
            inSending.set(false);
            return;
        }

        ackService.sendAck(brokerGroup, subject, group, sendEntry, new AckService.SendAckCallback() {
            @Override
            public void success() {
                if (lastSendOkOffset != -1 && lastSendOkOffset + 1 != sendEntry.getPullOffsetBegin()) {
                    LOGGER.warn("{}/{} ack send not continous. last={}, send={}", subject, group, lastSendOkOffset,
                            sendEntry);
                    sendErrorCount.inc();
                }
                lastSendOkOffset = sendEntry.getPullOffsetLast();

                minPullOffset.set(sendEntry.getPullOffsetLast() + 1);
                final int sendNum = (int) (sendEntry.getPullOffsetLast() - sendEntry.getPullOffsetBegin()) + 1;
                toSendNum.getAndAdd(-sendNum);
                sendNumQps.mark(sendNum);

                AckSendEntry head = sendEntryQueue.peek();
                if (head == null || head.getPullOffsetBegin() != sendEntry.getPullOffsetBegin()) {
                    LOGGER.error("ack send error: {}, {}", sendEntry, head);
                    sendErrorCount.inc();
                } else {
                    LOGGER.debug("AckSendRet ok [{}, {}]", sendEntry.getPullOffsetBegin(),
                            sendEntry.getPullOffsetLast());
                    sendEntryQueue.poll();
                }

                inSending.set(false);
                AckSendQueue.this.sendAck();
            }

            @Override
            public void fail(Exception ex) {
                if (ackSendFailLogLimit.tryAcquire()) {
                    LOGGER.warn("send ack fail, will retry next", ex);
                }
                LOGGER.debug("AckSendRet fail [{}, {}]", sendEntry.getPullOffsetBegin(),
                        sendEntry.getPullOffsetLast());
                sendFailCount.inc();
                inSending.set(false);
            }
        });
    }

    private BrokerGroupInfo getBrokerGroup() {
        return brokerService.getClusterBySubject(ClientType.CONSUMER, subject, group)
                .getGroupByName(brokerGroupName);
    }

    AckSendInfo getAckSendInfo() {
        AckSendInfo info = new AckSendInfo();
        info.setMinPullOffset(minPullOffset.get());
        info.setMaxPullOffset(maxPullOffset.get());
        info.setToSendNum(toSendNum.get());
        return info;
    }

    void init() {
        TimerUtil.newTimeout(this, ACK_INTERVAL_SECONDS, TimeUnit.SECONDS);

        String[] values = new String[] { subject, group };
        sendNumQps = Metrics.meter("qmq_pull_ack_sendnum_qps", SUBJECT_GROUP_ARRAY, values);
        appendErrorCount = Metrics.counter("qmq_pull_ack_appenderror_count", SUBJECT_GROUP_ARRAY, values);
        sendErrorCount = Metrics.counter("qmq_pull_ack_senderror_count", SUBJECT_GROUP_ARRAY, values);
        sendFailCount = Metrics.counter("qmq_pull_ack_sendfail_count", SUBJECT_GROUP_ARRAY, values);
        deadQueueCount = Metrics.counter("qmq_deadqueue_send_count", SUBJECT_GROUP_ARRAY, values);

        Metrics.gauge("qmq_pull_ack_min_offset", SUBJECT_GROUP_ARRAY, values, new Supplier<Double>() {
            @Override
            public Double get() {
                return (double) minPullOffset.get();
            }
        });
        Metrics.gauge("qmq_pull_ack_max_offset", SUBJECT_GROUP_ARRAY, values, new Supplier<Double>() {
            @Override
            public Double get() {
                return (double) maxPullOffset.get();
            }
        });
        Metrics.gauge("qmq_pull_ack_tosendnum", SUBJECT_GROUP_ARRAY, values, new Supplier<Double>() {
            @Override
            public Double get() {
                return (double) toSendNum.get();
            }
        });
    }

    public String getSubject() {
        return subject;
    }

    public String getGroup() {
        return group;
    }

    private static final AckSendEntry EMPTY_ACK = new AckSendEntry();

    private static final AckService.SendAckCallback EMPTY_ACK_CALLBACK = new AckService.SendAckCallback() {
        @Override
        public void success() {
            LOGGER.debug("send heartbeat ok");
        }

        @Override
        public void fail(Exception ex) {
            LOGGER.error("send heartbeat fail", ex);
        }
    };

    @Override
    public void run(Timeout timeout) {
        try {
            if (!trySendAck(ACK_TRY_SEND_TIMEOUT_MILLIS)) {
                final BrokerGroupInfo brokerGroup = getBrokerGroup();
                if (brokerGroup == null) {
                    LOGGER.debug("lost broker group: {}. subject={}, consumeGroup={}", brokerGroupName, subject,
                            group);
                    return;
                }
                ackService.sendAck(brokerGroup, subject, group, EMPTY_ACK, EMPTY_ACK_CALLBACK);
            }
        } finally {
            TimerUtil.newTimeout(this, ACK_INTERVAL_SECONDS, TimeUnit.SECONDS);
        }
    }

    void destroy(long waitTime) {
        while (waitTime > 0 && toSendNum.get() > 0) {
            try {
                Thread.sleep(DESTROY_CHECK_WAIT_MILLIS);
            } catch (Exception e) {
                break;
            }
            waitTime -= DESTROY_CHECK_WAIT_MILLIS;
        }
    }
}