Java tutorial
/** * Copyright 2016 Yahoo Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.yahoo.pulsar.client.impl; import static com.google.common.base.Preconditions.checkArgument; import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.collect.Lists; import com.yahoo.pulsar.common.api.PulsarDecoder; import com.yahoo.pulsar.common.api.Commands; import com.yahoo.pulsar.common.api.proto.PulsarApi; import com.yahoo.pulsar.common.compression.CompressionCodec; import com.yahoo.pulsar.common.compression.CompressionCodecProvider; import io.netty.buffer.ByteBuf; import io.netty.buffer.PooledByteBufAllocator; /** * container for individual messages being published until they are batched and sent to broker */ class BatchMessageContainer { private SendCallback previousCallback = null; private final PulsarApi.CompressionType compressionType; private final CompressionCodec compressor; private final String topicName; private final String producerName; final int maxNumMessagesInBatch; PulsarApi.MessageMetadata.Builder messageMetadata = PulsarApi.MessageMetadata.newBuilder(); int numMessagesInBatch = 0; long currentBatchSizeBytes = 0; // sequence id for this batch which will be persisted as a single entry by broker long sequenceId = -1; ByteBuf batchedMessageMetadataAndPayload; List<MessageImpl> messages = Lists.newArrayList(); // keep track of callbacks for individual messages being published in a batch SendCallback firstCallback; protected static final long MAX_MESSAGE_BATCH_SIZE_BYTES = 128 * 1024; BatchMessageContainer(int maxNumMessagesInBatch, PulsarApi.CompressionType compressionType, String topicName, String producerName) { this.maxNumMessagesInBatch = maxNumMessagesInBatch; this.compressionType = compressionType; this.compressor = CompressionCodecProvider.getCompressionCodec(compressionType); this.topicName = topicName; this.producerName = producerName; } boolean hasSpaceInBatch(MessageImpl msg) { int messageSize = msg.getDataBuffer().readableBytes(); return ((messageSize + currentBatchSizeBytes) <= MAX_MESSAGE_BATCH_SIZE_BYTES && numMessagesInBatch < maxNumMessagesInBatch); } void add(MessageImpl msg, SendCallback callback) { if (log.isDebugEnabled()) { log.debug("[{}] [{}] add message to batch, num messages in batch so far {}", topicName, producerName, numMessagesInBatch); } if (++numMessagesInBatch == 1) { // some properties are common amongst the different messages in the batch, hence we just pick it up from // the first message sequenceId = Commands.initBatchMessageMetadata(messageMetadata, msg.getMessageBuilder()); this.firstCallback = callback; batchedMessageMetadataAndPayload = PooledByteBufAllocator.DEFAULT .buffer((int) MAX_MESSAGE_BATCH_SIZE_BYTES, (int) (PulsarDecoder.MaxMessageSize)); } if (previousCallback != null) { previousCallback.addCallback(callback); } previousCallback = callback; currentBatchSizeBytes += msg.getDataBuffer().readableBytes(); PulsarApi.MessageMetadata.Builder msgBuilder = msg.getMessageBuilder(); batchedMessageMetadataAndPayload = Commands.serializeSingleMessageInBatchWithPayload(msgBuilder, msg.getDataBuffer(), batchedMessageMetadataAndPayload); messages.add(msg); msgBuilder.recycle(); } ByteBuf getCompressedBatchMetadataAndPayload() { int uncompressedSize = batchedMessageMetadataAndPayload.readableBytes(); ByteBuf compressedPayload = compressor.encode(batchedMessageMetadataAndPayload); batchedMessageMetadataAndPayload.release(); if (compressionType != PulsarApi.CompressionType.NONE) { messageMetadata.setCompression(compressionType); messageMetadata.setUncompressedSize(uncompressedSize); } return compressedPayload; } PulsarApi.MessageMetadata setBatchAndBuild() { messageMetadata.setNumMessagesInBatch(numMessagesInBatch); if (log.isDebugEnabled()) { log.debug("[{}] [{}] num messages in batch being closed are {}", topicName, producerName, numMessagesInBatch); } return messageMetadata.build(); } ByteBuf getBatchedSingleMessageMetadataAndPayload() { return batchedMessageMetadataAndPayload; } void clear() { messages = Lists.newArrayList(); firstCallback = null; previousCallback = null; messageMetadata.clear(); numMessagesInBatch = 0; currentBatchSizeBytes = 0; sequenceId = -1; } boolean isEmpty() { return messages.isEmpty(); } private static final Logger log = LoggerFactory.getLogger(BatchMessageContainer.class); }