Example usage for io.netty.buffer ByteBuf release

Introduction

In this page you can find the example usage for io.netty.buffer ByteBuf release.

Prototype

boolean release();

Source Link

Document

Decreases the reference count by 1 and deallocates this object if the reference count reaches at 0 .

Usage

From source file:com.yahoo.pulsar.broker.admin.PersistentTopics.java

License:Apache License

@GET
@Path("/{property}/{cluster}/{namespace}/{destination}/subscription/{subName}/position/{messagePosition}")
@ApiOperation(value = "Peek nth message on a topic subscription.")
@ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"),
        @ApiResponse(code = 404, message = "Topic, subscription or the message position does not exist") })
public Response peekNthMessage(@PathParam("property") String property, @PathParam("cluster") String cluster,
        @PathParam("namespace") String namespace, @PathParam("destination") @Encoded String destination,
        @PathParam("subName") String subName, @PathParam("messagePosition") int messagePosition,
        @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) {
    destination = decode(destination);/* w w  w .j a v a  2s.  co m*/
    DestinationName dn = DestinationName.get(domain(), property, cluster, namespace, destination);
    PartitionedTopicMetadata partitionMetadata = getPartitionedTopicMetadata(property, cluster, namespace,
            destination, authoritative);
    if (partitionMetadata.partitions > 0) {
        throw new RestException(Status.METHOD_NOT_ALLOWED,
                "Peek messages on a partitioned topic is not allowed");
    }
    validateAdminOperationOnDestination(dn, authoritative);
    PersistentTopic topic = getTopicReference(dn);
    PersistentReplicator repl = null;
    PersistentSubscription sub = null;
    Entry entry = null;
    if (subName.startsWith(topic.replicatorPrefix)) {
        repl = getReplicatorReference(subName, topic);
    } else {
        sub = getSubscriptionReference(subName, topic);
    }
    try {
        if (subName.startsWith(topic.replicatorPrefix)) {
            entry = repl.peekNthMessage(messagePosition).get();
        } else {
            entry = sub.peekNthMessage(messagePosition).get();
        }
        checkNotNull(entry);
        PositionImpl pos = (PositionImpl) entry.getPosition();
        ByteBuf metadataAndPayload = entry.getDataBuffer();

        // moves the readerIndex to the payload
        MessageMetadata metadata = Commands.parseMessageMetadata(metadataAndPayload);

        ResponseBuilder responseBuilder = Response.ok();
        responseBuilder.header("X-Pulsar-Message-ID", pos.toString());
        for (KeyValue keyValue : metadata.getPropertiesList()) {
            responseBuilder.header("X-Pulsar-PROPERTY-" + keyValue.getKey(), keyValue.getValue());
        }
        if (metadata.hasPublishTime()) {
            responseBuilder.header("X-Pulsar-publish-time",
                    DATE_FORMAT.format(Instant.ofEpochMilli(metadata.getPublishTime())));
        }

        // Decode if needed
        CompressionCodec codec = CompressionCodecProvider.getCompressionCodec(metadata.getCompression());
        ByteBuf uncompressedPayload = codec.decode(metadataAndPayload, metadata.getUncompressedSize());

        // Copy into a heap buffer for output stream compatibility
        ByteBuf data = PooledByteBufAllocator.DEFAULT.heapBuffer(uncompressedPayload.readableBytes(),
                uncompressedPayload.readableBytes());
        data.writeBytes(uncompressedPayload);
        uncompressedPayload.release();

        StreamingOutput stream = new StreamingOutput() {

            @Override
            public void write(OutputStream output) throws IOException, WebApplicationException {
                output.write(data.array(), data.arrayOffset(), data.readableBytes());
                data.release();
            }
        };

        return responseBuilder.entity(stream).build();
    } catch (NullPointerException npe) {
        throw new RestException(Status.NOT_FOUND, "Message not found");
    } catch (Exception exception) {
        log.error("[{}] Failed to get message at position {} from {} {}", clientAppId(), messagePosition, dn,
                subName, exception);
        throw new RestException(exception);
    } finally {
        if (entry != null) {
            entry.release();
        }
    }
}

---

From source file:com.yahoo.pulsar.broker.service.PersistentMessageFinderTest.java

License:Apache License

public static byte[] createMessageWrittenToLedger(String msg) throws Exception {
    PulsarApi.MessageMetadata.Builder messageMetadataBuilder = PulsarApi.MessageMetadata.newBuilder();
    messageMetadataBuilder.setPublishTime(System.currentTimeMillis());
    messageMetadataBuilder.setProducerName("createMessageWrittenToLedger");
    messageMetadataBuilder.setSequenceId(1);
    PulsarApi.MessageMetadata messageMetadata = messageMetadataBuilder.build();
    ByteBuf data = UnpooledByteBufAllocator.DEFAULT.heapBuffer().writeBytes(msg.getBytes());

    int msgMetadataSize = messageMetadata.getSerializedSize();
    int payloadSize = data.readableBytes();
    int totalSize = 4 + msgMetadataSize + payloadSize;

    ByteBuf headers = PooledByteBufAllocator.DEFAULT.heapBuffer(totalSize, totalSize);
    ByteBufCodedOutputStream outStream = ByteBufCodedOutputStream.get(headers);
    headers.writeInt(msgMetadataSize);//from w  w w. j a v  a  2  s  .c  o m
    messageMetadata.writeTo(outStream);
    ByteBuf headersAndPayload = DoubleByteBuf.get(headers, data);
    byte[] byteMessage = headersAndPayload.nioBuffer().array();
    headersAndPayload.release();
    return byteMessage;
}

---

From source file:com.yahoo.pulsar.broker.service.ServerCnxTest.java

License:Apache License

@Test(timeOut = 30000)
public void testSendCommand() throws Exception {
    resetChannel();//from   ww w. ja  v  a2 s . co  m
    setChannelConnected();

    ByteBuf clientCommand = Commands.newProducer(successTopicName, 1 /* producer id */, 1 /* request id */,
            "prod-name");
    channel.writeInbound(clientCommand);
    assertTrue(getResponse() instanceof CommandProducerSuccess);

    // test SEND success
    MessageMetadata messageMetadata = MessageMetadata.newBuilder().setPublishTime(System.currentTimeMillis())
            .setProducerName("prod-name").setSequenceId(0).build();
    ByteBuf data = Unpooled.buffer(1024);

    clientCommand = Commands.newSend(1, 0, 1, ChecksumType.None, messageMetadata, data);
    channel.writeInbound(Unpooled.copiedBuffer(clientCommand));
    clientCommand.release();

    assertTrue(getResponse() instanceof CommandSendReceipt);
    channel.finish();
}

---

From source file:com.yahoo.pulsar.broker.stats.BookieClientsStatsGeneratorTest.java

License:Apache License

@Test
public void testJvmDirectMemoryUsedMetric() throws Exception {
    PooledByteBufAllocator allocator = new PooledByteBufAllocator( //
            true, // preferDirect
            0, // nHeapArenas,
            1, // nDirectArena
            8192, // pageSize
            11, // maxOrder
            64, // tinyCacheSize
            32, // smallCacheSize
            8 // normalCacheSize
    );/*from ww  w . j av a 2s  .com*/
    int allocateMemory = 17777216;
    long directMemory1 = JvmMetrics.getJvmDirectMemoryUsed();
    ByteBuf buf2 = allocator.directBuffer(allocateMemory, allocateMemory);
    long directMemory2 = JvmMetrics.getJvmDirectMemoryUsed();
    assertEquals(directMemory2, directMemory1 + allocateMemory);
    ByteBuf buf3 = allocator.directBuffer(allocateMemory, allocateMemory);
    long directMemory3 = JvmMetrics.getJvmDirectMemoryUsed();
    assertEquals(directMemory3, directMemory2 + allocateMemory);
    buf3.release();
    directMemory3 = JvmMetrics.getJvmDirectMemoryUsed();
    assertEquals(directMemory3, directMemory2);
    buf2.release();
    directMemory2 = JvmMetrics.getJvmDirectMemoryUsed();
    assertEquals(directMemory2, directMemory1);

}

---

From source file:com.yahoo.pulsar.checksum.utils.Crc32cChecksumTest.java

License:Apache License

@Test
public void testCrc32c() {

    ByteBuf payload = Unpooled.wrappedBuffer(ByteBuffer.wrap(inputBytes, 0, inputBytes.length));
    int checksum = Crc32cChecksum.computeChecksum(payload);
    payload.release();
    Assert.assertEquals(expectedChecksum, checksum);
}

---

From source file:com.yahoo.pulsar.checksum.utils.Crc32cChecksumTest.java

License:Apache License

@Test
public void testCrc32cDirectMemory() {
    ByteBuffer bytes = ByteBuffer.allocateDirect(4);
    bytes.put(inputBytes);//from   w w  w .j ava2s.  co  m
    bytes.flip();
    ByteBuf payload = Unpooled.wrappedBuffer(bytes);
    int checksum = Crc32cChecksum.computeChecksum(payload);
    payload.release();
    Assert.assertEquals(expectedChecksum, checksum);
}

---

From source file:com.yahoo.pulsar.client.impl.ConsumerImpl.java

License:Apache License

void messageReceived(MessageIdData messageId, ByteBuf headersAndPayload, ClientCnx cnx) {
    if (log.isDebugEnabled()) {
        log.debug("[{}][{}] Received message: {}/{}", topic, subscription, messageId.getLedgerId(),
                messageId.getEntryId());
    }//from   w ww.  j  a va2s. c o  m

    MessageMetadata msgMetadata = null;
    ByteBuf payload = headersAndPayload;

    if (!verifyChecksum(headersAndPayload, messageId)) {
        // discard message with checksum error
        discardCorruptedMessage(messageId, cnx, ValidationError.ChecksumMismatch);
        return;
    }

    try {
        msgMetadata = Commands.parseMessageMetadata(payload);
    } catch (Throwable t) {
        discardCorruptedMessage(messageId, cnx, ValidationError.ChecksumMismatch);
        return;
    }

    ByteBuf uncompressedPayload = uncompressPayloadIfNeeded(messageId, msgMetadata, payload, cnx);
    if (uncompressedPayload == null) {
        // Message was discarded on decompression error
        return;
    }

    final int numMessages = msgMetadata.getNumMessagesInBatch();

    if (numMessages == 1 && !msgMetadata.hasNumMessagesInBatch()) {
        final MessageImpl message = new MessageImpl(messageId, msgMetadata, uncompressedPayload,
                getPartitionIndex(), cnx);
        uncompressedPayload.release();
        msgMetadata.recycle();

        try {
            lock.readLock().lock();
            // Enqueue the message so that it can be retrieved when application calls receive()
            // if the conf.getReceiverQueueSize() is 0 then discard message if no one is waiting for it.
            // if asyncReceive is waiting then notify callback without adding to incomingMessages queue
            unAckedMessageTracker.add((MessageIdImpl) message.getMessageId());
            boolean asyncReceivedWaiting = !pendingReceives.isEmpty();
            if ((conf.getReceiverQueueSize() != 0 || waitingOnReceiveForZeroQueueSize)
                    && !asyncReceivedWaiting) {
                incomingMessages.add(message);
            }
            if (asyncReceivedWaiting) {
                notifyPendingReceivedCallback(message, null);
            }
        } finally {
            lock.readLock().unlock();
        }
    } else {
        if (conf.getReceiverQueueSize() == 0) {
            log.warn(
                    "Closing consumer [{}]-[{}] due to unsupported received batch-message with zero receiver queue size",
                    subscription, consumerName);
            // close connection
            closeAsync().handle((ok, e) -> {
                // notify callback with failure result
                notifyPendingReceivedCallback(null,
                        new PulsarClientException.InvalidMessageException(
                                format("Unsupported Batch message with 0 size receiver queue for [%s]-[%s] ",
                                        subscription, consumerName)));
                return null;
            });
        } else {
            // handle batch message enqueuing; uncompressed payload has all messages in batch
            receiveIndividualMessagesFromBatch(msgMetadata, uncompressedPayload, messageId, cnx);
        }
        uncompressedPayload.release();
        msgMetadata.recycle();
    }

    if (listener != null) {
        // Trigger the notification on the message listener in a separate thread to avoid blocking the networking
        // thread while the message processing happens
        listenerExecutor.execute(() -> {
            for (int i = 0; i < numMessages; i++) {
                try {
                    Message msg = internalReceive(0, TimeUnit.MILLISECONDS);
                    // complete the callback-loop in case queue is cleared up
                    if (msg == null) {
                        if (log.isDebugEnabled()) {
                            log.debug("[{}] [{}] Message has been cleared from the queue", topic, subscription);
                        }
                        break;
                    }
                    try {
                        if (log.isDebugEnabled()) {
                            log.debug("[{}][{}] Calling message listener for message {}", topic, subscription,
                                    msg.getMessageId());
                        }
                        listener.received(ConsumerImpl.this, msg);
                    } catch (Throwable t) {
                        log.error("[{}][{}] Message listener error in processing message: {}", topic,
                                subscription, msg.getMessageId(), t);
                    }

                } catch (PulsarClientException e) {
                    log.warn("[{}] [{}] Failed to dequeue the message for listener", topic, subscription, e);
                    return;
                }
            }
        });
    }
}

---

From source file:com.yahoo.pulsar.client.impl.ConsumerImpl.java

License:Apache License

void receiveIndividualMessagesFromBatch(MessageMetadata msgMetadata, ByteBuf uncompressedPayload,
        MessageIdData messageId, ClientCnx cnx) {
    int batchSize = msgMetadata.getNumMessagesInBatch();

    // create ack tracker for entry aka batch
    BitSet bitSet = new BitSet(batchSize);
    MessageIdImpl batchMessage = new MessageIdImpl(messageId.getLedgerId(), messageId.getEntryId(),
            getPartitionIndex());/*  w w w .  j a  v  a2 s.c o  m*/
    bitSet.set(0, batchSize);
    if (log.isDebugEnabled()) {
        log.debug("[{}] [{}] added bit set for message {}, cardinality {}, length {}", subscription,
                consumerName, batchMessage, bitSet.cardinality(), bitSet.length());
    }
    batchMessageAckTracker.put(batchMessage, bitSet);
    unAckedMessageTracker.add(batchMessage);
    try {
        for (int i = 0; i < batchSize; ++i) {
            if (log.isDebugEnabled()) {
                log.debug("[{}] [{}] processing message num - {} in batch", subscription, consumerName, i);
            }
            PulsarApi.SingleMessageMetadata.Builder singleMessageMetadataBuilder = PulsarApi.SingleMessageMetadata
                    .newBuilder();
            ByteBuf singleMessagePayload = Commands.deSerializeSingleMessageInBatch(uncompressedPayload,
                    singleMessageMetadataBuilder, i, batchSize);
            BatchMessageIdImpl batchMessageIdImpl = new BatchMessageIdImpl(messageId.getLedgerId(),
                    messageId.getEntryId(), getPartitionIndex(), i);
            final MessageImpl message = new MessageImpl(batchMessageIdImpl, msgMetadata,
                    singleMessageMetadataBuilder.build(), singleMessagePayload, cnx);
            lock.readLock().lock();
            if (pendingReceives.isEmpty()) {
                incomingMessages.add(message);
            } else {
                notifyPendingReceivedCallback(message, null);
            }
            lock.readLock().unlock();
            singleMessagePayload.release();
            singleMessageMetadataBuilder.recycle();
        }
    } catch (IOException e) {
        //
        log.warn("[{}] [{}] unable to obtain message in batch", subscription, consumerName);
        batchMessageAckTracker.remove(batchMessage);
        discardCorruptedMessage(messageId, cnx, ValidationError.BatchDeSerializeError);
    }
    if (log.isDebugEnabled()) {
        log.debug("[{}] [{}] enqueued messages in batch. queue size - {}, available queue size - {}",
                subscription, consumerName, incomingMessages.size(), incomingMessages.remainingCapacity());
    }
}

---

From source file:com.yahoo.pulsar.client.impl.ProducerImpl.java

License:Apache License

public void sendAsync(Message message, SendCallback callback) {
    checkArgument(message instanceof MessageImpl);

    if (!isValidProducerState(callback)) {
        return;//  w w w.  ja  v a  2s.c  om
    }

    if (!canEnqueueRequest(callback)) {
        return;
    }

    MessageImpl msg = (MessageImpl) message;
    MessageMetadata.Builder msgMetadata = msg.getMessageBuilder();
    ByteBuf payload = msg.getDataBuffer();

    // If compression is enabled, we are compressing, otherwise it will simply use the same buffer
    int uncompressedSize = payload.readableBytes();
    ByteBuf compressedPayload = payload;
    // batch will be compressed when closed
    if (!isBatchMessagingEnabled()) {
        compressedPayload = compressor.encode(payload);
        payload.release();
    }

    if (!msg.isReplicated() && msgMetadata.hasProducerName()) {
        callback.sendComplete(
                new PulsarClientException.InvalidMessageException("Cannot re-use the same message"));
        compressedPayload.release();
        return;
    }

    try {
        synchronized (this) {
            long sequenceId = msgIdGeneratorUpdater.getAndIncrement(this);
            if (!msgMetadata.hasPublishTime()) {
                msgMetadata.setPublishTime(System.currentTimeMillis());

                checkArgument(!msgMetadata.hasProducerName());
                checkArgument(!msgMetadata.hasSequenceId());

                msgMetadata.setProducerName(producerName);
                msgMetadata.setSequenceId(sequenceId);
                if (conf.getCompressionType() != CompressionType.NONE) {
                    msgMetadata.setCompression(convertCompressionType(conf.getCompressionType()));
                    msgMetadata.setUncompressedSize(uncompressedSize);
                }
            }

            if (isBatchMessagingEnabled()) {
                // handle boundary cases where message being added would exceed
                // batch size and/or max message size
                if (batchMessageContainer.hasSpaceInBatch(msg)) {
                    batchMessageContainer.add(msg, callback);
                    payload.release();
                    if (batchMessageContainer.numMessagesInBatch == maxNumMessagesInBatch
                            || batchMessageContainer.currentBatchSizeBytes >= BatchMessageContainer.MAX_MESSAGE_BATCH_SIZE_BYTES) {
                        batchMessageAndSend();
                    }
                } else {
                    doBatchSendAndAdd(msg, callback, payload);
                }
            } else {
                ByteBuf cmd = sendMessage(producerId, sequenceId, 1, msgMetadata.build(), compressedPayload);
                msgMetadata.recycle();

                final OpSendMsg op = OpSendMsg.create(msg, cmd, sequenceId, callback);
                op.setNumMessagesInBatch(1);
                op.setBatchSizeByte(payload.readableBytes());
                pendingMessages.put(op);

                if (isConnected()) {
                    // If we do have a connection, the message is sent immediately, otherwise we'll try again once a
                    // new
                    // connection is established
                    cmd.retain();
                    cnx().ctx().channel().eventLoop().execute(WriteInEventLoopCallback.create(this, cnx(), op));
                    stats.updateNumMsgsSent(op.numMessagesInBatch, op.batchSizeByte);
                } else {
                    if (log.isDebugEnabled()) {
                        log.debug("[{}] [{}] Connection is not ready -- sequenceId {}", topic, producerName,
                                sequenceId);
                    }
                }
            }
        }
    } catch (InterruptedException ie) {
        Thread.currentThread().interrupt();
        semaphore.release();
        callback.sendComplete(new PulsarClientException(ie));
    } catch (Throwable t) {
        semaphore.release();
        callback.sendComplete(new PulsarClientException(t));
    }
}

---

From source file:com.yahoo.pulsar.client.impl.ProducerImpl.java

License:Apache License

private void doBatchSendAndAdd(MessageImpl msg, SendCallback callback, ByteBuf payload) {
    if (log.isDebugEnabled()) {
        log.debug("[{}] [{}] Closing out batch to accomodate large message with size {}", topic, producerName,
                msg.getDataBuffer().readableBytes());
    }//  w  w w  .j  a  v a 2  s .c o  m
    batchMessageAndSend();
    batchMessageContainer.add(msg, callback);
    payload.release();
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »