Example usage for io.netty.buffer ByteBuf retain

List of usage examples for io.netty.buffer ByteBuf retain

  1. HOME
  2. Java
  3. io.netty
  4. io.netty.buffer.*
  5. ByteBuf
  6. retain

Introduction

In this page you can find the example usage for io.netty.buffer ByteBuf retain.

Prototype

@Override
    public abstract ByteBuf retain();

Source Link

Usage

From source file:org.apache.flink.runtime.io.network.netty.InboundEnvelopeDecoderTest.java

License:Apache License

@Test
public void testEncodeDecode() throws Exception {
    final EmbeddedChannel ch = new EmbeddedChannel(new OutboundEnvelopeEncoder(),
            new InboundEnvelopeDecoder(this.bufferProviderBroker));

    when(this.bufferProviderBroker.getBufferProvider(anyJobId(), anyChannelId()))
            .thenReturn(this.bufferProvider);

    when(this.bufferProvider.requestBuffer(anyInt())).thenAnswer(new Answer<Object>() {
        @Override/*from   w  ww .  ja  v a 2s  .c  o  m*/
        public Object answer(InvocationOnMock invocation) throws Throwable {
            // fulfill the buffer request
            return allocBuffer((Integer) invocation.getArguments()[0]);
        }
    });

    // --------------------------------------------------------------------

    Envelope[] envelopes = new Envelope[] { nextEnvelope(0), nextEnvelope(2), nextEnvelope(32768),
            nextEnvelope(3782, new TestEvent1(34872527)),
            nextEnvelope(88, new TestEvent1(8749653), new TestEvent1(365345)),
            nextEnvelope(0, new TestEvent2(34563456), new TestEvent1(598432), new TestEvent2(976293845)),
            nextEnvelope(23) };

    ByteBuf buf = encode(ch, envelopes);

    // 1. complete ByteBuf as input
    int refCount = buf.retain().refCnt();

    decodeAndVerify(ch, buf, envelopes);
    Assert.assertEquals(refCount - 1, buf.refCnt());

    // 2. random slices
    buf.readerIndex(0);
    ByteBuf[] slices = randomSlices(buf);

    ch.writeInbound((Object[]) slices);

    for (ByteBuf slice : slices) {
        Assert.assertEquals(1, slice.refCnt());
    }

    decodeAndVerify(ch, envelopes);

    buf.release();
}

From source file:org.apache.helix.ipc.benchmark.BenchmarkDriver.java

License:Apache License

private void startTraffic(final String remoteHost, final int remotePort) {
    if (isShutdown.getAndSet(false)) {
        System.out.println("Starting " + trafficThreads.length + " threads to generate traffic");
        for (int i = 0; i < trafficThreads.length; i++) {
            Thread t = new Thread() {
                @Override//from   w w  w .  j  ava  2 s  . c o m
                public void run() {
                    ByteBuf m = ByteBufAllocator.DEFAULT.buffer(messageBytes.length);
                    m.writeBytes(messageBytes);
                    while (!isShutdown.get()) {
                        for (int i = 0; i < numPartitions; i++) {
                            HelixMessageScope scope = new HelixMessageScope.Builder().cluster("CLUSTER")
                                    .resource("RESOURCE").partition("PARTITION_" + i)
                                    .sourceInstance(localhost + "_" + port).build();

                            Set<HelixAddress> destinations = ImmutableSet
                                    .of(new HelixAddress(scope, remoteHost + "_" + remotePort,
                                            new InetSocketAddress(remoteHost, remotePort)));

                            UUID uuid = UUID.randomUUID();

                            try {
                                for (HelixAddress destination : destinations) {
                                    m.retain();
                                    ipcService.send(destination, MESSAGE_TYPE, uuid, m);
                                }
                            } catch (Exception e) {
                                e.printStackTrace();
                            }
                        }
                    }
                }
            };
            t.start();
            trafficThreads[i] = t;
        }
        System.out.println("Started traffic to " + remoteHost + ":" + remotePort);
    }
}

From source file:org.apache.helix.ipc.HelixIPCMessageManager.java

License:Apache License

@Override
public void send(final HelixAddress destination, final int messageType, final UUID messageId,
        final ByteBuf message) {
    // State/* www  .  jav a2s. co m*/
    pendingMessages.put(messageId, true);
    retriesLeft.putIfAbsent(messageId, new AtomicInteger(maxNumRetries));
    messageBuffers.put(messageId, message);

    // Will free it when we've finally received response
    message.retain();

    // Send initial message
    baseIpcService.send(destination, messageType, messageId, message);

    // Retries
    scheduler.schedule(new Runnable() {
        @Override
        public void run() {
            Boolean isPending = pendingMessages.get(messageId);
            AtomicInteger numLeft = retriesLeft.get(messageId);
            if (numLeft != null && isPending != null && isPending) {
                if (numLeft.decrementAndGet() > 0 || maxNumRetries == -1) {
                    // n.b. will schedule another retry
                    send(destination, messageType, messageId, message);
                } else {
                    LOG.warn("Message " + messageId + " timed out after " + maxNumRetries + " retries");
                }
            }
        }
    }, messageTimeoutMillis, TimeUnit.MILLISECONDS);
}

From source file:org.apache.pulsar.broker.service.Consumer.java

License:Apache License

/**
 * Dispatch a list of entries to the consumer. <br/>
 * <b>It is also responsible to release entries data and recycle entries object.</b>
 *
 * @return a promise that can be use to track when all the data has been written into the socket
 *///from w ww. ja  v  a2s  .c  o m
public Pair<ChannelPromise, Integer> sendMessages(final List<Entry> entries) {
    final ChannelHandlerContext ctx = cnx.ctx();
    final MutablePair<ChannelPromise, Integer> sentMessages = new MutablePair<ChannelPromise, Integer>();
    final ChannelPromise writePromise = ctx.newPromise();
    sentMessages.setLeft(writePromise);
    if (entries.isEmpty()) {
        if (log.isDebugEnabled()) {
            log.debug("[{}] List of messages is empty, triggering write future immediately for consumerId {}",
                    subscription, consumerId);
        }
        writePromise.setSuccess();
        sentMessages.setRight(0);
        return sentMessages;
    }

    try {
        sentMessages.setRight(updatePermitsAndPendingAcks(entries));
    } catch (PulsarServerException pe) {
        log.warn("[{}] [{}] consumer doesn't support batch-message {}", subscription, consumerId,
                cnx.getRemoteEndpointProtocolVersion());

        subscription.markTopicWithBatchMessagePublished();
        sentMessages.setRight(0);
        // disconnect consumer: it will update dispatcher's availablePermits and resend pendingAck-messages of this
        // consumer to other consumer
        disconnect();
        return sentMessages;
    }

    ctx.channel().eventLoop().execute(() -> {
        for (int i = 0; i < entries.size(); i++) {
            Entry entry = entries.get(i);
            PositionImpl pos = (PositionImpl) entry.getPosition();
            MessageIdData.Builder messageIdBuilder = MessageIdData.newBuilder();
            MessageIdData messageId = messageIdBuilder.setLedgerId(pos.getLedgerId())
                    .setEntryId(pos.getEntryId()).build();

            ByteBuf metadataAndPayload = entry.getDataBuffer();
            // increment ref-count of data and release at the end of process: so, we can get chance to call entry.release
            metadataAndPayload.retain();
            // skip checksum by incrementing reader-index if consumer-client doesn't support checksum verification
            if (cnx.getRemoteEndpointProtocolVersion() < ProtocolVersion.v6.getNumber()) {
                readChecksum(metadataAndPayload);
            }

            if (log.isDebugEnabled()) {
                log.debug("[{}] Sending message to consumerId {}, entry id {}", subscription, consumerId,
                        pos.getEntryId());
            }

            // We only want to pass the "real" promise on the last entry written
            ChannelPromise promise = ctx.voidPromise();
            if (i == (entries.size() - 1)) {
                promise = writePromise;
            }
            ctx.write(Commands.newMessage(consumerId, messageId, metadataAndPayload), promise);
            messageId.recycle();
            messageIdBuilder.recycle();
            entry.release();
        }

        ctx.flush();
    });

    return sentMessages;
}

From source file:org.apache.pulsar.broker.service.nonpersistent.NonPersistentReplicator.java

License:Apache License

public void sendMessage(Entry entry) {
    if ((STATE_UPDATER.get(this) == State.Started) && isWritable()) {

        int length = entry.getLength();
        ByteBuf headersAndPayload = entry.getDataBuffer();
        MessageImpl msg;/*  w  w  w . j a va 2 s  .c o m*/
        try {
            msg = MessageImpl.deserialize(headersAndPayload);
        } catch (Throwable t) {
            log.error("[{}][{} -> {}] Failed to deserialize message at {} (buffer size: {}): {}", topicName,
                    localCluster, remoteCluster, entry.getPosition(), length, t.getMessage(), t);
            entry.release();
            return;
        }

        if (msg.isReplicated()) {
            // Discard messages that were already replicated into this region
            entry.release();
            msg.recycle();
            return;
        }

        if (msg.hasReplicateTo() && !msg.getReplicateTo().contains(remoteCluster)) {
            if (log.isDebugEnabled()) {
                log.debug("[{}][{} -> {}] Skipping message at {} / msg-id: {}: replicateTo {}", topicName,
                        localCluster, remoteCluster, entry.getPosition(), msg.getMessageId(),
                        msg.getReplicateTo());
            }
            entry.release();
            msg.recycle();
            return;
        }

        msgOut.recordEvent(headersAndPayload.readableBytes());

        msg.setReplicatedFrom(localCluster);

        headersAndPayload.retain();

        producer.sendAsync(msg, ProducerSendCallback.create(this, entry, msg));

    } else {
        if (log.isDebugEnabled()) {
            log.debug("[{}][{} -> {}] dropping message because replicator producer is not started/writable",
                    topicName, localCluster, remoteCluster);
        }
        msgDrop.recordEvent();
        entry.release();
    }
}

From source file:org.apache.pulsar.broker.service.persistent.PersistentReplicator.java

License:Apache License

@Override
public void readEntriesComplete(List<Entry> entries, Object ctx) {
    if (log.isDebugEnabled()) {
        log.debug("[{}][{} -> {}] Read entries complete of {} messages", topicName, localCluster, remoteCluster,
                entries.size());//from   ww  w  .j a  v  a2 s  .c o m
    }

    if (readBatchSize < MaxReadBatchSize) {
        int newReadBatchSize = Math.min(readBatchSize * 2, MaxReadBatchSize);
        if (log.isDebugEnabled()) {
            log.debug("[{}][{} -> {}] Increasing read batch size from {} to {}", topicName, localCluster,
                    remoteCluster, readBatchSize, newReadBatchSize);
        }

        readBatchSize = newReadBatchSize;
    }

    readFailureBackoff.reduceToHalf();

    boolean atLeastOneMessageSentForReplication = false;

    try {
        // This flag is set to true when we skip atleast one local message,
        // in order to skip remaining local messages.
        boolean isLocalMessageSkippedOnce = false;
        for (int i = 0; i < entries.size(); i++) {
            Entry entry = entries.get(i);
            int length = entry.getLength();
            ByteBuf headersAndPayload = entry.getDataBuffer();
            MessageImpl msg;
            try {
                msg = MessageImpl.deserialize(headersAndPayload);
            } catch (Throwable t) {
                log.error("[{}][{} -> {}] Failed to deserialize message at {} (buffer size: {}): {}", topicName,
                        localCluster, remoteCluster, entry.getPosition(), length, t.getMessage(), t);
                cursor.asyncDelete(entry.getPosition(), this, entry.getPosition());
                entry.release();
                continue;
            }

            if (msg.isReplicated()) {
                // Discard messages that were already replicated into this region
                cursor.asyncDelete(entry.getPosition(), this, entry.getPosition());
                entry.release();
                msg.recycle();
                continue;
            }

            if (msg.hasReplicateTo() && !msg.getReplicateTo().contains(remoteCluster)) {
                if (log.isDebugEnabled()) {
                    log.debug("[{}][{} -> {}] Skipping message at {} / msg-id: {}: replicateTo {}", topicName,
                            localCluster, remoteCluster, entry.getPosition(), msg.getMessageId(),
                            msg.getReplicateTo());
                }
                cursor.asyncDelete(entry.getPosition(), this, entry.getPosition());
                entry.release();
                msg.recycle();
                continue;
            }

            if (msg.isExpired(messageTTLInSeconds)) {
                msgExpired.recordEvent(0 /* no value stat */);
                if (log.isDebugEnabled()) {
                    log.debug("[{}][{} -> {}] Discarding expired message at {} / msg-id: {}", topicName,
                            localCluster, remoteCluster, entry.getPosition(), msg.getMessageId());
                }
                cursor.asyncDelete(entry.getPosition(), this, entry.getPosition());
                entry.release();
                msg.recycle();
                continue;
            }

            if (STATE_UPDATER.get(this) != State.Started || isLocalMessageSkippedOnce) {
                // The producer is not ready yet after having stopped/restarted. Drop the message because it will
                // recovered when the producer is ready
                if (log.isDebugEnabled()) {
                    log.debug("[{}][{} -> {}] Dropping read message at {} because producer is not ready",
                            topicName, localCluster, remoteCluster, entry.getPosition());
                }
                isLocalMessageSkippedOnce = true;
                entry.release();
                msg.recycle();
                continue;
            }

            // Increment pending messages for messages produced locally
            PENDING_MESSAGES_UPDATER.incrementAndGet(this);

            msgOut.recordEvent(headersAndPayload.readableBytes());

            msg.setReplicatedFrom(localCluster);

            headersAndPayload.retain();

            producer.sendAsync(msg, ProducerSendCallback.create(this, entry, msg));
            atLeastOneMessageSentForReplication = true;
        }
    } catch (Exception e) {
        log.error("[{}][{} -> {}] Unexpected exception: {}", topicName, localCluster, remoteCluster,
                e.getMessage(), e);
    }

    HAVE_PENDING_READ_UPDATER.set(this, FALSE);

    if (atLeastOneMessageSentForReplication && !isWritable()) {
        // Don't read any more entries until the current pending entries are persisted
        if (log.isDebugEnabled()) {
            log.debug("[{}][{} -> {}] Pausing replication traffic. at-least-one: {} is-writable: {}", topicName,
                    localCluster, remoteCluster, atLeastOneMessageSentForReplication, isWritable());
        }
    } else {
        readMoreEntries();
    }
}

From source file:org.apache.pulsar.client.impl.ProducerImpl.java

License:Apache License

public void sendAsync(Message message, SendCallback callback) {
    checkArgument(message instanceof MessageImpl);

    if (!isValidProducerState(callback)) {
        return;//from   w ww  .  j  av a  2 s.  c  om
    }

    if (!canEnqueueRequest(callback)) {
        return;
    }

    MessageImpl msg = (MessageImpl) message;
    MessageMetadata.Builder msgMetadata = msg.getMessageBuilder();
    ByteBuf payload = msg.getDataBuffer();

    // If compression is enabled, we are compressing, otherwise it will simply use the same buffer
    int uncompressedSize = payload.readableBytes();
    ByteBuf compressedPayload = payload;
    // batch will be compressed when closed
    if (!isBatchMessagingEnabled()) {
        compressedPayload = compressor.encode(payload);
        payload.release();
    }

    if (!msg.isReplicated() && msgMetadata.hasProducerName()) {
        callback.sendComplete(
                new PulsarClientException.InvalidMessageException("Cannot re-use the same message"));
        compressedPayload.release();
        return;
    }

    try {
        synchronized (this) {
            long sequenceId = msgIdGeneratorUpdater.getAndIncrement(this);
            if (!msgMetadata.hasPublishTime()) {
                msgMetadata.setPublishTime(System.currentTimeMillis());

                checkArgument(!msgMetadata.hasProducerName());
                checkArgument(!msgMetadata.hasSequenceId());

                msgMetadata.setProducerName(producerName);
                msgMetadata.setSequenceId(sequenceId);
                if (conf.getCompressionType() != CompressionType.NONE) {
                    msgMetadata.setCompression(convertCompressionType(conf.getCompressionType()));
                    msgMetadata.setUncompressedSize(uncompressedSize);
                }
            }

            if (isBatchMessagingEnabled()) {
                // handle boundary cases where message being added would exceed
                // batch size and/or max message size
                if (batchMessageContainer.hasSpaceInBatch(msg)) {
                    batchMessageContainer.add(msg, callback);
                    payload.release();
                    if (batchMessageContainer.numMessagesInBatch == maxNumMessagesInBatch
                            || batchMessageContainer.currentBatchSizeBytes >= BatchMessageContainer.MAX_MESSAGE_BATCH_SIZE_BYTES) {
                        batchMessageAndSend();
                    }
                } else {
                    doBatchSendAndAdd(msg, callback, payload);
                }
            } else {
                ByteBuf cmd = sendMessage(producerId, sequenceId, 1, msgMetadata.build(), compressedPayload);
                msgMetadata.recycle();

                final OpSendMsg op = OpSendMsg.create(msg, cmd, sequenceId, callback);
                op.setNumMessagesInBatch(1);
                op.setBatchSizeByte(payload.readableBytes());
                pendingMessages.put(op);

                // Read the connection before validating if it's still connected, so that we avoid reading a null
                // value
                ClientCnx cnx = cnx();
                if (isConnected()) {
                    // If we do have a connection, the message is sent immediately, otherwise we'll try again once a
                    // new
                    // connection is established
                    cmd.retain();
                    cnx.ctx().channel().eventLoop().execute(WriteInEventLoopCallback.create(this, cnx, op));
                    stats.updateNumMsgsSent(op.numMessagesInBatch, op.batchSizeByte);
                } else {
                    if (log.isDebugEnabled()) {
                        log.debug("[{}] [{}] Connection is not ready -- sequenceId {}", topic, producerName,
                                sequenceId);
                    }
                }
            }
        }
    } catch (InterruptedException ie) {
        Thread.currentThread().interrupt();
        semaphore.release();
        callback.sendComplete(new PulsarClientException(ie));
    } catch (Throwable t) {
        semaphore.release();
        callback.sendComplete(new PulsarClientException(t));
    }
}

From source file:org.apache.pulsar.common.api.raw.MessageParser.java

License:Apache License

/**
 * Parse a raw Pulsar entry payload and extract all the individual message that may be included in the batch. The
 * provided {@link MessageProcessor} will be invoked for each individual message.
 *//*from www.j a v a 2s .  co m*/
public static void parseMessage(TopicName topicName, long ledgerId, long entryId, ByteBuf headersAndPayload,
        MessageProcessor processor) throws IOException {
    MessageMetadata msgMetadata = null;
    ByteBuf payload = headersAndPayload;
    ByteBuf uncompressedPayload = null;
    ReferenceCountedObject<MessageMetadata> refCntMsgMetadata = null;

    try {
        if (!verifyChecksum(topicName, headersAndPayload, ledgerId, entryId)) {
            // discard message with checksum error
            return;
        }

        try {
            msgMetadata = Commands.parseMessageMetadata(payload);
        } catch (Throwable t) {
            log.warn("[{}] Failed to deserialize metadata for message {}:{} - Ignoring", topicName, ledgerId,
                    entryId);
            return;
        }

        if (msgMetadata.getEncryptionKeysCount() > 0) {
            throw new IOException("Cannot parse encrypted message " + msgMetadata + " on topic " + topicName);
        }

        uncompressedPayload = uncompressPayloadIfNeeded(topicName, msgMetadata, headersAndPayload, ledgerId,
                entryId);

        if (uncompressedPayload == null) {
            // Message was discarded on decompression error
            return;
        }

        final int numMessages = msgMetadata.getNumMessagesInBatch();
        refCntMsgMetadata = new ReferenceCountedObject<>(msgMetadata, (x) -> x.recycle());

        if (numMessages == 1 && !msgMetadata.hasNumMessagesInBatch()) {
            processor.process(RawMessageImpl.get(refCntMsgMetadata, null, uncompressedPayload.retain(),
                    ledgerId, entryId, 0));
        } else {
            // handle batch message enqueuing; uncompressed payload has all messages in batch
            receiveIndividualMessagesFromBatch(refCntMsgMetadata, uncompressedPayload, ledgerId, entryId,
                    processor);
        }

    } finally {
        ReferenceCountUtil.safeRelease(uncompressedPayload);
        ReferenceCountUtil.safeRelease(refCntMsgMetadata);
    }
}

From source file:org.apache.spark.network.sasl.SaslMessage.java

License:Apache License

public static SaslMessage decode(ByteBuf buf) {
    if (buf.readByte() != TAG_BYTE) {
        throw new IllegalStateException("Expected SaslMessage, received something else"
                + " (maybe your client does not have SASL enabled?)");
    }//www  .j a v  a 2s. c om

    String appId = Encoders.Strings.decode(buf);
    // See comment in encodedLength().
    buf.readInt();
    return new SaslMessage(appId, buf.retain());
}

From source file:org.asynchttpclient.netty.util.ByteBufUtils.java

License:Open Source License

public static String byteBuf2String(Charset charset, ByteBuf... bufs) throws CharacterCodingException {
    if (charset == UTF_8 || charset == US_ASCII) {
        return Utf8ByteBufCharsetDecoder.decodeUtf8(bufs);
    } else {/*from   w  w w  .  ja v  a2 s.co  m*/
        CompositeByteBuf composite = Unpooled.compositeBuffer(bufs.length);

        try {
            for (ByteBuf buf : bufs) {
                buf.retain();
                composite.addComponent(buf);
            }

            return composite.toString(charset);

        } finally {
            composite.release();
        }
    }
}