Example usage for io.netty.buffer PooledByteBufAllocator DEFAULT

Introduction

In this page you can find the example usage for io.netty.buffer PooledByteBufAllocator DEFAULT.

Prototype

PooledByteBufAllocator DEFAULT

To view the source code for io.netty.buffer PooledByteBufAllocator DEFAULT.

Click Source Link

Usage

From source file:com.yahoo.pulsar.broker.service.PersistentMessageFinderTest.java

License:Apache License

public static byte[] createMessageWrittenToLedger(String msg) throws Exception {
    PulsarApi.MessageMetadata.Builder messageMetadataBuilder = PulsarApi.MessageMetadata.newBuilder();
    messageMetadataBuilder.setPublishTime(System.currentTimeMillis());
    messageMetadataBuilder.setProducerName("createMessageWrittenToLedger");
    messageMetadataBuilder.setSequenceId(1);
    PulsarApi.MessageMetadata messageMetadata = messageMetadataBuilder.build();
    ByteBuf data = UnpooledByteBufAllocator.DEFAULT.heapBuffer().writeBytes(msg.getBytes());

    int msgMetadataSize = messageMetadata.getSerializedSize();
    int payloadSize = data.readableBytes();
    int totalSize = 4 + msgMetadataSize + payloadSize;

    ByteBuf headers = PooledByteBufAllocator.DEFAULT.heapBuffer(totalSize, totalSize);
    ByteBufCodedOutputStream outStream = ByteBufCodedOutputStream.get(headers);
    headers.writeInt(msgMetadataSize);//from   w ww  . ja  va  2 s.  c  om
    messageMetadata.writeTo(outStream);
    ByteBuf headersAndPayload = DoubleByteBuf.get(headers, data);
    byte[] byteMessage = headersAndPayload.nioBuffer().array();
    headersAndPayload.release();
    return byteMessage;
}

---

From source file:com.yahoo.pulsar.broker.service.PulsarStats.java

License:Apache License

public PulsarStats(PulsarService pulsar) {
    this.topicStatsBuf = PooledByteBufAllocator.DEFAULT.heapBuffer(16 * 1024);
    this.tempTopicStatsBuf = PooledByteBufAllocator.DEFAULT.heapBuffer(16 * 1024);

    this.nsStats = new NamespaceStats();
    this.clusterReplicationMetrics = new ClusterReplicationMetrics(pulsar.getConfiguration().getClusterName(),
            pulsar.getConfiguration().isReplicationMetricsEnabled());
    this.bundleStats = Maps.newHashMap();
    this.tempMetricsCollection = Lists.newArrayList();
    this.metricsCollection = Lists.newArrayList();
    this.brokerOperabilityMetrics = new BrokerOperabilityMetrics(pulsar.getConfiguration().getClusterName(),
            pulsar.getAdvertisedAddress());
}

---

From source file:com.yahoo.pulsar.broker.stats.AllocatorStatsGenerator.java

License:Apache License

public static AllocatorStats generate(String allocatorName) {
    PooledByteBufAllocator allocator = null;
    if ("default".equals(allocatorName)) {
        allocator = PooledByteBufAllocator.DEFAULT;
    } else if ("ml-cache".equals(allocatorName)) {
        allocator = EntryCacheImpl.allocator;
    } else {/*  ww w  . ja v a 2  s.  c  o  m*/
        throw new IllegalArgumentException("Invalid allocator name : " + allocatorName);
    }

    AllocatorStats stats = new AllocatorStats();
    stats.directArenas = allocator.directArenas().stream().map(x -> newPoolArenaStats(x))
            .collect(Collectors.toList());
    stats.heapArenas = allocator.heapArenas().stream().map(x -> newPoolArenaStats(x))
            .collect(Collectors.toList());

    stats.numDirectArenas = allocator.numDirectArenas();
    stats.numHeapArenas = allocator.numHeapArenas();
    stats.numThreadLocalCaches = allocator.numThreadLocalCaches();
    stats.normalCacheSize = allocator.normalCacheSize();
    stats.smallCacheSize = allocator.smallCacheSize();
    stats.tinyCacheSize = allocator.tinyCacheSize();
    return stats;
}

---

From source file:com.yahoo.pulsar.broker.stats.metrics.JvmMetrics.java

License:Apache License

@SuppressWarnings("restriction")
@Override//from   w w w.ja v  a  2  s  .  c o  m
public List<Metrics> generate() {

    Metrics m = createMetrics();

    Runtime r = Runtime.getRuntime();

    m.put("jvm_heap_used", r.totalMemory() - r.freeMemory());
    m.put("jvm_max_memory", r.maxMemory());
    m.put("jvm_total_memory", r.totalMemory());

    m.put("jvm_direct_memory_used",
            sun.misc.SharedSecrets.getJavaNioAccess().getDirectBufferPool().getMemoryUsed());
    m.put("jvm_max_direct_memory", sun.misc.VM.maxDirectMemory());
    m.put("jvm_thread_cnt", getThreadCount());

    m.put("jvm_gc_young_pause", currentYoungGcTime);
    m.put("jvm_gc_young_count", currentYoungGcCount);
    m.put("jvm_gc_old_pause", currentOldGcTime);
    m.put("jvm_gc_old_count", currentOldGcCount);

    long totalAllocated = 0;
    long totalUsed = 0;

    for (PoolArenaMetric arena : PooledByteBufAllocator.DEFAULT.directArenas()) {
        for (PoolChunkListMetric list : arena.chunkLists()) {
            for (PoolChunkMetric chunk : list) {
                int size = chunk.chunkSize();
                int used = size - chunk.freeBytes();

                totalAllocated += size;
                totalUsed += used;
            }
        }
    }

    m.put("brk_default_pool_allocated", totalAllocated);
    m.put("brk_default_pool_used", totalUsed);

    return Lists.newArrayList(m);
}

---

From source file:com.yahoo.pulsar.client.impl.BatchMessageContainer.java

License:Apache License

void add(MessageImpl msg, SendCallback callback) {

    if (log.isDebugEnabled()) {
        log.debug("[{}] [{}] add message to batch, num messages in batch so far {}", topicName, producerName,
                numMessagesInBatch);/*from   w  ww  . ja  v a  2s .c  om*/
    }

    if (++numMessagesInBatch == 1) {
        // some properties are common amongst the different messages in the batch, hence we just pick it up from
        // the first message
        sequenceId = Commands.initBatchMessageMetadata(messageMetadata, msg.getMessageBuilder());
        this.firstCallback = callback;
        batchedMessageMetadataAndPayload = PooledByteBufAllocator.DEFAULT
                .buffer((int) MAX_MESSAGE_BATCH_SIZE_BYTES, (int) (PulsarDecoder.MaxMessageSize));
    }

    if (previousCallback != null) {
        previousCallback.addCallback(callback);
    }
    previousCallback = callback;

    currentBatchSizeBytes += msg.getDataBuffer().readableBytes();
    PulsarApi.MessageMetadata.Builder msgBuilder = msg.getMessageBuilder();
    batchedMessageMetadataAndPayload = Commands.serializeSingleMessageInBatchWithPayload(msgBuilder,
            msg.getDataBuffer(), batchedMessageMetadataAndPayload);
    messages.add(msg);
    msgBuilder.recycle();
}

---

From source file:com.yahoo.pulsar.client.impl.ConnectionPool.java

License:Apache License

public ConnectionPool(final PulsarClientImpl client, EventLoopGroup eventLoopGroup) {
    this.eventLoopGroup = eventLoopGroup;
    this.maxConnectionsPerHosts = client.getConfiguration().getConnectionsPerBroker();

    pool = new ConcurrentHashMap<>();
    bootstrap = new Bootstrap();
    bootstrap.group(eventLoopGroup);/*from  ww w  . ja v a  2 s.  c o m*/
    if (SystemUtils.IS_OS_LINUX && eventLoopGroup instanceof EpollEventLoopGroup) {
        bootstrap.channel(EpollSocketChannel.class);
    } else {
        bootstrap.channel(NioSocketChannel.class);
    }

    bootstrap.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, 10000);
    bootstrap.option(ChannelOption.TCP_NODELAY, client.getConfiguration().isUseTcpNoDelay());
    bootstrap.option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT);
    bootstrap.handler(new ChannelInitializer<SocketChannel>() {
        public void initChannel(SocketChannel ch) throws Exception {
            ClientConfiguration clientConfig = client.getConfiguration();
            if (clientConfig.isUseTls()) {
                SslContextBuilder builder = SslContextBuilder.forClient();
                if (clientConfig.isTlsAllowInsecureConnection()) {
                    builder.trustManager(InsecureTrustManagerFactory.INSTANCE);
                } else {
                    if (clientConfig.getTlsTrustCertsFilePath().isEmpty()) {
                        // Use system default
                        builder.trustManager((File) null);
                    } else {
                        File trustCertCollection = new File(clientConfig.getTlsTrustCertsFilePath());
                        builder.trustManager(trustCertCollection);
                    }
                }

                // Set client certificate if available
                AuthenticationDataProvider authData = clientConfig.getAuthentication().getAuthData();
                if (authData.hasDataForTls()) {
                    builder.keyManager(authData.getTlsPrivateKey(),
                            (X509Certificate[]) authData.getTlsCertificates());
                }

                SslContext sslCtx = builder.build();
                ch.pipeline().addLast(TLS_HANDLER, sslCtx.newHandler(ch.alloc()));
            }
            ch.pipeline().addLast("frameDecoder",
                    new PulsarLengthFieldFrameDecoder(MaxMessageSize, 0, 4, 0, 4));
            ch.pipeline().addLast("handler", new ClientCnx(client));
        }
    });
}

---

From source file:com.yahoo.pulsar.common.api.Commands.java

License:Apache License

private static ByteBuf serializeWithSize(BaseCommand.Builder cmdBuilder) {
    // / Wire format
    // [TOTAL_SIZE] [CMD_SIZE][CMD]
    BaseCommand cmd = cmdBuilder.build();

    int cmdSize = cmd.getSerializedSize();
    int totalSize = cmdSize + 4;
    int frameSize = totalSize + 4;

    ByteBuf buf = PooledByteBufAllocator.DEFAULT.buffer(frameSize, frameSize);

    // Prepend 2 lengths to the buffer
    buf.writeInt(totalSize);//from   w w w .  ja va2  s  . c o m
    buf.writeInt(cmdSize);

    ByteBufCodedOutputStream outStream = ByteBufCodedOutputStream.get(buf);

    try {
        cmd.writeTo(outStream);
    } catch (IOException e) {
        // This is in-memory serialization, should not fail
        throw new RuntimeException(e);
    } finally {
        cmd.recycle();
        cmdBuilder.recycle();
        outStream.recycle();
    }

    return buf;
}

---

From source file:com.yahoo.pulsar.common.api.Commands.java

License:Apache License

private static ByteBuf serializeCommandSendWithSize(BaseCommand.Builder cmdBuilder, ChecksumType checksumType,
        MessageMetadata msgMetadata, ByteBuf payload) {
    // / Wire format
    // [TOTAL_SIZE] [CMD_SIZE][CMD] [MAGIC_NUMBER][CHECKSUM] [METADATA_SIZE][METADATA] [PAYLOAD]

    BaseCommand cmd = cmdBuilder.build();
    int cmdSize = cmd.getSerializedSize();
    int msgMetadataSize = msgMetadata.getSerializedSize();
    int payloadSize = payload.readableBytes();
    int magicAndChecksumLength = ChecksumType.Crc32c.equals(checksumType) ? (2 + 4 /* magic + checksumLength*/)
            : 0;//from  w  w  w  .java2  s  .  c o m
    boolean includeChecksum = magicAndChecksumLength > 0;
    int headerContentSize = 4 + cmdSize + magicAndChecksumLength + 4 + msgMetadataSize; // cmdLength + cmdSize + magicLength +
    // checksumSize + msgMetadataLength +
    // msgMetadataSize
    int totalSize = headerContentSize + payloadSize;
    int headersSize = 4 + headerContentSize; // totalSize + headerLength
    int checksumReaderIndex = -1;

    ByteBuf headers = PooledByteBufAllocator.DEFAULT.buffer(headersSize, headersSize);
    headers.writeInt(totalSize); // External frame

    try {
        // Write cmd
        headers.writeInt(cmdSize);

        ByteBufCodedOutputStream outStream = ByteBufCodedOutputStream.get(headers);
        cmd.writeTo(outStream);
        cmd.recycle();
        cmdBuilder.recycle();

        //Create checksum placeholder
        if (includeChecksum) {
            headers.writeShort(magicCrc32c);
            checksumReaderIndex = headers.writerIndex();
            headers.writerIndex(headers.writerIndex() + checksumSize); //skip 4 bytes of checksum
        }

        // Write metadata
        headers.writeInt(msgMetadataSize);
        msgMetadata.writeTo(outStream);
        outStream.recycle();
    } catch (IOException e) {
        // This is in-memory serialization, should not fail
        throw new RuntimeException(e);
    }

    ByteBuf command = DoubleByteBuf.get(headers, payload);

    // write checksum at created checksum-placeholder
    if (includeChecksum) {
        headers.markReaderIndex();
        headers.readerIndex(checksumReaderIndex + checksumSize);
        int metadataChecksum = computeChecksum(headers);
        int computedChecksum = resumeChecksum(metadataChecksum, payload);
        // set computed checksum
        headers.setInt(checksumReaderIndex, computedChecksum);
        headers.resetReaderIndex();
    }
    return command;
}

---

From source file:com.yahoo.pulsar.common.api.Commands.java

License:Apache License

private static ByteBuf serializeCommandMessageWithSize(BaseCommand cmd, ByteBuf metadataAndPayload) {
    // / Wire format
    // [TOTAL_SIZE] [CMD_SIZE][CMD] [METADATA_SIZE][METADATA] [PAYLOAD]

    int cmdSize = cmd.getSerializedSize();
    int totalSize = 4 + cmdSize + metadataAndPayload.readableBytes();
    int headersSize = 4 + 4 + cmdSize;

    ByteBuf headers = PooledByteBufAllocator.DEFAULT.buffer(headersSize);
    headers.writeInt(totalSize); // External frame

    try {//from w ww  .j  a va  2  s  .c o  m
        // Write cmd
        headers.writeInt(cmdSize);

        ByteBufCodedOutputStream outStream = ByteBufCodedOutputStream.get(headers);
        cmd.writeTo(outStream);
        outStream.recycle();
    } catch (IOException e) {
        // This is in-memory serialization, should not fail
        throw new RuntimeException(e);
    }

    return DoubleByteBuf.get(headers, metadataAndPayload);
}

---

From source file:com.yahoo.pulsar.common.api.DoubleByteBuf.java

License:Apache License

@Override
public ByteBufAllocator alloc() {
    return PooledByteBufAllocator.DEFAULT;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• »