Example usage for io.netty.buffer PooledByteBufAllocator DEFAULT

Introduction

In this page you can find the example usage for io.netty.buffer PooledByteBufAllocator DEFAULT.

Prototype

PooledByteBufAllocator DEFAULT

To view the source code for io.netty.buffer PooledByteBufAllocator DEFAULT.

Click Source Link

Usage

From source file:org.apache.pulsar.broker.stats.metrics.JvmMetrics.java

License:Apache License

@SuppressWarnings("restriction")
@Override/* ww w . ja v  a 2s .  c om*/
public List<Metrics> generate() {

    Metrics m = createMetrics();

    Runtime r = Runtime.getRuntime();

    m.put("jvm_heap_used", r.totalMemory() - r.freeMemory());
    m.put("jvm_max_memory", r.maxMemory());
    m.put("jvm_total_memory", r.totalMemory());

    m.put("jvm_direct_memory_used", getJvmDirectMemoryUsed());
    m.put("jvm_max_direct_memory", sun.misc.VM.maxDirectMemory());
    m.put("jvm_thread_cnt", getThreadCount());

    m.put("jvm_gc_young_pause", currentYoungGcTime);
    m.put("jvm_gc_young_count", currentYoungGcCount);
    m.put("jvm_gc_old_pause", currentOldGcTime);
    m.put("jvm_gc_old_count", currentOldGcCount);

    long totalAllocated = 0;
    long totalUsed = 0;

    for (PoolArenaMetric arena : PooledByteBufAllocator.DEFAULT.directArenas()) {
        for (PoolChunkListMetric list : arena.chunkLists()) {
            for (PoolChunkMetric chunk : list) {
                int size = chunk.chunkSize();
                int used = size - chunk.freeBytes();

                totalAllocated += size;
                totalUsed += used;
            }
        }
    }

    m.put("brk_default_pool_allocated", totalAllocated);
    m.put("brk_default_pool_used", totalUsed);

    return Lists.newArrayList(m);
}

---

From source file:org.apache.pulsar.client.impl.MessageCrypto.java

License:Apache License

public synchronized ByteBuf encrypt(Set<String> encKeys, CryptoKeyReader keyReader,
        MessageMetadata.Builder msgMetadata, ByteBuf payload) throws PulsarClientException {

    if (encKeys.isEmpty()) {
        return payload;
    }/*from   w w w  . ja  v  a  2 s  .  co m*/

    // Update message metadata with encrypted data key
    for (String keyName : encKeys) {
        if (encryptedDataKeyMap.get(keyName) == null) {
            // Attempt to load the key. This will allow us to load keys as soon as
            // a new key is added to producer config
            addPublicKeyCipher(keyName, keyReader);
        }
        EncryptionKeyInfo keyInfo = encryptedDataKeyMap.get(keyName);
        if (keyInfo != null) {
            if (keyInfo.getMetadata() != null && !keyInfo.getMetadata().isEmpty()) {
                List<KeyValue> kvList = new ArrayList<KeyValue>();
                keyInfo.getMetadata().forEach((key, value) -> {
                    kvList.add(KeyValue.newBuilder().setKey(key).setValue(value).build());
                });
                msgMetadata.addEncryptionKeys(EncryptionKeys.newBuilder().setKey(keyName)
                        .setValue(ByteString.copyFrom(keyInfo.getKey())).addAllMetadata(kvList).build());
            } else {
                msgMetadata.addEncryptionKeys(EncryptionKeys.newBuilder().setKey(keyName)
                        .setValue(ByteString.copyFrom(keyInfo.getKey())).build());
            }
        } else {
            // We should never reach here.
            log.error("{} Failed to find encrypted Data key for key {}.", logCtx, keyName);
        }

    }

    // Create gcm param
    // TODO: Replace random with counter and periodic refreshing based on timer/counter value
    secureRandom.nextBytes(iv);
    GCMParameterSpec gcmParam = new GCMParameterSpec(tagLen, iv);

    // Update message metadata with encryption param
    msgMetadata.setEncryptionParam(ByteString.copyFrom(iv));

    ByteBuf targetBuf = null;
    try {
        // Encrypt the data
        cipher.init(Cipher.ENCRYPT_MODE, dataKey, gcmParam);

        ByteBuffer sourceNioBuf = payload.nioBuffer(payload.readerIndex(), payload.readableBytes());

        int maxLength = cipher.getOutputSize(payload.readableBytes());
        targetBuf = PooledByteBufAllocator.DEFAULT.buffer(maxLength, maxLength);
        ByteBuffer targetNioBuf = targetBuf.nioBuffer(0, maxLength);

        int bytesStored = cipher.doFinal(sourceNioBuf, targetNioBuf);
        targetBuf.writerIndex(bytesStored);

    } catch (IllegalBlockSizeException | BadPaddingException | InvalidKeyException
            | InvalidAlgorithmParameterException | ShortBufferException e) {

        targetBuf.release();
        log.error("{} Failed to encrypt message. {}", logCtx, e);
        throw new PulsarClientException.CryptoException(e.getMessage());

    }

    payload.release();
    return targetBuf;
}

---

From source file:org.apache.pulsar.client.impl.MessageCrypto.java

License:Apache License

private ByteBuf decryptData(SecretKey dataKeySecret, MessageMetadata msgMetadata, ByteBuf payload) {

    // unpack iv and encrypted data
    ByteString ivString = msgMetadata.getEncryptionParam();
    ivString.copyTo(iv, 0);//  w ww.j a v a  2s .  c  om

    GCMParameterSpec gcmParams = new GCMParameterSpec(tagLen, iv);
    ByteBuf targetBuf = null;
    try {
        cipher.init(Cipher.DECRYPT_MODE, dataKeySecret, gcmParams);

        ByteBuffer sourceNioBuf = payload.nioBuffer(payload.readerIndex(), payload.readableBytes());

        int maxLength = cipher.getOutputSize(payload.readableBytes());
        targetBuf = PooledByteBufAllocator.DEFAULT.buffer(maxLength, maxLength);
        ByteBuffer targetNioBuf = targetBuf.nioBuffer(0, maxLength);

        int decryptedSize = cipher.doFinal(sourceNioBuf, targetNioBuf);
        targetBuf.writerIndex(decryptedSize);

    } catch (InvalidKeyException | InvalidAlgorithmParameterException | IllegalBlockSizeException
            | BadPaddingException | ShortBufferException e) {
        log.error("{} Failed to decrypt message {}", logCtx, e.getMessage());
        if (targetBuf != null) {
            targetBuf.release();
            targetBuf = null;
        }
    }

    return targetBuf;
}

---

From source file:org.apache.pulsar.client.impl.RawBatchConverter.java

License:Apache License

/**
 * Take a batched message and a filter, and returns a message with the only the sub-messages
 * which match the filter. Returns an empty optional if no messages match.
 *
 * This takes ownership of the passes in message, and if the returned optional is not empty,
 * the ownership of that message is returned also.
 *//*  w ww. j a  v  a 2s . co m*/
public static Optional<RawMessage> rebatchMessage(RawMessage msg, BiPredicate<String, MessageId> filter)
        throws IOException {
    checkArgument(msg.getMessageIdData().getBatchIndex() == -1);

    ByteBuf payload = msg.getHeadersAndPayload();
    MessageMetadata metadata = Commands.parseMessageMetadata(payload);
    ByteBuf batchBuffer = PooledByteBufAllocator.DEFAULT.buffer(payload.capacity());

    CompressionType compressionType = metadata.getCompression();
    CompressionCodec codec = CompressionCodecProvider.getCompressionCodec(compressionType);

    int uncompressedSize = metadata.getUncompressedSize();
    ByteBuf uncompressedPayload = codec.decode(payload, uncompressedSize);
    try {
        int batchSize = metadata.getNumMessagesInBatch();
        int messagesRetained = 0;

        SingleMessageMetadata.Builder emptyMetadataBuilder = SingleMessageMetadata.newBuilder()
                .setCompactedOut(true);
        for (int i = 0; i < batchSize; i++) {
            SingleMessageMetadata.Builder singleMessageMetadataBuilder = SingleMessageMetadata.newBuilder();
            ByteBuf singleMessagePayload = Commands.deSerializeSingleMessageInBatch(uncompressedPayload,
                    singleMessageMetadataBuilder, 0, batchSize);
            MessageId id = new BatchMessageIdImpl(msg.getMessageIdData().getLedgerId(),
                    msg.getMessageIdData().getEntryId(), msg.getMessageIdData().getPartition(), i);
            if (!singleMessageMetadataBuilder.hasPartitionKey()) {
                messagesRetained++;
                Commands.serializeSingleMessageInBatchWithPayload(singleMessageMetadataBuilder,
                        singleMessagePayload, batchBuffer);
            } else if (filter.test(singleMessageMetadataBuilder.getPartitionKey(), id)
                    && singleMessagePayload.readableBytes() > 0) {
                messagesRetained++;
                Commands.serializeSingleMessageInBatchWithPayload(singleMessageMetadataBuilder,
                        singleMessagePayload, batchBuffer);
            } else {
                Commands.serializeSingleMessageInBatchWithPayload(emptyMetadataBuilder, Unpooled.EMPTY_BUFFER,
                        batchBuffer);
            }
            singleMessageMetadataBuilder.recycle();
            singleMessagePayload.release();
        }
        emptyMetadataBuilder.recycle();

        if (messagesRetained > 0) {
            int newUncompressedSize = batchBuffer.readableBytes();
            ByteBuf compressedPayload = codec.encode(batchBuffer);

            MessageMetadata.Builder metadataBuilder = metadata.toBuilder();
            metadataBuilder.setUncompressedSize(newUncompressedSize);
            MessageMetadata newMetadata = metadataBuilder.build();

            ByteBuf metadataAndPayload = Commands.serializeMetadataAndPayload(Commands.ChecksumType.Crc32c,
                    newMetadata, compressedPayload);
            Optional<RawMessage> result = Optional
                    .of(new RawMessageImpl(msg.getMessageIdData(), metadataAndPayload));
            metadataBuilder.recycle();
            newMetadata.recycle();
            metadataAndPayload.release();
            compressedPayload.release();
            return result;
        } else {
            return Optional.empty();
        }
    } finally {
        batchBuffer.release();
        metadata.recycle();
        msg.close();
    }
}

---

From source file:org.apache.pulsar.client.impl.RawMessageImpl.java

License:Apache License

@Override
public ByteBuf serialize() {
    ByteBuf headersAndPayload = this.headersAndPayload.slice();

    // Format: [IdSize][Id][PayloadAndMetadataSize][PayloadAndMetadata]
    int idSize = id.getSerializedSize();
    int headerSize = 4 /* IdSize */ + idSize + 4 /* PayloadAndMetadataSize */;
    int totalSize = headerSize + headersAndPayload.readableBytes();

    ByteBuf buf = PooledByteBufAllocator.DEFAULT.buffer(totalSize);
    buf.writeInt(idSize);/*from  w w  w .ja va2  s.c  om*/
    try {
        ByteBufCodedOutputStream outStream = ByteBufCodedOutputStream.get(buf);
        id.writeTo(outStream);
        outStream.recycle();
    } catch (IOException e) {
        // This is in-memory serialization, should not fail
        log.error("IO exception serializing to ByteBuf (this shouldn't happen as operation is in-memory)", e);
        throw new RuntimeException(e);
    }
    buf.writeInt(headersAndPayload.readableBytes());
    buf.writeBytes(headersAndPayload);

    return buf;
}

---

From source file:org.apache.pulsar.common.api.ByteBufPairTest.java

License:Apache License

@Test
public void testDoubleByteBuf() throws Exception {
    ByteBuf b1 = PooledByteBufAllocator.DEFAULT.heapBuffer(128, 128);
    b1.writerIndex(b1.capacity());// w w w  .  j  ava 2s.  c  o m
    ByteBuf b2 = PooledByteBufAllocator.DEFAULT.heapBuffer(128, 128);
    b2.writerIndex(b2.capacity());
    ByteBufPair buf = ByteBufPair.get(b1, b2);

    assertEquals(buf.readableBytes(), 256);
    assertEquals(buf.getFirst(), b1);
    assertEquals(buf.getSecond(), b2);

    assertEquals(buf.refCnt(), 1);
    assertEquals(b1.refCnt(), 1);
    assertEquals(b2.refCnt(), 1);

    buf.release();

    assertEquals(buf.refCnt(), 0);
    assertEquals(b1.refCnt(), 0);
    assertEquals(b2.refCnt(), 0);
}

---

From source file:org.apache.pulsar.common.api.Commands.java

License:Apache License

private static ByteBuf serializeCommandMessageWithSize(BaseCommand cmd, ByteBuf metadataAndPayload) {
    // / Wire format
    // [TOTAL_SIZE] [CMD_SIZE][CMD] [MAGIC_NUMBER][CHECKSUM] [METADATA_SIZE][METADATA] [PAYLOAD]
    ////from   w w  w. j a va2  s  .  c o  m
    // metadataAndPayload contains from magic-number to the payload included

    int cmdSize = cmd.getSerializedSize();
    int totalSize = 4 + cmdSize + metadataAndPayload.readableBytes();
    int headersSize = 4 + 4 + cmdSize;

    ByteBuf headers = PooledByteBufAllocator.DEFAULT.buffer(headersSize);
    headers.writeInt(totalSize); // External frame

    try {
        // Write cmd
        headers.writeInt(cmdSize);

        ByteBufCodedOutputStream outStream = ByteBufCodedOutputStream.get(headers);
        cmd.writeTo(outStream);
        outStream.recycle();
    } catch (IOException e) {
        // This is in-memory serialization, should not fail
        throw new RuntimeException(e);
    }

    return DoubleByteBuf.get(headers, metadataAndPayload);
}

---

From source file:org.apache.pulsar.common.api.DoubleByteBufTest.java

License:Apache License

/**
 * Verify that readableBytes() returns writerIndex - readerIndex. In this case writerIndex is the end of the buffer
 * and readerIndex is increased by 64./*  ww w  .ja  v a 2s.c  o m*/
 * 
 * @throws Exception
 */
@Test
public void testReadableBytes() throws Exception {

    ByteBuf b1 = PooledByteBufAllocator.DEFAULT.heapBuffer(128, 128);
    b1.writerIndex(b1.capacity());
    ByteBuf b2 = PooledByteBufAllocator.DEFAULT.heapBuffer(128, 128);
    b2.writerIndex(b2.capacity());
    ByteBuf buf = DoubleByteBuf.get(b1, b2);

    assertEquals(buf.readerIndex(), 0);
    assertEquals(buf.writerIndex(), 256);
    assertEquals(buf.readableBytes(), 256);

    for (int i = 0; i < 4; ++i) {
        buf.skipBytes(64);
        assertEquals(buf.readableBytes(), 256 - 64 * (i + 1));
    }
}

---

From source file:org.apache.pulsar.common.compression.CompressionCodecZstd.java

License:Apache License

@Override
public ByteBuf encode(ByteBuf source) {
    int uncompressedLength = source.readableBytes();
    int maxLength = (int) Zstd.compressBound(uncompressedLength);

    ByteBuf target = PooledByteBufAllocator.DEFAULT.directBuffer(maxLength, maxLength);
    int compressedLength;

    if (source.hasMemoryAddress()) {
        compressedLength = (int) Zstd.compressUnsafe(target.memoryAddress(), maxLength,
                source.memoryAddress() + source.readerIndex(), uncompressedLength, ZSTD_COMPRESSION_LEVEL);
    } else {//from   www . jav  a2s. co  m
        ByteBuffer sourceNio = source.nioBuffer(source.readerIndex(), source.readableBytes());
        ByteBuffer targetNio = target.nioBuffer(0, maxLength);

        compressedLength = Zstd.compress(targetNio, sourceNio, ZSTD_COMPRESSION_LEVEL);
    }

    target.writerIndex(compressedLength);
    return target;
}

---

From source file:org.apache.pulsar.common.compression.CompressionCodecZstd.java

License:Apache License

@Override
public ByteBuf decode(ByteBuf encoded, int uncompressedLength) throws IOException {
    ByteBuf uncompressed = PooledByteBufAllocator.DEFAULT.directBuffer(uncompressedLength, uncompressedLength);

    if (encoded.hasMemoryAddress()) {
        Zstd.decompressUnsafe(uncompressed.memoryAddress(), uncompressedLength,
                encoded.memoryAddress() + encoded.readerIndex(), encoded.readableBytes());
    } else {/* w  w w.ja  va  2 s. c o  m*/
        ByteBuffer uncompressedNio = uncompressed.nioBuffer(0, uncompressedLength);
        ByteBuffer encodedNio = encoded.nioBuffer(encoded.readerIndex(), encoded.readableBytes());

        Zstd.decompress(uncompressedNio, encodedNio);
    }

    uncompressed.writerIndex(uncompressedLength);
    return uncompressed;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• »