Example usage for io.netty.buffer ByteBuf writerIndex

Introduction

In this page you can find the example usage for io.netty.buffer ByteBuf writerIndex.

Prototype

public abstract ByteBuf writerIndex(int writerIndex);

Source Link

Document

Sets the writerIndex of this buffer.

Usage

From source file:org.apache.pulsar.common.api.DoubleByteBufTest.java

License:Apache License

/**
 * Verify that readableBytes() returns writerIndex - readerIndex. In this case writerIndex is the end of the buffer
 * and readerIndex is increased by 64./*from   ww  w .  j  ava 2s.co  m*/
 * 
 * @throws Exception
 */
@Test
public void testReadableBytes() throws Exception {

    ByteBuf b1 = PooledByteBufAllocator.DEFAULT.heapBuffer(128, 128);
    b1.writerIndex(b1.capacity());
    ByteBuf b2 = PooledByteBufAllocator.DEFAULT.heapBuffer(128, 128);
    b2.writerIndex(b2.capacity());
    ByteBuf buf = DoubleByteBuf.get(b1, b2);

    assertEquals(buf.readerIndex(), 0);
    assertEquals(buf.writerIndex(), 256);
    assertEquals(buf.readableBytes(), 256);

    for (int i = 0; i < 4; ++i) {
        buf.skipBytes(64);
        assertEquals(buf.readableBytes(), 256 - 64 * (i + 1));
    }
}

---

From source file:org.apache.pulsar.common.compression.CompressionCodecZstd.java

License:Apache License

@Override
public ByteBuf encode(ByteBuf source) {
    int uncompressedLength = source.readableBytes();
    int maxLength = (int) Zstd.compressBound(uncompressedLength);

    ByteBuf target = PooledByteBufAllocator.DEFAULT.directBuffer(maxLength, maxLength);
    int compressedLength;

    if (source.hasMemoryAddress()) {
        compressedLength = (int) Zstd.compressUnsafe(target.memoryAddress(), maxLength,
                source.memoryAddress() + source.readerIndex(), uncompressedLength, ZSTD_COMPRESSION_LEVEL);
    } else {/*w  w w  . j a v  a 2  s .c om*/
        ByteBuffer sourceNio = source.nioBuffer(source.readerIndex(), source.readableBytes());
        ByteBuffer targetNio = target.nioBuffer(0, maxLength);

        compressedLength = Zstd.compress(targetNio, sourceNio, ZSTD_COMPRESSION_LEVEL);
    }

    target.writerIndex(compressedLength);
    return target;
}

---

From source file:org.apache.pulsar.common.compression.CompressionCodecZstd.java

License:Apache License

@Override
public ByteBuf decode(ByteBuf encoded, int uncompressedLength) throws IOException {
    ByteBuf uncompressed = PooledByteBufAllocator.DEFAULT.directBuffer(uncompressedLength, uncompressedLength);

    if (encoded.hasMemoryAddress()) {
        Zstd.decompressUnsafe(uncompressed.memoryAddress(), uncompressedLength,
                encoded.memoryAddress() + encoded.readerIndex(), encoded.readableBytes());
    } else {// w ww.j a  v  a 2s  .com
        ByteBuffer uncompressedNio = uncompressed.nioBuffer(0, uncompressedLength);
        ByteBuffer encodedNio = encoded.nioBuffer(encoded.readerIndex(), encoded.readableBytes());

        Zstd.decompress(uncompressedNio, encodedNio);
    }

    uncompressed.writerIndex(uncompressedLength);
    return uncompressed;
}

---

From source file:org.apache.pulsar.testclient.ManagedLedgerWriter.java

License:Apache License

public static void main(String[] args) throws Exception {

    final Arguments arguments = new Arguments();
    JCommander jc = new JCommander(arguments);
    jc.setProgramName("pulsar-perf-producer");

    try {// w w w  .j  a va  2  s .  c o m
        jc.parse(args);
    } catch (ParameterException e) {
        System.out.println(e.getMessage());
        jc.usage();
        System.exit(-1);
    }

    if (arguments.help) {
        jc.usage();
        System.exit(-1);
    }

    arguments.testTime = TimeUnit.SECONDS.toMillis(arguments.testTime);

    // Dump config variables
    ObjectMapper m = new ObjectMapper();
    ObjectWriter w = m.writerWithDefaultPrettyPrinter();
    log.info("Starting Pulsar managed-ledger perf writer with config: {}", w.writeValueAsString(arguments));

    byte[] payloadData = new byte[arguments.msgSize];
    ByteBuf payloadBuffer = PooledByteBufAllocator.DEFAULT.directBuffer(arguments.msgSize);
    payloadBuffer.writerIndex(arguments.msgSize);

    // Now processing command line arguments
    String managedLedgerPrefix = "test-" + DigestUtils.sha1Hex(UUID.randomUUID().toString()).substring(0, 5);

    ClientConfiguration bkConf = new ClientConfiguration();
    bkConf.setUseV2WireProtocol(true);
    bkConf.setAddEntryTimeout(30);
    bkConf.setReadEntryTimeout(30);
    bkConf.setThrottleValue(0);
    bkConf.setNumChannelsPerBookie(arguments.maxConnections);
    bkConf.setZkServers(arguments.zookeeperServers);

    ManagedLedgerFactoryConfig mlFactoryConf = new ManagedLedgerFactoryConfig();
    mlFactoryConf.setMaxCacheSize(0);
    ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkConf, mlFactoryConf);

    ManagedLedgerConfig mlConf = new ManagedLedgerConfig();
    mlConf.setEnsembleSize(arguments.ensembleSize);
    mlConf.setWriteQuorumSize(arguments.writeQuorum);
    mlConf.setAckQuorumSize(arguments.ackQuorum);
    mlConf.setMinimumRolloverTime(10, TimeUnit.MINUTES);
    mlConf.setMetadataEnsembleSize(arguments.ensembleSize);
    mlConf.setMetadataWriteQuorumSize(arguments.writeQuorum);
    mlConf.setMetadataAckQuorumSize(arguments.ackQuorum);
    mlConf.setDigestType(arguments.digestType);
    mlConf.setMaxSizePerLedgerMb(2048);

    List<CompletableFuture<ManagedLedger>> futures = new ArrayList<>();

    for (int i = 0; i < arguments.numManagedLedgers; i++) {
        String name = String.format("%s-%03d", managedLedgerPrefix, i);
        CompletableFuture<ManagedLedger> future = new CompletableFuture<>();
        futures.add(future);
        factory.asyncOpen(name, mlConf, new OpenLedgerCallback() {

            @Override
            public void openLedgerComplete(ManagedLedger ledger, Object ctx) {
                future.complete(ledger);
            }

            @Override
            public void openLedgerFailed(ManagedLedgerException exception, Object ctx) {
                future.completeExceptionally(exception);
            }
        }, null);
    }

    List<ManagedLedger> managedLedgers = futures.stream().map(CompletableFuture::join)
            .collect(Collectors.toList());

    log.info("Created {} managed ledgers", managedLedgers.size());

    Runtime.getRuntime().addShutdownHook(new Thread() {
        public void run() {
            printAggregatedStats();
        }
    });

    Collections.shuffle(managedLedgers);
    AtomicBoolean isDone = new AtomicBoolean();

    List<List<ManagedLedger>> managedLedgersPerThread = Lists.partition(managedLedgers,
            Math.max(1, managedLedgers.size() / arguments.numThreads));

    for (int i = 0; i < arguments.numThreads; i++) {
        List<ManagedLedger> managedLedgersForThisThread = managedLedgersPerThread.get(i);
        int nunManagedLedgersForThisThread = managedLedgersForThisThread.size();
        long numMessagesForThisThread = arguments.numMessages / arguments.numThreads;
        int maxOutstandingForThisThread = arguments.maxOutstanding;

        executor.submit(() -> {
            try {
                final double msgRate = arguments.msgRate / (double) arguments.numThreads;
                final RateLimiter rateLimiter = RateLimiter.create(msgRate);

                // Acquire 1 sec worth of messages to have a slower ramp-up
                rateLimiter.acquire((int) msgRate);
                final long startTime = System.currentTimeMillis();

                final Semaphore semaphore = new Semaphore(maxOutstandingForThisThread);

                final AddEntryCallback addEntryCallback = new AddEntryCallback() {
                    @Override
                    public void addComplete(Position position, Object ctx) {
                        long sendTime = (Long) (ctx);
                        messagesSent.increment();
                        bytesSent.add(payloadData.length);

                        long latencyMicros = NANOSECONDS.toMicros(System.nanoTime() - sendTime);
                        recorder.recordValue(latencyMicros);
                        cumulativeRecorder.recordValue(latencyMicros);

                        semaphore.release();
                    }

                    @Override
                    public void addFailed(ManagedLedgerException exception, Object ctx) {
                        log.warn("Write error on message", exception);
                        System.exit(-1);
                    }
                };

                // Send messages on all topics/producers
                long totalSent = 0;
                while (true) {
                    for (int j = 0; j < nunManagedLedgersForThisThread; j++) {
                        if (arguments.testTime > 0) {
                            if (System.currentTimeMillis() - startTime > arguments.testTime) {
                                log.info("------------------- DONE -----------------------");
                                printAggregatedStats();
                                isDone.set(true);
                                Thread.sleep(5000);
                                System.exit(0);
                            }
                        }

                        if (numMessagesForThisThread > 0) {
                            if (totalSent++ >= numMessagesForThisThread) {
                                log.info("------------------- DONE -----------------------");
                                printAggregatedStats();
                                isDone.set(true);
                                Thread.sleep(5000);
                                System.exit(0);
                            }
                        }

                        semaphore.acquire();
                        rateLimiter.acquire();

                        final long sendTime = System.nanoTime();
                        managedLedgersForThisThread.get(j).asyncAddEntry(payloadBuffer, addEntryCallback,
                                sendTime);
                    }
                }
            } catch (Throwable t) {
                log.error("Got error", t);
            }
        });
    }

    // Print report stats
    long oldTime = System.nanoTime();

    Histogram reportHistogram = null;

    while (true) {
        try {
            Thread.sleep(10000);
        } catch (InterruptedException e) {
            break;
        }

        if (isDone.get()) {
            break;
        }

        long now = System.nanoTime();
        double elapsed = (now - oldTime) / 1e9;

        double rate = messagesSent.sumThenReset() / elapsed;
        double throughput = bytesSent.sumThenReset() / elapsed / 1024 / 1024 * 8;

        reportHistogram = recorder.getIntervalHistogram(reportHistogram);

        log.info(
                "Throughput produced: {}  msg/s --- {} Mbit/s --- Latency: mean: {} ms - med: {} - 95pct: {} - 99pct: {} - 99.9pct: {} - 99.99pct: {} - Max: {}",
                throughputFormat.format(rate), throughputFormat.format(throughput),
                dec.format(reportHistogram.getMean() / 1000.0),
                dec.format(reportHistogram.getValueAtPercentile(50) / 1000.0),
                dec.format(reportHistogram.getValueAtPercentile(95) / 1000.0),
                dec.format(reportHistogram.getValueAtPercentile(99) / 1000.0),
                dec.format(reportHistogram.getValueAtPercentile(99.9) / 1000.0),
                dec.format(reportHistogram.getValueAtPercentile(99.99) / 1000.0),
                dec.format(reportHistogram.getMaxValue() / 1000.0));

        reportHistogram.reset();

        oldTime = now;
    }

    factory.shutdown();
}

---

From source file:org.apache.qpid.jms.provider.amqp.message.AmqpWritableBufferTest.java

License:Apache License

@Test
public void testHasRemaining() {
    ByteBuf buffer = Unpooled.buffer(100, 100);
    AmqpWritableBuffer writable = new AmqpWritableBuffer(buffer);

    assertTrue(writable.hasRemaining());
    writable.put((byte) 0);
    assertTrue(writable.hasRemaining());
    buffer.writerIndex(buffer.maxCapacity());
    assertFalse(writable.hasRemaining());
}

---

From source file:org.curioswitch.common.server.framework.redis.ProtobufRedisCodec.java

License:Open Source License

@Override
public void encodeKey(K key, ByteBuf target) {
    target.writeBytes(keyPrefix);/*from w  w w  .j  a  va 2 s. c o  m*/
    encodeTo(key, target);
    target.writerIndex(key.getSerializedSize() + keyPrefix.length);
}

---

From source file:org.curioswitch.common.server.framework.redis.ProtobufRedisCodec.java

License:Open Source License

@Override
public void encodeValue(V value, ByteBuf target) {
    encodeTo(value, target);
    target.writerIndex(value.getSerializedSize());
}

---

From source file:org.dcache.http.ReusableChunkedNioFile.java

License:Apache License

/**
 * Like {@link ChunkedNioFile#readChunk}, but uses position independent
 * IO calls./*from w  ww.jav  a2 s. c o m*/
 */
@Override
public ByteBuf readChunk(ByteBufAllocator allocator) throws Exception {
    long offset = _offset;
    if (offset >= _endOffset) {
        return null;
    }

    int length = (int) Math.min(_chunkSize, _endOffset - offset);
    ByteBuf chunk = allocator.buffer(length);
    boolean release = true;
    try {
        ByteBuffer buffer = chunk.nioBuffer(0, length);

        while (buffer.hasRemaining()) {
            /* use position independent thread safe call */
            int bytes = _channel.read(buffer, offset);
            if (bytes < 0) {
                break;
            }
            offset += bytes;
        }
        chunk.writerIndex(buffer.position());
        _offset = offset;
        release = false;
        return chunk;
    } finally {
        if (release) {
            chunk.release();
        }
    }
}

---

From source file:org.dcache.xrootd.stream.ChunkedFileChannelReadResponse.java

License:Open Source License

@Override
protected ByteBuf read(ByteBufAllocator alloc, long position, int length) throws IOException {
    ByteBuf chunk = alloc.ioBuffer(length);
    try {//ww  w. j a v a2 s .  c o m
        chunk.writerIndex(length);
        ByteBuffer buffer = chunk.nioBuffer();
        while (length > 0) {
            /* use position independent thread safe call */
            int bytes = channel.read(buffer, position);
            if (bytes < 0) {
                break;
            }
            position += bytes;
            length -= bytes;
        }
        chunk.writerIndex(chunk.writerIndex() - length);
        return chunk;
    } catch (RuntimeException | IOException e) {
        ReferenceCountUtil.release(chunk);
        throw e;
    }
}

---

From source file:org.dcache.xrootd.stream.ChunkedFileChannelReadvResponse.java

License:Open Source License

@Override
protected ByteBuf read(ByteBufAllocator alloc, int fd, long position, int length)
        throws IOException, XrootdException {
    checkValidFileDescriptor(fd);/* ww  w.ja v a2 s  .com*/

    FileChannel channel = channels.get(fd);

    ByteBuf chunk = alloc.ioBuffer(length);
    try {
        chunk.writerIndex(length);
        ByteBuffer buffer = chunk.nioBuffer();
        while (length > 0) {
            /* use position independent thread safe call */
            int bytes = channel.read(buffer, position);
            if (bytes < 0) {
                break;
            }
            position += bytes;
            length -= bytes;
        }
        chunk.writerIndex(chunk.writerIndex() - length);
        return chunk;
    } catch (RuntimeException | IOException e) {
        ReferenceCountUtil.release(chunk);
        throw e;
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• »