Example usage for io.netty.buffer PooledByteBufAllocator DEFAULT

List of usage examples for io.netty.buffer PooledByteBufAllocator DEFAULT

  1. HOME
  2. Java
  3. io.netty
  4. io.netty.buffer.*
  5. PooledByteBufAllocator
  6. DEFAULT

Introduction

In this page you can find the example usage for io.netty.buffer PooledByteBufAllocator DEFAULT.

Prototype

PooledByteBufAllocator DEFAULT

To view the source code for io.netty.buffer PooledByteBufAllocator DEFAULT.

Click Source Link

Usage

From source file:org.apache.pulsar.common.stats.JvmMetrics.java

License:Apache License

public List<Metrics> generate() {

    Metrics m = createMetrics();/*w  ww  .  j  av  a  2s  .  co  m*/

    Runtime r = Runtime.getRuntime();

    m.put("jvm_heap_used", r.totalMemory() - r.freeMemory());
    m.put("jvm_max_memory", r.maxMemory());
    m.put("jvm_total_memory", r.totalMemory());

    m.put("jvm_direct_memory_used", getJvmDirectMemoryUsed());
    m.put("jvm_max_direct_memory", PlatformDependent.maxDirectMemory());
    m.put("jvm_thread_cnt", getThreadCount());

    this.gcLogger.logMetrics(m);

    long totalAllocated = 0;
    long totalUsed = 0;

    for (PoolArenaMetric arena : PooledByteBufAllocator.DEFAULT.directArenas()) {
        for (PoolChunkListMetric list : arena.chunkLists()) {
            for (PoolChunkMetric chunk : list) {
                int size = chunk.chunkSize();
                int used = size - chunk.freeBytes();

                totalAllocated += size;
                totalUsed += used;
            }
        }
    }

    m.put(this.componentName + "_default_pool_allocated", totalAllocated);
    m.put(this.componentName + "_default_pool_used", totalUsed);

    return Lists.newArrayList(m);
}

From source file:org.apache.pulsar.proxy.server.DirectProxyHandler.java

License:Apache License

public DirectProxyHandler(ProxyService service, ProxyConnection proxyConnection, String targetBrokerUrl,
        int protocolVersion, SslContext sslCtx) {
    this.authentication = proxyConnection.getClientAuthentication();
    this.inboundChannel = proxyConnection.ctx().channel();
    this.originalPrincipal = proxyConnection.clientAuthRole;
    this.clientAuthData = proxyConnection.clientAuthData;
    this.clientAuthMethod = proxyConnection.clientAuthMethod;
    this.protocolVersion = protocolVersion;
    this.sslCtx = sslCtx;
    ProxyConfiguration config = service.getConfiguration();

    // Start the connection attempt.
    Bootstrap b = new Bootstrap();
    // Tie the backend connection on the same thread to avoid context
    // switches when passing data between the 2
    // connections
    b.option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT);
    b.group(inboundChannel.eventLoop()).channel(inboundChannel.getClass()).option(ChannelOption.AUTO_READ,
            false);/*w w w .  j a  v a2  s.c o  m*/
    b.handler(new ChannelInitializer<SocketChannel>() {
        @Override
        protected void initChannel(SocketChannel ch) throws Exception {
            if (sslCtx != null) {
                ch.pipeline().addLast(TLS_HANDLER, sslCtx.newHandler(ch.alloc()));
            }
            ch.pipeline().addLast("frameDecoder",
                    new LengthFieldBasedFrameDecoder(PulsarDecoder.MaxFrameSize, 0, 4, 0, 4));
            ch.pipeline().addLast("proxyOutboundHandler", new ProxyBackendHandler(config, protocolVersion));
        }
    });

    URI targetBroker;
    try {
        // targetBrokerUrl is coming in the "hostname:6650" form, so we need
        // to extract host and port
        targetBroker = new URI("pulsar://" + targetBrokerUrl);
    } catch (URISyntaxException e) {
        log.warn("[{}] Failed to parse broker url '{}'", inboundChannel, targetBrokerUrl, e);
        inboundChannel.close();
        return;
    }

    ChannelFuture f = b.connect(targetBroker.getHost(), targetBroker.getPort());
    outboundChannel = f.channel();
    f.addListener(future -> {
        if (!future.isSuccess()) {
            // Close the connection if the connection attempt has failed.
            inboundChannel.close();
            return;
        }
        final ProxyBackendHandler cnx = (ProxyBackendHandler) outboundChannel.pipeline()
                .get("proxyOutboundHandler");
        cnx.setRemoteHostName(targetBroker.getHost());
    });
}

From source file:org.apache.pulsar.proxy.server.ProxyService.java

License:Apache License

public void start() throws Exception {
    if (!isBlank(proxyConfig.getZookeeperServers()) && !isBlank(proxyConfig.getConfigurationStoreServers())) {
        discoveryProvider = new BrokerDiscoveryProvider(this.proxyConfig, getZooKeeperClientFactory());
        this.configurationCacheService = new ConfigurationCacheService(discoveryProvider.globalZkCache);
        authorizationService = new AuthorizationService(PulsarConfigurationLoader.convertFrom(proxyConfig),
                configurationCacheService);
    }/*from   www.j av a 2  s . c  om*/

    ServerBootstrap bootstrap = new ServerBootstrap();
    bootstrap.childOption(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT);
    bootstrap.group(acceptorGroup, workerGroup);
    bootstrap.childOption(ChannelOption.TCP_NODELAY, true);
    bootstrap.childOption(ChannelOption.RCVBUF_ALLOCATOR,
            new AdaptiveRecvByteBufAllocator(1024, 16 * 1024, 1 * 1024 * 1024));

    bootstrap.channel(EventLoopUtil.getServerSocketChannelClass(workerGroup));
    EventLoopUtil.enableTriggeredMode(bootstrap);

    bootstrap.childHandler(new ServiceChannelInitializer(this, proxyConfig, false));
    // Bind and start to accept incoming connections.
    if (proxyConfig.getServicePort().isPresent()) {
        try {
            bootstrap.bind(proxyConfig.getServicePort().get()).sync();
            LOG.info("Started Pulsar Proxy at {}", serviceUrl);
        } catch (Exception e) {
            throw new IOException("Failed to bind Pulsar Proxy on port " + proxyConfig.getServicePort().get(),
                    e);
        }
    }
    LOG.info("Started Pulsar Proxy at {}", serviceUrl);

    if (proxyConfig.getServicePortTls().isPresent()) {
        ServerBootstrap tlsBootstrap = bootstrap.clone();
        tlsBootstrap.childHandler(new ServiceChannelInitializer(this, proxyConfig, true));
        tlsBootstrap.bind(proxyConfig.getServicePortTls().get()).sync();
        LOG.info("Started Pulsar TLS Proxy on port {}", proxyConfig.getServicePortTls().get());
    }
}

From source file:org.apache.pulsar.testclient.ManagedLedgerWriter.java

License:Apache License

public static void main(String[] args) throws Exception {

    final Arguments arguments = new Arguments();
    JCommander jc = new JCommander(arguments);
    jc.setProgramName("pulsar-perf-producer");

    try {/*  www  . j av  a  2  s. c  om*/
        jc.parse(args);
    } catch (ParameterException e) {
        System.out.println(e.getMessage());
        jc.usage();
        System.exit(-1);
    }

    if (arguments.help) {
        jc.usage();
        System.exit(-1);
    }

    arguments.testTime = TimeUnit.SECONDS.toMillis(arguments.testTime);

    // Dump config variables
    ObjectMapper m = new ObjectMapper();
    ObjectWriter w = m.writerWithDefaultPrettyPrinter();
    log.info("Starting Pulsar managed-ledger perf writer with config: {}", w.writeValueAsString(arguments));

    byte[] payloadData = new byte[arguments.msgSize];
    ByteBuf payloadBuffer = PooledByteBufAllocator.DEFAULT.directBuffer(arguments.msgSize);
    payloadBuffer.writerIndex(arguments.msgSize);

    // Now processing command line arguments
    String managedLedgerPrefix = "test-" + DigestUtils.sha1Hex(UUID.randomUUID().toString()).substring(0, 5);

    ClientConfiguration bkConf = new ClientConfiguration();
    bkConf.setUseV2WireProtocol(true);
    bkConf.setAddEntryTimeout(30);
    bkConf.setReadEntryTimeout(30);
    bkConf.setThrottleValue(0);
    bkConf.setNumChannelsPerBookie(arguments.maxConnections);
    bkConf.setZkServers(arguments.zookeeperServers);

    ManagedLedgerFactoryConfig mlFactoryConf = new ManagedLedgerFactoryConfig();
    mlFactoryConf.setMaxCacheSize(0);
    ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkConf, mlFactoryConf);

    ManagedLedgerConfig mlConf = new ManagedLedgerConfig();
    mlConf.setEnsembleSize(arguments.ensembleSize);
    mlConf.setWriteQuorumSize(arguments.writeQuorum);
    mlConf.setAckQuorumSize(arguments.ackQuorum);
    mlConf.setMinimumRolloverTime(10, TimeUnit.MINUTES);
    mlConf.setMetadataEnsembleSize(arguments.ensembleSize);
    mlConf.setMetadataWriteQuorumSize(arguments.writeQuorum);
    mlConf.setMetadataAckQuorumSize(arguments.ackQuorum);
    mlConf.setDigestType(arguments.digestType);
    mlConf.setMaxSizePerLedgerMb(2048);

    List<CompletableFuture<ManagedLedger>> futures = new ArrayList<>();

    for (int i = 0; i < arguments.numManagedLedgers; i++) {
        String name = String.format("%s-%03d", managedLedgerPrefix, i);
        CompletableFuture<ManagedLedger> future = new CompletableFuture<>();
        futures.add(future);
        factory.asyncOpen(name, mlConf, new OpenLedgerCallback() {

            @Override
            public void openLedgerComplete(ManagedLedger ledger, Object ctx) {
                future.complete(ledger);
            }

            @Override
            public void openLedgerFailed(ManagedLedgerException exception, Object ctx) {
                future.completeExceptionally(exception);
            }
        }, null);
    }

    List<ManagedLedger> managedLedgers = futures.stream().map(CompletableFuture::join)
            .collect(Collectors.toList());

    log.info("Created {} managed ledgers", managedLedgers.size());

    Runtime.getRuntime().addShutdownHook(new Thread() {
        public void run() {
            printAggregatedStats();
        }
    });

    Collections.shuffle(managedLedgers);
    AtomicBoolean isDone = new AtomicBoolean();

    List<List<ManagedLedger>> managedLedgersPerThread = Lists.partition(managedLedgers,
            Math.max(1, managedLedgers.size() / arguments.numThreads));

    for (int i = 0; i < arguments.numThreads; i++) {
        List<ManagedLedger> managedLedgersForThisThread = managedLedgersPerThread.get(i);
        int nunManagedLedgersForThisThread = managedLedgersForThisThread.size();
        long numMessagesForThisThread = arguments.numMessages / arguments.numThreads;
        int maxOutstandingForThisThread = arguments.maxOutstanding;

        executor.submit(() -> {
            try {
                final double msgRate = arguments.msgRate / (double) arguments.numThreads;
                final RateLimiter rateLimiter = RateLimiter.create(msgRate);

                // Acquire 1 sec worth of messages to have a slower ramp-up
                rateLimiter.acquire((int) msgRate);
                final long startTime = System.currentTimeMillis();

                final Semaphore semaphore = new Semaphore(maxOutstandingForThisThread);

                final AddEntryCallback addEntryCallback = new AddEntryCallback() {
                    @Override
                    public void addComplete(Position position, Object ctx) {
                        long sendTime = (Long) (ctx);
                        messagesSent.increment();
                        bytesSent.add(payloadData.length);

                        long latencyMicros = NANOSECONDS.toMicros(System.nanoTime() - sendTime);
                        recorder.recordValue(latencyMicros);
                        cumulativeRecorder.recordValue(latencyMicros);

                        semaphore.release();
                    }

                    @Override
                    public void addFailed(ManagedLedgerException exception, Object ctx) {
                        log.warn("Write error on message", exception);
                        System.exit(-1);
                    }
                };

                // Send messages on all topics/producers
                long totalSent = 0;
                while (true) {
                    for (int j = 0; j < nunManagedLedgersForThisThread; j++) {
                        if (arguments.testTime > 0) {
                            if (System.currentTimeMillis() - startTime > arguments.testTime) {
                                log.info("------------------- DONE -----------------------");
                                printAggregatedStats();
                                isDone.set(true);
                                Thread.sleep(5000);
                                System.exit(0);
                            }
                        }

                        if (numMessagesForThisThread > 0) {
                            if (totalSent++ >= numMessagesForThisThread) {
                                log.info("------------------- DONE -----------------------");
                                printAggregatedStats();
                                isDone.set(true);
                                Thread.sleep(5000);
                                System.exit(0);
                            }
                        }

                        semaphore.acquire();
                        rateLimiter.acquire();

                        final long sendTime = System.nanoTime();
                        managedLedgersForThisThread.get(j).asyncAddEntry(payloadBuffer, addEntryCallback,
                                sendTime);
                    }
                }
            } catch (Throwable t) {
                log.error("Got error", t);
            }
        });
    }

    // Print report stats
    long oldTime = System.nanoTime();

    Histogram reportHistogram = null;

    while (true) {
        try {
            Thread.sleep(10000);
        } catch (InterruptedException e) {
            break;
        }

        if (isDone.get()) {
            break;
        }

        long now = System.nanoTime();
        double elapsed = (now - oldTime) / 1e9;

        double rate = messagesSent.sumThenReset() / elapsed;
        double throughput = bytesSent.sumThenReset() / elapsed / 1024 / 1024 * 8;

        reportHistogram = recorder.getIntervalHistogram(reportHistogram);

        log.info(
                "Throughput produced: {}  msg/s --- {} Mbit/s --- Latency: mean: {} ms - med: {} - 95pct: {} - 99pct: {} - 99.9pct: {} - 99.99pct: {} - Max: {}",
                throughputFormat.format(rate), throughputFormat.format(throughput),
                dec.format(reportHistogram.getMean() / 1000.0),
                dec.format(reportHistogram.getValueAtPercentile(50) / 1000.0),
                dec.format(reportHistogram.getValueAtPercentile(95) / 1000.0),
                dec.format(reportHistogram.getValueAtPercentile(99) / 1000.0),
                dec.format(reportHistogram.getValueAtPercentile(99.9) / 1000.0),
                dec.format(reportHistogram.getValueAtPercentile(99.99) / 1000.0),
                dec.format(reportHistogram.getMaxValue() / 1000.0));

        reportHistogram.reset();

        oldTime = now;
    }

    factory.shutdown();
}

From source file:org.apache.qpid.jms.transports.TransportSupportTest.java

License:Apache License

@Test
public void testLegacySslProtocolsDisabledByDefaultOpenSSL() throws Exception {
    assumeTrue(OpenSsl.isAvailable());/*from   w w  w .  j  av a2s  . c  o m*/
    assumeTrue(OpenSsl.supportsKeyManagerFactory());

    TransportOptions options = createJksSslOptions(null);

    SslContext context = TransportSupport.createOpenSslContext(options);
    assertNotNull(context);

    SSLEngine engine = TransportSupport.createOpenSslEngine(PooledByteBufAllocator.DEFAULT, null, context,
            options);
    assertNotNull(engine);

    List<String> engineProtocols = Arrays.asList(engine.getEnabledProtocols());
    assertFalse("SSLv3 should not be enabled by default", engineProtocols.contains("SSLv3"));

    // TODO - Netty is currently unable to disable OpenSSL SSLv2Hello so we are stuck with it for now.
    // assertFalse("SSLv2Hello should not be enabled by default", engineProtocols.contains("SSLv2Hello"));
}

From source file:org.apache.qpid.jms.transports.TransportSupportTest.java

License:Apache License

@Test
public void testCreateSslEngineFromPkcs12StoreOpenSSL() throws Exception {
    assumeTrue(OpenSsl.isAvailable());/*  w  w w.j  a v a  2s . co  m*/
    assumeTrue(OpenSsl.supportsKeyManagerFactory());

    TransportOptions options = createPkcs12SslOptions();

    SslContext context = TransportSupport.createOpenSslContext(options);
    assertNotNull(context);

    SSLEngine engine = TransportSupport.createOpenSslEngine(PooledByteBufAllocator.DEFAULT, null, context,
            options);
    assertNotNull(engine);

    List<String> engineProtocols = Arrays.asList(engine.getEnabledProtocols());
    assertFalse(engineProtocols.isEmpty());
}

From source file:org.apache.qpid.jms.transports.TransportSupportTest.java

License:Apache License

@Test
public void testCreateSslEngineFromPkcs12StoreWithExplicitEnabledProtocolsOpenSSL() throws Exception {
    assumeTrue(OpenSsl.isAvailable());/*from   w  w w.  j  a  v a  2  s .  c  o m*/
    assumeTrue(OpenSsl.supportsKeyManagerFactory());

    TransportOptions options = createPkcs12SslOptions(ENABLED_PROTOCOLS);

    SslContext context = TransportSupport.createOpenSslContext(options);
    assertNotNull(context);

    SSLEngine engine = TransportSupport.createOpenSslEngine(PooledByteBufAllocator.DEFAULT, null, context,
            options);
    assertNotNull(engine);

    assertArrayEquals("Enabled protocols not as expected", ENABLED_OPENSSL_PROTOCOLS,
            engine.getEnabledProtocols());
}

From source file:org.apache.qpid.jms.transports.TransportSupportTest.java

License:Apache License

@Test
public void testCreateSslEngineFromJksStoreOpenSSL() throws Exception {
    assumeTrue(OpenSsl.isAvailable());//  w  w  w . j  a v  a  2  s.  c o  m
    assumeTrue(OpenSsl.supportsKeyManagerFactory());

    TransportOptions options = createJksSslOptions();

    SslContext context = TransportSupport.createOpenSslContext(options);
    assertNotNull(context);

    SSLEngine engine = TransportSupport.createOpenSslEngine(PooledByteBufAllocator.DEFAULT, null, context,
            options);
    assertNotNull(engine);

    List<String> engineProtocols = Arrays.asList(engine.getEnabledProtocols());
    assertFalse(engineProtocols.isEmpty());
}

From source file:org.apache.qpid.jms.transports.TransportSupportTest.java

License:Apache License

@Test
public void testCreateSslEngineFromJksStoreWithExplicitEnabledProtocolsOpenSSL() throws Exception {
    assumeTrue(OpenSsl.isAvailable());/*from w  ww. j av  a 2  s.co  m*/
    assumeTrue(OpenSsl.supportsKeyManagerFactory());

    TransportOptions options = createJksSslOptions(ENABLED_PROTOCOLS);

    SslContext context = TransportSupport.createOpenSslContext(options);
    assertNotNull(context);

    SSLEngine engine = TransportSupport.createOpenSslEngine(PooledByteBufAllocator.DEFAULT, null, context,
            options);
    assertNotNull(engine);

    assertArrayEquals("Enabled protocols not as expected", ENABLED_OPENSSL_PROTOCOLS,
            engine.getEnabledProtocols());
}

From source file:org.apache.qpid.jms.transports.TransportSupportTest.java

License:Apache License

@Test
public void testCreateSslEngineFromJksStoreWithExplicitDisabledProtocolsOpenSSL() throws Exception {
    assumeTrue(OpenSsl.isAvailable());//from   ww  w.j a  va2  s  . c  om
    assumeTrue(OpenSsl.supportsKeyManagerFactory());

    // Discover the default enabled protocols
    TransportOptions options = createJksSslOptions();
    SSLEngine directEngine = createOpenSSLEngineDirectly(options);
    String[] protocols = directEngine.getEnabledProtocols();
    assertTrue("There were no initial protocols to choose from!", protocols.length > 0);

    // Pull out one to disable specifically
    String[] disabledProtocol = new String[] { protocols[protocols.length - 1] };
    String[] trimmedProtocols = Arrays.copyOf(protocols, protocols.length - 1);
    options.setDisabledProtocols(disabledProtocol);
    SslContext context = TransportSupport.createOpenSslContext(options);
    SSLEngine engine = TransportSupport.createOpenSslEngine(PooledByteBufAllocator.DEFAULT, null, context,
            options);

    // verify the option took effect
    assertNotNull(engine);
    assertArrayEquals("Enabled protocols not as expected", trimmedProtocols, engine.getEnabledProtocols());
}