Example usage for io.netty.buffer PooledByteBufAllocator PooledByteBufAllocator

List of usage examples for io.netty.buffer PooledByteBufAllocator PooledByteBufAllocator

Introduction

In this page you can find the example usage for io.netty.buffer PooledByteBufAllocator PooledByteBufAllocator.

Prototype

@Deprecated
public PooledByteBufAllocator(boolean preferDirect, int nHeapArena, int nDirectArena, int pageSize,
        int maxOrder) 

Source Link

Usage

From source file:io.urmia.st.Main.java

License:Open Source License

public static void main(String[] args) throws Exception {

    final int port;
    final String base;

    boolean autoRegister = ArgumentParseUtil.isAutoRegister(args);
    String zkURL = ArgumentParseUtil.getZooKeeperURL(args);

    log.info("starting with zk at: {}, auto register: {}", zkURL, autoRegister);

    ns = new ZkNamingServiceImpl(zkURL, AZ);

    Optional<ServiceInstance<NodeType>> meOpt = ns.whoAmI(NodeType.ODS, autoRegister);

    if (!meOpt.isPresent()) {
        System.err.println("unable to find my instance. use auto register or cli-admin to add my node");
        System.exit(1);//  w  w  w  .  j  av  a  2s  . co m
        return;
    }

    Runtime.getRuntime().addShutdownHook(new ShutdownHook());

    EventLoopGroup bossGroup = new NioEventLoopGroup(/*1*/);
    //EventLoopGroup bossGroup = new EpollEventLoopGroup(1);
    //EventLoopGroup workerGroup = new NioEventLoopGroup();

    try {
        me = meOpt.get();

        log.info("my service instance: {}", me);

        ns.register(me);

        base = me.getUriSpec().getParts().get(0).getValue();
        port = me.getPort();

        if (!(new File(base).isDirectory())) {
            System.err.println("base in not directory: " + base);
            return;
        }

        int nHeapArena = 1;
        int nDirectArena = 1;
        int pageSize = /*8192*/4096;
        int maxOrder = 1;

        // http://normanmaurer.me/presentations/2014-facebook-eng-netty/slides.html#14.0
        ServerBootstrap b = new ServerBootstrap();

        b.group(bossGroup).channel(NioServerSocketChannel.class).childOption(ChannelOption.AUTO_READ, false)
                .childOption(ChannelOption.ALLOCATOR,
                        new PooledByteBufAllocator(true, nHeapArena, nDirectArena, pageSize, maxOrder))
                .childHandler(new HttpUploadServerInitializer(base));

        Channel ch = b.bind(port).sync().channel();
        log.info("object storage Server (ODS) at port: {}", port);

        System.err.println("starting ODS " + me.getId() + " on port: " + port + ", base: " + base);

        ch.closeFuture().sync();
    } finally {
        bossGroup.shutdownGracefully();
        //workerGroup.shutdownGracefully();
    }
}

From source file:org.apache.flink.runtime.io.network.netty.NettyBufferPool.java

License:Apache License

/**
 * Creates Netty's buffer pool with the specified number of direct arenas.
 *
 * @param numberOfArenas Number of arenas (recommended: 2 * number of task
 *                       slots)//from www.j av a2 s .  c  o  m
 */
NettyBufferPool(int numberOfArenas) {
    checkArgument(numberOfArenas >= 1, "Number of arenas");
    this.numberOfArenas = numberOfArenas;

    if (!PlatformDependent.hasUnsafe()) {
        LOG.warn("Using direct buffers, but sun.misc.Unsafe not available.");
    }

    // We strictly prefer direct buffers and disallow heap allocations.
    boolean preferDirect = true;

    // Arenas allocate chunks of pageSize << maxOrder bytes. With these
    // defaults, this results in chunks of 16 MB.
    int pageSize = 8192;
    int maxOrder = 11;

    this.chunkSize = pageSize << maxOrder;

    // Number of direct arenas. Each arena allocates a chunk of 16 MB, i.e.
    // we allocate numDirectArenas * 16 MB of direct memory. This can grow
    // to multiple chunks per arena during runtime, but this should only
    // happen with a large amount of connections per task manager. We
    // control the memory allocations with low/high watermarks when writing
    // to the TCP channels. Chunks are allocated lazily.
    int numDirectArenas = numberOfArenas;

    // No heap arenas, please.
    int numHeapArenas = 0;

    this.alloc = new PooledByteBufAllocator(preferDirect, numHeapArenas, numDirectArenas, pageSize, maxOrder);

    Object[] allocDirectArenas = null;
    try {
        Field directArenasField = alloc.getClass().getDeclaredField("directArenas");
        directArenasField.setAccessible(true);

        allocDirectArenas = (Object[]) directArenasField.get(alloc);
    } catch (Exception ignored) {
        LOG.warn("Memory statistics not available");
    } finally {
        this.directArenas = allocDirectArenas;
    }
}

From source file:org.apache.flink.runtime.io.network.netty.NettyConnectionManager.java

License:Apache License

@Override
public void start(ChannelManager channelManager) throws IOException {
    LOG.info(String.format("Starting with %d incoming and %d outgoing connection threads.", numInThreads,
            numOutThreads));//w w w. ja  v a  2s .  c o  m
    LOG.info(String.format("Setting low water mark to %d and high water mark to %d bytes.", lowWaterMark,
            highWaterMark));
    LOG.info(String.format("Close channels after idle for %d ms.", closeAfterIdleForMs));

    final BufferProviderBroker bufferProviderBroker = channelManager;
    final EnvelopeDispatcher envelopeDispatcher = channelManager;

    int numHeapArenas = 0;
    int numDirectArenas = numInThreads + numOutThreads;
    int pageSize = bufferSize << 1;
    int chunkSize = 16 << 20; // 16 MB

    // shift pageSize maxOrder times to get to chunkSize
    int maxOrder = (int) (Math.log(chunkSize / pageSize) / Math.log(2));

    PooledByteBufAllocator pooledByteBufAllocator = new PooledByteBufAllocator(true, numHeapArenas,
            numDirectArenas, pageSize, maxOrder);

    String msg = String.format(
            "Instantiated PooledByteBufAllocator with direct arenas: %d, heap arenas: %d, "
                    + "page size (bytes): %d, chunk size (bytes): %d.",
            numDirectArenas, numHeapArenas, pageSize, (pageSize << maxOrder));
    LOG.info(msg);

    // --------------------------------------------------------------------
    // server bootstrap (incoming connections)
    // --------------------------------------------------------------------
    in = new ServerBootstrap();
    in.group(new NioEventLoopGroup(numInThreads)).channel(NioServerSocketChannel.class)
            .localAddress(bindAddress, bindPort).childHandler(new ChannelInitializer<SocketChannel>() {
                @Override
                public void initChannel(SocketChannel channel) throws Exception {
                    channel.pipeline().addLast(new InboundEnvelopeDecoder(bufferProviderBroker))
                            .addLast(new InboundEnvelopeDispatcher(envelopeDispatcher));
                }
            }).option(ChannelOption.RCVBUF_ALLOCATOR, new FixedRecvByteBufAllocator(pageSize))
            .option(ChannelOption.ALLOCATOR, pooledByteBufAllocator);

    // --------------------------------------------------------------------
    // client bootstrap (outgoing connections)
    // --------------------------------------------------------------------
    out = new Bootstrap();
    out.group(new NioEventLoopGroup(numOutThreads)).channel(NioSocketChannel.class)
            .handler(new ChannelInitializer<SocketChannel>() {
                @Override
                public void initChannel(SocketChannel channel) throws Exception {
                    channel.pipeline().addLast(new OutboundEnvelopeEncoder());
                }
            }).option(ChannelOption.WRITE_BUFFER_LOW_WATER_MARK, lowWaterMark)
            .option(ChannelOption.WRITE_BUFFER_HIGH_WATER_MARK, highWaterMark)
            .option(ChannelOption.ALLOCATOR, pooledByteBufAllocator).option(ChannelOption.TCP_NODELAY, false)
            .option(ChannelOption.SO_KEEPALIVE, true);

    try {
        in.bind().sync();
    } catch (InterruptedException e) {
        throw new IOException(e);
    }

    if (LOG.isDebugEnabled()) {
        new Thread(new Runnable() {
            @Override
            public void run() {
                Date date = new Date();

                while (true) {
                    try {
                        Thread.sleep(DEBUG_PRINT_QUEUED_ENVELOPES_EVERY_MS);

                        date.setTime(System.currentTimeMillis());

                        System.out.println(date);
                        System.out.println(getNonZeroNumQueuedEnvelopes());
                    } catch (InterruptedException e) {
                        e.printStackTrace();
                    }
                }
            }
        }).start();
    }
}