Example usage for io.netty.buffer PooledByteBufAllocator DEFAULT

List of usage examples for io.netty.buffer PooledByteBufAllocator DEFAULT

  1. HOME
  2. Java
  3. io.netty
  4. io.netty.buffer.*
  5. PooledByteBufAllocator
  6. DEFAULT

Introduction

In this page you can find the example usage for io.netty.buffer PooledByteBufAllocator DEFAULT.

Prototype

PooledByteBufAllocator DEFAULT

To view the source code for io.netty.buffer PooledByteBufAllocator DEFAULT.

Click Source Link

Usage

From source file:com.addthis.hydra.store.skiplist.SkipListCache.java

License:Apache License

private void pushAllPagesToDisk() {
    final ByteBufOutputStream byteStream = new ByteBufOutputStream(PooledByteBufAllocator.DEFAULT.buffer());
    try {//from   w  w  w.  ja v a  2s. c  om

        for (Page<K, V> page : evictionQueue) {

            page.writeLock();

            if (!page.inTransientState() && page.keys != null) {
                pushPageToDisk(page, byteStream);
            }

            page.writeUnlock();
        }
    } finally {
        byteStream.buffer().release();
    }

    assert (pushAllPagesToDiskAssertion());
}

From source file:com.addthis.hydra.store.skiplist.SkipListCache.java

License:Apache License

/**
 * Emit a log message that a page has been detected with a null nextFirstKey
 * and the page is not the largest page in the database.
 *
 * @param repair   if true then repair the page
 * @param counter  page number./*from   w  ww .j  ava 2s . com*/
 * @param page     contents of the page.
 * @param key      key associated with the page.
 * @param nextKey  key associated with the next page.
 */
private void missingNextFirstKey(final boolean repair, final int counter, Page<K, V> page, final K key,
        final K nextKey) {
    log.warn(
            "On page {} the firstKey is {} " + " the length is {} "
                    + " the nextFirstKey is null and the next page" + " is associated with key {}.",
            counter, page.firstKey, page.size, nextKey);
    if (repair) {
        log.info("Repairing nextFirstKey on page {}.", counter);
        page.nextFirstKey = nextKey;
        ByteBufOutputStream byteBufOutputStream = new ByteBufOutputStream(
                PooledByteBufAllocator.DEFAULT.buffer());
        try {
            byte[] pageEncoded = page.encode(byteBufOutputStream);
            externalStore.put(keyCoder.keyEncode(key), pageEncoded);
        } finally {
            byteBufOutputStream.buffer().release();
        }
    }
}

From source file:com.addthis.hydra.store.skiplist.SkipListCache.java

License:Apache License

private void repairInvalidKeys(final int counter, Page<K, V> page, final K key, final K nextKey) {
    boolean pageTransfer = false;
    Page<K, V> nextPage = pageFactory.generateEmptyPage(this, nextKey, page.getEncodeType());
    byte[] encodedNextPage = externalStore.get(keyCoder.keyEncode(nextKey));
    nextPage.decode(encodedNextPage);/*from   w  w  w  . ja va 2  s .co m*/
    for (int i = 0, pos = 0; i < page.size; i++, pos++) {
        K testKey = page.keys.get(i);
        // if testKey >= nextKey then we need to move the testKey off the current page
        if (compareKeys(testKey, nextKey) >= 0) {
            // non-negative value from binary search indicates the key was found on the next page
            if (binarySearch(nextPage.keys, testKey, comparator) >= 0) {
                log.info("Key {} was detected on next page. Deleting from page {}.", pos, counter);
            } else {
                log.info("Moving key {} on page {}.", pos, counter);
                page.fetchValue(i);
                V value = page.values.get(i);
                putIntoPage(nextPage, testKey, value);
                pageTransfer = true;
            }
            page.keys.remove(i);
            page.rawValues.remove(i);
            page.values.remove(i);
            page.size--;
            i--;
        }
    }
    ByteBufOutputStream byteBufOutputStream = new ByteBufOutputStream(PooledByteBufAllocator.DEFAULT.buffer());
    try {
        byte[] pageEncoded = page.encode(byteBufOutputStream);
        externalStore.put(keyCoder.keyEncode(key), pageEncoded);
        if (pageTransfer) {
            encodedNextPage = nextPage.encode(byteBufOutputStream);
            externalStore.put(keyCoder.keyEncode(nextKey), encodedNextPage);
        }
    } finally {
        byteBufOutputStream.buffer().release();
    }

}

From source file:com.addthis.hydra.task.source.Mark.java

License:Apache License

@Override
public byte[] bytesEncode(long version) {
    byte[] retBytes = null;
    ByteBuf buffer = PooledByteBufAllocator.DEFAULT.buffer();
    try {/*from www .  ja  v a2 s  .c o m*/
        byte[] valBytes = getValue().getBytes();
        Varint.writeUnsignedVarInt(valBytes.length, buffer);
        buffer.writeBytes(valBytes);
        Varint.writeUnsignedVarLong(getIndex(), buffer);
        buffer.writeByte(isEnd() ? 1 : 0);
        Varint.writeUnsignedVarInt(error, buffer);
        retBytes = new byte[buffer.readableBytes()];
        buffer.readBytes(retBytes);
    } finally {
        buffer.release();
    }
    return retBytes;
}

From source file:com.addthis.hydra.task.source.SimpleMark.java

License:Apache License

@Override
public byte[] bytesEncode(long version) {
    byte[] retBytes = null;
    ByteBuf buffer = PooledByteBufAllocator.DEFAULT.buffer();
    try {//from  ww  w.  j a  v  a2 s  .  c  o  m
        byte[] valBytes = val.getBytes();
        Varint.writeUnsignedVarInt(valBytes.length, buffer);
        buffer.writeBytes(valBytes);
        Varint.writeUnsignedVarLong(index, buffer);
        buffer.writeByte(end ? 1 : 0);
        retBytes = new byte[buffer.readableBytes()];
        buffer.readBytes(retBytes);
    } finally {
        buffer.release();
    }
    return retBytes;
}

From source file:com.addthis.meshy.Meshy.java

License:Apache License

protected Meshy() {
    if (HOSTNAME != null) {
        uuid = HOSTNAME + "-" + Long.toHexString(System.currentTimeMillis() & 0xffffff);
    } else {/* w  w w  .j  a v  a  2s. co m*/
        uuid = Long.toHexString(UUID.randomUUID().getMostSignificantBits());
    }
    workerGroup = new NioEventLoopGroup();
    clientBootstrap = new Bootstrap().option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT)
            .option(ChannelOption.TCP_NODELAY, true).option(ChannelOption.SO_KEEPALIVE, true)
            .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, 30000)
            .option(ChannelOption.WRITE_BUFFER_HIGH_WATER_MARK, HIGH_WATERMARK)
            .option(ChannelOption.WRITE_BUFFER_LOW_WATER_MARK, LOW_WATERMARK).channel(NioSocketChannel.class)
            .group(workerGroup).handler(new ChannelInitializer<NioSocketChannel>() {
                @Override
                protected void initChannel(final NioSocketChannel ch) throws Exception {
                    ch.pipeline().addLast(new ChannelState(Meshy.this, ch));
                }
            });
    updateLastEventTime();
}

From source file:com.addthis.meshy.MeshyServer.java

License:Apache License

public MeshyServer(final int port, final File rootDir, @Nullable String[] netif, final MeshyServerGroup group)
        throws IOException {
    super();/* w w w.j a v a  2s  .  co  m*/
    this.group = group;
    this.rootDir = rootDir;
    this.filesystems = loadFileSystems(rootDir);
    this.serverPeers = new AtomicInteger(0);
    bossGroup = new NioEventLoopGroup(1);
    ServerBootstrap bootstrap = new ServerBootstrap()
            .option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT)
            .option(ChannelOption.SO_BACKLOG, 1024).option(ChannelOption.CONNECT_TIMEOUT_MILLIS, 30000)
            .option(ChannelOption.SO_REUSEADDR, true)
            .childOption(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT)
            .childOption(ChannelOption.TCP_NODELAY, true).childOption(ChannelOption.SO_KEEPALIVE, true)
            .childOption(ChannelOption.WRITE_BUFFER_HIGH_WATER_MARK, HIGH_WATERMARK)
            .childOption(ChannelOption.WRITE_BUFFER_LOW_WATER_MARK, LOW_WATERMARK)
            .channel(NioServerSocketChannel.class).group(bossGroup, workerGroup)
            .childHandler(new ChannelInitializer<NioSocketChannel>() {
                @Override
                protected void initChannel(final NioSocketChannel ch) throws Exception {
                    ch.pipeline().addLast(new ChannelState(MeshyServer.this, ch));
                }
            });
    /* bind to one or more interfaces, if supplied, otherwise all */
    if ((netif == null) || (netif.length == 0)) {
        ServerSocketChannel serverChannel = (ServerSocketChannel) bootstrap.bind(new InetSocketAddress(port))
                .syncUninterruptibly().channel();
        serverLocal = serverChannel.localAddress();
    } else {
        InetSocketAddress primaryServerLocal = null;
        for (String net : netif) {
            NetworkInterface nicif = NetworkInterface.getByName(net);
            if (nicif == null) {
                log.warn("missing speficied NIC: {}", net);
                continue;
            }
            for (InterfaceAddress addr : nicif.getInterfaceAddresses()) {
                InetAddress inAddr = addr.getAddress();
                if (inAddr.getAddress().length != 4) {
                    log.trace("skip non-ipV4 address: {}", inAddr);
                    continue;
                }
                ServerSocketChannel serverChannel = (ServerSocketChannel) bootstrap
                        .bind(new InetSocketAddress(inAddr, port)).syncUninterruptibly().channel();
                if (primaryServerLocal != null) {
                    log.info("server [{}-*] binding to extra address: {}", super.getUUID(), primaryServerLocal);
                }
                primaryServerLocal = serverChannel.localAddress();
            }
        }
        if (primaryServerLocal == null) {
            throw new IllegalArgumentException("no valid interface / port specified");
        }
        serverLocal = primaryServerLocal;
    }
    this.serverNetIf = NetworkInterface.getByInetAddress(serverLocal.getAddress());
    this.serverPort = serverLocal.getPort();
    if (serverNetIf != null) {
        serverUuid = super.getUUID() + "-" + serverPort + "-" + serverNetIf.getName();
    } else {
        serverUuid = super.getUUID() + "-" + serverPort;
    }
    log.info("server [{}] on {} @ {}", getUUID(), serverLocal, rootDir);
    closeFuture = new DefaultPromise<>(GlobalEventExecutor.INSTANCE);
    workerGroup.terminationFuture().addListener((Future<Object> workerFuture) -> {
        bossGroup.terminationFuture().addListener((Future<Object> bossFuture) -> {
            if (!workerFuture.isSuccess()) {
                closeFuture.tryFailure(workerFuture.cause());
            } else if (!bossFuture.isSuccess()) {
                closeFuture.tryFailure(bossFuture.cause());
            } else {
                closeFuture.trySuccess(null);
            }
        });
    });
    addMessageFileSystemPaths();
    group.join(this);
    if (autoMesh) {
        startAutoMesh(serverPort, autoMeshTimeout);
    }
}

From source file:com.addthis.muxy.MuxStreamDirectory.java

License:Apache License

protected long writeStreamsToBlock() throws IOException {
    long writtenBytes = 0;
    openWritesLock.lock();//from   w  w  w  .ja v  a 2s  .  c o m
    try {
        /* yes, this could be optimized for concurrency by writing after lock is released, etc */
        if (openWriteBytes.get() == 0) {
            return 0;
        }
        List<TempData> streamsWithData = new ArrayList<>(openStreamWrites.size());
        for (StreamOut out : openStreamWrites.values()) {
            synchronized (out) {
                StreamOut pendingOut = pendingStreamCloses.get(out.meta.streamId);
                if (pendingOut != null) {
                    pendingOut.outputBuffer.writeBytes(out.outputBuffer);
                    assert out.outputBuffer.readableBytes() == 0;
                    out.outputBuffer.discardSomeReadBytes();
                } else if (out.output.buffer().readableBytes() > 0) {
                    streamsWithData.add(new TempData(out));
                    out.outputBuffer.retain();
                }
            }
        }
        for (StreamOut out : pendingStreamCloses.values()) { // guarded by openStreamWrites
            streamsWithData.add(new TempData(out));
        }
        pendingStreamCloses.clear();
        if (streamsWithData.isEmpty()) {
            return 0;
        }
        for (TempData td : streamsWithData) {
            writtenBytes += td.snapshotLength;
        }
        int streams = streamsWithData.size();
        publishEvent(MuxyStreamEvent.BLOCK_FILE_WRITE, streams);
        int currentFileOffset = (int) openWriteFile.size();
        /* write out IDs in this block */
        ByteBuf metaBuffer = PooledByteBufAllocator.DEFAULT.directBuffer(2 + 4 * streams + 4 + 8 * streams);
        metaBuffer.writeShort(streams);
        int bodyOutputSize = 0;
        for (TempData out : streamsWithData) {
            metaBuffer.writeInt(out.meta.streamId);
            /* (4) chunk body offset (4) chunk length (n) chunk bytes */
            bodyOutputSize += 8 + out.snapshotLength;
        }
        /* write remainder size for rest of block data so that readers can skip if desired ID isn't present */
        metaBuffer.writeInt(bodyOutputSize);
        /* write offsets and lengths for each stream id */
        int bodyOffset = streamsWithData.size() * 8;
        for (TempData out : streamsWithData) {
            metaBuffer.writeInt(bodyOffset); //TODO - reconsider how frequently this shortcut is placed on disk
            metaBuffer.writeInt(out.snapshotLength);
            bodyOffset += out.snapshotLength;
        }
        while (metaBuffer.readableBytes() > 0) {
            metaBuffer.readBytes(openWriteFile, metaBuffer.readableBytes());
        }
        metaBuffer.release();
        /* write bytes for each stream id */
        for (TempData out : streamsWithData) {
            synchronized (out.stream) { // need less confusing variable names for concurrency
                int toWrite = out.snapshotLength;
                while (toWrite > 0) {
                    int numBytesRead = out.data.readBytes(openWriteFile, toWrite);
                    assert numBytesRead > 0;
                    toWrite -= numBytesRead;
                }
                openWriteBytes.addAndGet((long) -out.snapshotLength);
                eventListener.reportWrite((long) -out.snapshotLength);
                out.meta.endFile = streamDirectoryConfig.currentFile.get();
                out.meta.endFileBlockOffset = currentFileOffset;
                if (out.meta.startFile == 0) {
                    out.meta.startFile = out.meta.endFile;
                    out.meta.startFileBlockOffset = out.meta.endFileBlockOffset;
                }
                if (!out.data.release()) { // release the pending writes that did not get an extra retain
                    out.data.discardSomeReadBytes();
                }
            }
        }
        /* check for rolling current file on size threshold */
        if (openWriteFile.size() > streamDirectoryConfig.maxFileSize) {
            openWriteFile.close();
            openWriteFile = FileChannel.open(getFileByID(bumpCurrentFile()), APPEND, CREATE);
            publishEvent(MuxyStreamEvent.BLOCK_FILE_WRITE_ROLL, streamDirectoryConfig.currentFile);
        }
    } finally {
        openWritesLock.unlock();
    }
    return writtenBytes;
}

From source file:com.alibaba.dubbo.remoting.transport.netty.NettyClient.java

License:Apache License

@Override
protected void doOpen() throws Throwable {
    NettyHelper.setNettyLoggerFactory();
    final NettyClientHandler nettyClientHandler = new NettyClientHandler(getUrl(), this);
    bootstrap = new Bootstrap();
    bootstrap.group(nioEventLoopGroup).option(ChannelOption.SO_KEEPALIVE, true)
            .option(ChannelOption.TCP_NODELAY, true)
            .option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT)
            //.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, getTimeout())
            .channel(NioSocketChannel.class);

    if (getTimeout() < 3000) {
        bootstrap.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, 3000);
    } else {//from   w ww  .j  a v a2 s . co  m
        bootstrap.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, getTimeout());
    }

    bootstrap.handler(new ChannelInitializer() {

        protected void initChannel(Channel ch) throws Exception {
            NettyCodecAdapter adapter = new NettyCodecAdapter(getCodec(), getUrl(), NettyClient.this);
            ch.pipeline()//.addLast("logging",new LoggingHandler(LogLevel.INFO))//for debug
                    .addLast("decoder", adapter.getDecoder())//
                    .addLast("encoder", adapter.getEncoder())//
                    .addLast("handler", nettyClientHandler);
        }
    });
}

From source file:com.alibaba.dubbo.remoting.transport.netty.NettyServer.java

License:Apache License

@Override
protected void doOpen() throws Throwable {
    NettyHelper.setNettyLoggerFactory();
    //netty4//from   w  w w .j  a va 2  s.  co m
    bootstrap = new ServerBootstrap();
    //1?
    bossGroup = new NioEventLoopGroup(1, new DefaultThreadFactory("NettyServerBoss", true));
    //       cpu*2 
    workerGroup = new NioEventLoopGroup(
            getUrl().getPositiveParameter(Constants.IO_THREADS_KEY, Constants.DEFAULT_IO_THREADS),
            new DefaultThreadFactory("NettyServerWorker", true));

    final NettyServerHandler nettyServerHandler = new NettyServerHandler(getUrl(), this);
    channels = nettyServerHandler.getChannels();

    bootstrap.group(bossGroup, workerGroup).channel(NioServerSocketChannel.class)
            .childOption(ChannelOption.TCP_NODELAY, Boolean.TRUE)
            .childOption(ChannelOption.SO_REUSEADDR, Boolean.TRUE)
            //                
            .childOption(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT)
            .childOption(ChannelOption.SO_KEEPALIVE, Boolean.TRUE)
            .childHandler(new ChannelInitializer<NioSocketChannel>() {
                @Override
                protected void initChannel(NioSocketChannel ch) throws Exception {
                    NettyCodecAdapter adapter = new NettyCodecAdapter(getCodec(), getUrl(), NettyServer.this);
                    ch.pipeline()//.addLast("logging",new LoggingHandler(LogLevel.INFO))//for debug
                            .addLast("decoder", adapter.getDecoder()).addLast("encoder", adapter.getEncoder())
                            .addLast("handler", nettyServerHandler);
                }
            });
    // bind
    ChannelFuture channelFuture = bootstrap.bind(getBindAddress());
    channelFuture.syncUninterruptibly();
    channel = channelFuture.channel();

}