Example usage for io.netty.buffer ByteBuf readBytes

List of usage examples for io.netty.buffer ByteBuf readBytes

  1. HOME
  2. Java
  3. io.netty
  4. io.netty.buffer.*
  5. ByteBuf
  6. readBytes

Introduction

In this page you can find the example usage for io.netty.buffer ByteBuf readBytes.

Prototype

public abstract ByteBuf readBytes(ByteBuffer dst);

Source Link

Document

Transfers this buffer's data to the specified destination starting at the current readerIndex until the destination's position reaches its limit, and increases the readerIndex by the number of the transferred bytes.

Usage

From source file:com.addthis.hydra.data.util.KeyTopper.java

License:Apache License

/**
 * Encode the data structure into a serialized representation.
 * Encode the number of elements followed by each (key, value)
 * pair. If the error estimation is used then encode the special
 * byte value 0 (since we will never encode 0 as the size
 * of a non-empty map) at the head of the byte array.
 * @param version//from   ww  w.j  av  a 2  s  . c  om
 * @return
 */
@Override
public byte[] bytesEncode(long version) {
    if (map.size() == 0) {
        return EMPTY;
    }
    byte[] retBytes = null;
    ByteBuf byteBuf = PooledByteBufAllocator.DEFAULT.buffer();
    try {
        if (hasErrors()) {
            byteBuf.writeByte(0);
        }
        Varint.writeUnsignedVarInt(map.size(), byteBuf);
        for (Map.Entry<String, Long> mapEntry : map.entrySet()) {
            String key = mapEntry.getKey();
            if (key == null) {
                throw new NullPointerException("KeyTopper decoded null key");
            }
            byte[] keyBytes = key.getBytes("UTF-8");
            Varint.writeUnsignedVarInt(keyBytes.length, byteBuf);
            byteBuf.writeBytes(keyBytes);
            Varint.writeUnsignedVarLong(mapEntry.getValue(), byteBuf);
            if (hasErrors()) {
                Long error = errors.get(key);
                if (error != null) {
                    Varint.writeUnsignedVarLong(error, byteBuf);
                } else {
                    Varint.writeUnsignedVarLong(0, byteBuf);
                }
            }
        }
        retBytes = new byte[byteBuf.readableBytes()];
        byteBuf.readBytes(retBytes);
    } catch (UnsupportedEncodingException e) {
        throw Throwables.propagate(e);
    } finally {
        byteBuf.release();
    }
    return retBytes;
}

From source file:com.addthis.hydra.data.util.KeyTopper.java

License:Apache License

@Override
public void bytesDecode(byte[] b, long version) {
    map = new HashMap<>();
    errors = null;//from  ww w .  j a  v a2s.  c o m
    if (b.length == 0) {
        return;
    }
    ByteBuf byteBuf = Unpooled.wrappedBuffer(b);
    try {
        byte marker = byteBuf.getByte(byteBuf.readerIndex());
        if (marker == 0) {
            errors = new HashMap<>();
            // Consume the sentinel byte value
            byteBuf.readByte();
        }
        int mapSize = Varint.readUnsignedVarInt(byteBuf);
        try {
            if (mapSize > 0) {
                for (int i = 0; i < mapSize; i++) {
                    int keyLength = Varint.readUnsignedVarInt(byteBuf);
                    byte[] keybytes = new byte[keyLength];
                    byteBuf.readBytes(keybytes);
                    String k = new String(keybytes, "UTF-8");
                    long value = Varint.readUnsignedVarLong(byteBuf);
                    map.put(k, value);
                    if (hasErrors()) {
                        long error = Varint.readUnsignedVarLong(byteBuf);
                        if (error != 0) {
                            errors.put(k, error);
                        }
                    }
                }
            }
        } catch (Exception e) {
            throw Throwables.propagate(e);
        }
    } finally {
        byteBuf.release();
    }
}

From source file:com.addthis.hydra.store.db.DBKey.java

License:Apache License

public static DBKey deltaDecode(byte[] encoding, @Nonnull IPageDB.Key baseKey) {
    ByteBuf buffer = Unpooled.copiedBuffer(encoding);
    long offset = Varint.readSignedVarLong(buffer);
    long id = offset + baseKey.id();
    Raw key;// w  w  w .j  ava2 s .  c o  m
    if (buffer.readableBytes() == 0) {
        key = null;
    } else {
        byte[] data = new byte[buffer.readableBytes()];
        buffer.readBytes(data);
        key = Raw.get(data);
    }
    return new DBKey(id, key);
}

From source file:com.addthis.hydra.store.skiplist.Page.java

License:Apache License

public byte[] encode(ByteBufOutputStream out, boolean record) {
    SkipListCacheMetrics metrics = parent.metrics;
    parent.numPagesEncoded.getAndIncrement();
    try {//  ww  w. j av  a  2 s . c  om
        OutputStream os = out;
        out.write(gztype | FLAGS_HAS_ESTIMATES | FLAGS_IS_SPARSE);
        switch (gztype) {
        case 0:
            break;
        case 1:
            os = new DeflaterOutputStream(out, new Deflater(gzlevel));
            break;
        case 2:
            os = new GZOut(out, gzbuf, gzlevel);
            break;
        case 3:
            os = new LZFOutputStream(out);
            break;
        case 4:
            os = new SnappyOutputStream(out);
            break;
        default:
            throw new RuntimeException("invalid gztype: " + gztype);
        }

        DataOutputStream dos = new DataOutputStream(os);
        byte[] firstKeyEncoded = keyCoder.keyEncode(firstKey);
        byte[] nextFirstKeyEncoded = keyCoder.keyEncode(nextFirstKey);

        updateHistogram(metrics.encodeNextFirstKeySize, nextFirstKeyEncoded.length, record);

        Varint.writeUnsignedVarInt(size, dos);
        Varint.writeUnsignedVarInt(firstKeyEncoded.length, dos);
        dos.write(firstKeyEncoded);
        Varint.writeUnsignedVarInt(nextFirstKeyEncoded.length, dos);
        if (nextFirstKeyEncoded.length > 0) {
            dos.write(nextFirstKeyEncoded);
        }
        for (int i = 0; i < size; i++) {
            byte[] keyEncoded = keyCoder.keyEncode(keys.get(i));
            byte[] rawVal = rawValues.get(i);

            if (rawVal == null || encodeType != KeyCoder.EncodeType.SPARSE) {
                fetchValue(i);
                rawVal = keyCoder.valueEncode(values.get(i), KeyCoder.EncodeType.SPARSE);
            }

            updateHistogram(metrics.encodeKeySize, keyEncoded.length, record);
            updateHistogram(metrics.encodeValueSize, rawVal.length, record);

            Varint.writeUnsignedVarInt(keyEncoded.length, dos);
            dos.write(keyEncoded);
            Varint.writeUnsignedVarInt(rawVal.length, dos);
            dos.write(rawVal);
        }

        Varint.writeUnsignedVarInt((estimateTotal > 0 ? estimateTotal : 1), dos);
        Varint.writeUnsignedVarInt((estimates > 0 ? estimates : 1), dos);
        switch (gztype) {
        case 1:
            ((DeflaterOutputStream) os).finish();
            break;
        case 2:
            ((GZOut) os).finish();
            break;
        case 4:
            os.flush();
            break;
        }
        os.flush();
        os.close();
        dos.close();

        ByteBuf buffer = out.buffer();

        byte[] returnValue = new byte[out.writtenBytes()];

        buffer.readBytes(returnValue);
        buffer.clear();
        updateHistogram(metrics.numberKeysPerPage, size, record);
        updateHistogram(metrics.encodePageSize, returnValue.length, record);
        return returnValue;
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    }
}

From source file:com.addthis.hydra.store.skiplist.SparsePage.java

License:Apache License

public byte[] encode(ByteBufOutputStream out, boolean record) {
    SkipListCacheMetrics metrics = parent.metrics;
    parent.numPagesEncoded.getAndIncrement();
    try {/*from   w w w .  ja v a2  s.c  o m*/
        OutputStream os = out;
        out.write(gztype | FLAGS_HAS_ESTIMATES | FLAGS_IS_SPARSE);
        switch (gztype) {
        case 0:
            break;
        case 1:
            os = new DeflaterOutputStream(out, new Deflater(gzlevel));
            break;
        case 2:
            os = new GZOut(out, gzbuf, gzlevel);
            break;
        case 3:
            os = new LZFOutputStream(out);
            break;
        case 4:
            os = new SnappyOutputStream(out);
            break;
        default:
            throw new RuntimeException("invalid gztype: " + gztype);
        }

        DataOutputStream dos = new DataOutputStream(os);
        byte[] firstKeyEncoded = keyCoder.keyEncode(firstKey);
        byte[] nextFirstKeyEncoded = keyCoder.keyEncode(nextFirstKey);

        updateHistogram(metrics.encodeNextFirstKeySize, nextFirstKeyEncoded.length, record);

        Varint.writeUnsignedVarInt(size, dos);
        Varint.writeUnsignedVarInt(firstKeyEncoded.length, dos);
        dos.write(firstKeyEncoded);
        Varint.writeUnsignedVarInt(nextFirstKeyEncoded.length, dos);
        if (nextFirstKeyEncoded.length > 0) {
            dos.write(nextFirstKeyEncoded);
        }
        for (int i = 0; i < size; i++) {
            byte[] keyEncoded = keyCoder.keyEncode(keys.get(i));
            byte[] rawVal = rawValues.get(i);

            if (rawVal == null || encodeType != PageEncodeType.SPARSE) {
                fetchValue(i);
                rawVal = keyCoder.valueEncode(values.get(i), PageEncodeType.SPARSE);
            }

            updateHistogram(metrics.encodeKeySize, keyEncoded.length, record);
            updateHistogram(metrics.encodeValueSize, rawVal.length, record);

            Varint.writeUnsignedVarInt(keyEncoded.length, dos);
            dos.write(keyEncoded);
            Varint.writeUnsignedVarInt(rawVal.length, dos);
            dos.write(rawVal);
        }

        Varint.writeUnsignedVarInt((estimateTotal > 0 ? estimateTotal : 1), dos);
        Varint.writeUnsignedVarInt((estimates > 0 ? estimates : 1), dos);
        switch (gztype) {
        case 1:
            ((DeflaterOutputStream) os).finish();
            break;
        case 2:
            ((GZOut) os).finish();
            break;
        }
        os.flush(); // flush should be called by dos.close(), but better safe than sorry
        dos.close();

        ByteBuf buffer = out.buffer();

        byte[] returnValue = new byte[out.writtenBytes()];

        buffer.readBytes(returnValue);
        buffer.clear();
        updateHistogram(metrics.numberKeysPerPage, size, record);
        updateHistogram(metrics.encodePageSize, returnValue.length, record);
        return returnValue;
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    }
}

From source file:com.addthis.hydra.task.source.Mark.java

License:Apache License

@Override
public byte[] bytesEncode(long version) {
    byte[] retBytes = null;
    ByteBuf buffer = PooledByteBufAllocator.DEFAULT.buffer();
    try {//from   ww  w.ja  va2s  . c o m
        byte[] valBytes = getValue().getBytes();
        Varint.writeUnsignedVarInt(valBytes.length, buffer);
        buffer.writeBytes(valBytes);
        Varint.writeUnsignedVarLong(getIndex(), buffer);
        buffer.writeByte(isEnd() ? 1 : 0);
        Varint.writeUnsignedVarInt(error, buffer);
        retBytes = new byte[buffer.readableBytes()];
        buffer.readBytes(retBytes);
    } finally {
        buffer.release();
    }
    return retBytes;
}

From source file:com.addthis.hydra.task.source.Mark.java

License:Apache License

@Override
public void bytesDecode(byte[] b, long version) {
    ByteBuf buffer = Unpooled.wrappedBuffer(b);
    try {//from   w ww .  ja v a 2 s.  c om
        int valLength = Varint.readUnsignedVarInt(buffer);
        byte[] valBytes = new byte[valLength];
        buffer.readBytes(valBytes);
        setValue(new String(valBytes));
        setIndex(Varint.readUnsignedVarLong(buffer));
        setEnd(buffer.readByte() == 1);
        setError(Varint.readUnsignedVarInt(buffer));
    } finally {
        buffer.release();
    }
}

From source file:com.addthis.hydra.task.source.SimpleMark.java

License:Apache License

@Override
public byte[] bytesEncode(long version) {
    byte[] retBytes = null;
    ByteBuf buffer = PooledByteBufAllocator.DEFAULT.buffer();
    try {/* www  . j ava  2  s  .c o  m*/
        byte[] valBytes = val.getBytes();
        Varint.writeUnsignedVarInt(valBytes.length, buffer);
        buffer.writeBytes(valBytes);
        Varint.writeUnsignedVarLong(index, buffer);
        buffer.writeByte(end ? 1 : 0);
        retBytes = new byte[buffer.readableBytes()];
        buffer.readBytes(retBytes);
    } finally {
        buffer.release();
    }
    return retBytes;
}

From source file:com.addthis.hydra.task.source.SimpleMark.java

License:Apache License

@Override
public void bytesDecode(byte[] b, long version) {
    ByteBuf buffer = Unpooled.wrappedBuffer(b);
    try {//from   www .j av  a2s . c om
        int valLength = Varint.readUnsignedVarInt(buffer);
        byte[] valBytes = new byte[valLength];
        buffer.readBytes(valBytes);
        val = new String(valBytes);
        index = Varint.readUnsignedVarLong(buffer);
        end = buffer.readByte() == 1;
    } finally {
        buffer.release();
    }
}

From source file:com.addthis.meshy.Meshy.java

License:Apache License

/**
 * utility//from   w  ww .  j a v a  2s .c o m
 */
public static byte[] getBytes(int length, ByteBuf buffer) {
    byte[] request = new byte[length];
    buffer.readBytes(request);
    return request;
}