Example usage for io.netty.buffer ByteBuf arrayOffset

List of usage examples for io.netty.buffer ByteBuf arrayOffset

  1. HOME
  2. Java
  3. io.netty
  4. io.netty.buffer.*
  5. ByteBuf
  6. arrayOffset

Introduction

In this page you can find the example usage for io.netty.buffer ByteBuf arrayOffset.

Prototype

public abstract int arrayOffset();

Source Link

Document

Returns the offset of the first byte within the backing byte array of this buffer.

Usage

From source file:com.yahoo.pulsar.common.compression.Crc32cChecksumTest.java

License:Apache License

@Test
public void testCrc32cSoftware() {
    ByteBuf payload = Unpooled.wrappedBuffer(inputBytes);

    // compute checksum using sw algo
    int sw = SOFTWARE_CRC32C_HASH.calculate(payload.array(), payload.arrayOffset() + payload.readerIndex(),
            payload.readableBytes());/*from   w  ww.  java 2s  . c  o  m*/
    assertEquals(sw, expectedChecksum);
}

From source file:com.yahoo.pulsar.common.util.XXHashChecksum.java

License:Apache License

public static long computeChecksum(ByteBuf payload) {
    if (payload.hasArray()) {
        return checksum.hash(payload.array(), payload.arrayOffset() + payload.readerIndex(),
                payload.readableBytes(), 0L);
    } else {//w w w.j  a  v a  2 s. c  o  m
        ByteBuffer payloadNio = payload.nioBuffer(payload.readerIndex(), payload.readableBytes());
        return checksum.hash(payloadNio, 0, payload.readableBytes(), 0L);
    }
}

From source file:de.dfki.kiara.netty.ByteBufferDecoder.java

License:Open Source License

@Override
protected void decode(ChannelHandlerContext ctx, ByteBuf msg, List<Object> out) throws Exception {
    final byte[] array;
    final int offset;
    final int length = msg.readableBytes();
    if (msg.hasArray()) {
        array = msg.array();//from   www. j a v  a 2 s .c om
        offset = msg.arrayOffset() + msg.readerIndex();
    } else {
        array = new byte[length];
        msg.getBytes(msg.readerIndex(), array, 0, length);
        offset = 0;
    }

    out.add(ByteBuffer.wrap(array, offset, length));
}

From source file:divconq.ctp.stream.FileSourceStream.java

License:Open Source License

public void readLocalFile() {
    FileSystemFile fs = (FileSystemFile) this.current;

    if (this.in == null) {
        this.insize = fs.getSize();

        // As a source we are responsible for progress tracking
        OperationContext.get().setAmountCompleted(0);

        try {//from  w  ww  .  j  a v a 2 s .com
            this.in = FileChannel.open(fs.localPath(), StandardOpenOption.READ);
        } catch (IOException x) {
            OperationContext.get().getTaskRun().kill("Unable to read source file " + x);
            return;
        }
    }

    while (true) {
        // TODO sizing?
        ByteBuf data = Hub.instance.getBufferAllocator().heapBuffer(32768);

        ByteBuffer buffer = ByteBuffer.wrap(data.array(), data.arrayOffset(), data.capacity());

        int pos = -1;

        try {
            pos = (int) this.in.read(buffer);
        } catch (IOException x1) {
            OperationContext.get().getTaskRun().kill("Problem reading source file: " + x1);
            data.release();
            return;
        }

        FileDescriptor fref = FileDescriptor.fromFileStore(this.current);
        fref.setPath(this.current.path().subpath(this.source.path()));

        System.out.println("writing: " + fref.getPath() + " from: " + this.inprog);

        if (pos == -1) {
            try {
                this.in.close();
            } catch (IOException x) {
                OperationContext.get().getTaskRun().kill("Problem closing source file: " + x);
                data.release();
                return;
            }

            OperationContext.get().setAmountCompleted(100);

            fref.setEof(true);

            this.current = null;
            this.in = null;
            this.insize = 0;
            this.inprog = 0;
        } else {
            this.inprog += pos;

            data.writerIndex(pos);
            OperationContext.get().setAmountCompleted((int) (this.inprog * 100 / this.insize));
        }

        if (this.downstream.handle(fref, data) != ReturnOption.CONTINUE)
            break;

        if (this.current == null) {
            // we need the next file
            OperationContext.get().getTaskRun().resume();

            // wait on the implied request
            break;
        }
    }
}

From source file:divconq.ctp.stream.GzipStream.java

License:Open Source License

@Override
public ReturnOption handle(FileDescriptor file, ByteBuf data) {
    if (file == FileDescriptor.FINAL)
        return this.downstream.handle(file, data);

    // we don't know what to do with a folder at this stage - gzip is for file content only
    // folder scanning is upstream in the FileSourceStream and partners
    if (file.isFolder())
        return ReturnOption.CONTINUE;

    // init if not set for this round of processing 
    if (this.deflater == null) {
        this.deflater = new Deflater(this.compressionLevel, true);
        this.crc.reset();
        this.writeHeader = true;
    }/*from w  w w .j  a va 2s.c  o  m*/

    ByteBuf in = data;
    ByteBuf out = null;

    if (in != null) {
        byte[] inAry = in.array();

        // always allow for a header (10) plus footer (8) plus extra (12)
        // in addition to content
        int sizeEstimate = (int) Math.ceil(in.readableBytes() * 1.001) + 30;
        out = Hub.instance.getBufferAllocator().heapBuffer(sizeEstimate);

        if (this.writeHeader) {
            this.writeHeader = false;
            out.writeBytes(gzipHeader);
        }

        this.crc.update(inAry, in.arrayOffset(), in.writerIndex());

        this.deflater.setInput(inAry, in.arrayOffset(), in.writerIndex());

        while (!this.deflater.needsInput())
            deflate(out);
    } else
        out = Hub.instance.getBufferAllocator().heapBuffer(30);

    FileDescriptor blk = new FileDescriptor();

    if (StringUtil.isEmpty(this.lastpath)) {
        if (StringUtil.isNotEmpty(this.nameHint))
            this.lastpath = "/" + this.nameHint;
        else if (file.getPath() != null)
            this.lastpath = "/" + GzipUtils.getCompressedFilename(file.path().getFileName());
        else
            this.lastpath = "/" + FileUtil.randomFilename("gz");
    }

    blk.setPath(this.lastpath);

    file.setModTime(System.currentTimeMillis());

    if (file.isEof()) {
        this.deflater.finish();

        while (!this.deflater.finished())
            deflate(out);

        int crcValue = (int) this.crc.getValue();

        out.writeByte(crcValue);
        out.writeByte(crcValue >>> 8);
        out.writeByte(crcValue >>> 16);
        out.writeByte(crcValue >>> 24);

        int uncBytes = this.deflater.getTotalIn();

        out.writeByte(uncBytes);
        out.writeByte(uncBytes >>> 8);
        out.writeByte(uncBytes >>> 16);
        out.writeByte(uncBytes >>> 24);

        this.deflater.end();
        this.deflater = null; // cause a reset for next time we use stream

        blk.setEof(true);
    }

    if (in != null)
        in.release();

    return this.downstream.handle(blk, out);
}

From source file:divconq.ctp.stream.GzipStream.java

License:Open Source License

protected void deflate(ByteBuf out) {
    int numBytes = 0;

    do {//from ww  w  .j  av a2  s  .  c  o  m
        byte[] o = out.array();

        numBytes = this.deflater.deflate(o, out.arrayOffset() + out.writerIndex(), out.writableBytes(),
                Deflater.SYNC_FLUSH);

        out.writerIndex(out.writerIndex() + numBytes);
    } while (numBytes > 0);
}

From source file:divconq.ctp.stream.TarStream.java

License:Open Source License

@Override
public ReturnOption handle(FileDescriptor file, ByteBuf data) {
    if (file == FileDescriptor.FINAL) {
        if (this.tstream == null)
            return this.downstream.handle(file, data);

        this.finalflag = true;
    }/*  w  w w  .jav  a  2 s. c  o m*/

    // I don't think tar cares about folder entries at this stage - tar is for file content only
    // folder scanning is upstream in the FileSourceStream and partners
    // TODO try with ending / to file name
    if (file.isFolder())
        return ReturnOption.CONTINUE;

    // init if not set for this round of processing 
    if (this.tstream == null) {
        this.bstream = new CyclingByteBufferOutputStream();
        this.tstream = new TarArchiveOutputStream(this.bstream);
        this.tstream.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU);
    }

    ByteBuf in = data;
    ByteBuf out = null;

    // always allow for a header (512) and/or footer (1024) in addition to content
    int sizeEstimate = (in != null) ? in.readableBytes() + 2048 : 2048;
    out = Hub.instance.getBufferAllocator().heapBuffer(sizeEstimate);

    this.bstream.installBuffer(out);

    // TODO if there is no output available to send and not EOF then just request more,
    // no need to send a message that is empty and not EOF

    FileDescriptor blk = new FileDescriptor();

    if (StringUtil.isNotEmpty(this.lastpath)) {
        blk.setPath(this.lastpath);
    } else {
        if (file.hasPath())
            this.lastpath = "/"
                    + (StringUtil.isNotEmpty(this.nameHint) ? this.nameHint : file.path().getFileName())
                    + ".tar";
        else if (StringUtil.isNotEmpty(this.nameHint))
            this.lastpath = "/" + this.nameHint + ".tar";
        else
            this.lastpath = "/" + FileUtil.randomFilename() + ".tar";

        blk.setPath(this.lastpath);
    }

    blk.setModTime(System.currentTimeMillis());

    if (!this.archiveopenflag && !this.finalflag) {
        TarArchiveEntry tentry = new TarArchiveEntry(file.getPath().toString().substring(1), true);
        tentry.setSize(file.getSize());
        tentry.setModTime(file.getModTime());

        try {
            this.tstream.putArchiveEntry(tentry);
        } catch (IOException x) {
            if (in != null)
                in.release();

            out.release();
            OperationContext.get().getTaskRun().kill("Problem writing tar entry: " + x);
            return ReturnOption.DONE;
        }

        this.archiveopenflag = true;
    }

    if (in != null)
        try {
            this.tstream.write(in.array(), in.arrayOffset(), in.writerIndex());
        } catch (IOException x) {
            in.release();
            out.release();
            OperationContext.get().getTaskRun().kill("Problem writing tar body: " + x);
            return ReturnOption.DONE;
        }

    if (file.isEof()) {
        try {
            this.tstream.closeArchiveEntry();
        } catch (IOException x) {
            if (in != null)
                in.release();

            out.release();
            OperationContext.get().getTaskRun().kill("Problem closing tar entry: " + x);
            return ReturnOption.DONE;
        }

        this.archiveopenflag = false;
    }

    if (in != null)
        in.release();

    if (file == FileDescriptor.FINAL) {
        blk.setEof(true);

        try {
            this.tstream.close();
        } catch (IOException x) {
            //in.release();
            out.release();
            OperationContext.get().getTaskRun().kill("Problem closing tar stream: " + x);
            return ReturnOption.DONE;
        }

        this.tstream = null;
        this.bstream = null;
    } else
        this.bstream.uninstallBuffer(); // we are done with out forever, don't reference it

    System.out.println("tar sending: " + out.readableBytes());

    ReturnOption v = this.downstream.handle(blk, out);

    if (!this.finalflag)
        return v;

    if (v == ReturnOption.CONTINUE)
        return this.downstream.handle(FileDescriptor.FINAL, null);

    return ReturnOption.DONE;
}

From source file:divconq.ctp.stream.UngzipStream.java

License:Open Source License

protected void inflate(ByteBuf in) {
    switch (this.gzipState) {
    case HEADER_START:
        if (in.readableBytes() < 10)
            return;

        // read magic numbers
        int magic0 = in.readByte();
        int magic1 = in.readByte();

        if (magic0 != 31) {
            OperationContext.get().getTaskRun().kill("Input is not in the GZIP format");
            return;
        }/*w  w  w. j  ava 2 s  .c  o  m*/

        this.crc.update(magic0);
        this.crc.update(magic1);

        int method = in.readUnsignedByte();

        if (method != Deflater.DEFLATED) {
            OperationContext.get().getTaskRun()
                    .kill("Unsupported compression method " + method + " in the GZIP header");
            return;
        }

        this.crc.update(method);

        this.flags = in.readUnsignedByte();
        this.crc.update(this.flags);

        if ((this.flags & FRESERVED) != 0) {
            OperationContext.get().getTaskRun().kill("Reserved flags are set in the GZIP header");
            return;
        }

        // mtime (int)
        this.crc.update(in.readByte());
        this.crc.update(in.readByte());
        this.crc.update(in.readByte());
        this.crc.update(in.readByte());

        this.crc.update(in.readUnsignedByte()); // extra flags
        this.crc.update(in.readUnsignedByte()); // operating system

        this.gzipState = GzipState.FLG_READ;
    case FLG_READ:
        if ((this.flags & FEXTRA) != 0) {
            if (in.readableBytes() < 2)
                return;

            int xlen1 = in.readUnsignedByte();
            int xlen2 = in.readUnsignedByte();

            this.crc.update(xlen1);
            this.crc.update(xlen2);

            this.xlen |= xlen1 << 8 | xlen2;
        }

        this.gzipState = GzipState.XLEN_READ;
    case XLEN_READ:
        if (this.xlen != -1) {
            if (in.readableBytes() < xlen)
                return;

            byte[] xtra = new byte[xlen];
            in.readBytes(xtra);
            this.crc.update(xtra);
        }

        this.gzipState = GzipState.SKIP_FNAME;
    case SKIP_FNAME:
        if ((this.flags & FNAME) != 0) {
            boolean gotend = false;

            while (in.isReadable()) {
                int b = in.readUnsignedByte();
                this.crc.update(b);

                if (b == 0x00) {
                    gotend = true;
                    break;
                }
            }

            if (!gotend)
                return;
        }

        this.gzipState = GzipState.SKIP_COMMENT;
    case SKIP_COMMENT:
        if ((this.flags & FCOMMENT) != 0) {
            boolean gotend = false;

            while (in.isReadable()) {
                int b = in.readUnsignedByte();
                this.crc.update(b);

                if (b == 0x00) {
                    gotend = true;
                    break;
                }
            }

            if (!gotend)
                return;
        }

        this.gzipState = GzipState.PROCESS_FHCRC;
    case PROCESS_FHCRC:
        if ((this.flags & FHCRC) != 0) {
            if (in.readableBytes() < 4)
                return;

            long crcValue = 0;

            for (int i = 0; i < 4; ++i)
                crcValue |= (long) in.readUnsignedByte() << i * 8;

            long readCrc = crc.getValue();

            if (crcValue != readCrc) {
                OperationContext.get().getTaskRun()
                        .kill("CRC value missmatch. Expected: " + crcValue + ", Got: " + readCrc);
                return;
            }
        }

        this.crc.reset();

        this.gzipState = GzipState.PRROCESS_CONTENT;
    case PRROCESS_CONTENT:
        int readableBytes = in.readableBytes();

        if (readableBytes < 1)
            return;

        if (in.hasArray()) {
            this.inflater.setInput(in.array(), in.arrayOffset() + in.readerIndex(), readableBytes);
        } else {
            byte[] array = new byte[readableBytes];
            in.getBytes(in.readerIndex(), array);
            this.inflater.setInput(array);
        }

        int maxOutputLength = this.inflater.getRemaining() << 1;
        ByteBuf decompressed = Hub.instance.getBufferAllocator().heapBuffer(maxOutputLength);

        boolean readFooter = false;
        byte[] outArray = decompressed.array();

        try {
            while (!this.inflater.needsInput()) {
                int writerIndex = decompressed.writerIndex();
                int outIndex = decompressed.arrayOffset() + writerIndex;
                int length = decompressed.writableBytes();

                if (length == 0) {
                    // completely filled the buffer allocate a new one and start to fill it
                    this.outlist.add(decompressed);
                    decompressed = Hub.instance.getBufferAllocator().heapBuffer(maxOutputLength);
                    outArray = decompressed.array();
                    continue;
                }

                int outputLength = this.inflater.inflate(outArray, outIndex, length);

                if (outputLength > 0) {
                    decompressed.writerIndex(writerIndex + outputLength);

                    this.crc.update(outArray, outIndex, outputLength);
                } else {
                    if (this.inflater.needsDictionary()) {
                        if (this.dictionary == null) {
                            OperationContext.get().getTaskRun().kill(
                                    "decompression failure, unable to set dictionary as non was specified");
                            return;
                        }

                        this.inflater.setDictionary(this.dictionary);
                    }
                }

                if (this.inflater.finished()) {
                    readFooter = true;
                    break;
                }
            }

            in.skipBytes(readableBytes - this.inflater.getRemaining());
        } catch (DataFormatException x) {
            OperationContext.get().getTaskRun().kill("decompression failure: " + x);
            return;
        } finally {
            if (decompressed.isReadable()) {
                this.outlist.add(decompressed);
            } else {
                decompressed.release();
            }
        }

        if (!readFooter)
            break;

        this.gzipState = GzipState.PROCESS_FOOTER;
    case PROCESS_FOOTER:
        if (in.readableBytes() < 8)
            return;

        long crcValue = 0;

        for (int i = 0; i < 4; ++i)
            crcValue |= (long) in.readUnsignedByte() << i * 8;

        long readCrc = this.crc.getValue();

        if (crcValue != readCrc) {
            OperationContext.get().getTaskRun()
                    .kill("CRC value missmatch. Expected: " + crcValue + ", Got: " + readCrc);
            return;
        }

        // read ISIZE and verify
        int dataLength = 0;

        for (int i = 0; i < 4; ++i)
            dataLength |= in.readUnsignedByte() << i * 8;

        int readLength = this.inflater.getTotalOut();

        if (dataLength != readLength) {
            OperationContext.get().getTaskRun()
                    .kill("Number of bytes mismatch. Expected: " + dataLength + ", Got: " + readLength);
            return;
        }

        this.gzipState = GzipState.DONE;
    case DONE:
        break;
    }
}

From source file:divconq.pgp.EncryptedFileStream.java

License:Open Source License

public void writeData(byte[] bytes, int offset, int len) {
    // the first time this is called we need to write headers - those headers
    // call into this method so clear flag immediately
    if (!this.writeFirst) {
        this.writeFirst = true;
        this.writeFirstLiteral(len);
    }//from   w w  w  .  j  a  v  a  2 s.  co  m

    int remaining = len;
    int avail = this.packetsize - this.packetpos;

    // packetbuf may have data that has not yet been processed, so if we are doing any writes
    // we need to write the packet buffer first
    ByteBuf pbb = this.packetbuf;

    if (pbb != null) {
        int bbremaining = pbb.readableBytes();

        // only write if there is space available in current packet or if we have a total
        // amount of data larger than max packet size
        while ((bbremaining > 0) && ((avail > 0) || (bbremaining + remaining) >= MAX_PACKET_SIZE)) {
            // out of current packet space? create more packets
            if (avail == 0) {
                this.packetsize = MAX_PACKET_SIZE;
                this.packetpos = 0;

                this.writeDataInternal((byte) MAX_PARTIAL_LEN); // partial packet length

                avail = this.packetsize;
            }

            // figure out how much we can write to the current packet, write it, update indexes
            int alen = Math.min(avail, bbremaining);

            this.writeDataInternal(pbb.array(), pbb.arrayOffset() + pbb.readerIndex(), alen);

            pbb.skipBytes(alen);
            bbremaining = pbb.readableBytes();
            this.packetpos += alen;
            avail = this.packetsize - this.packetpos;

            // our formula always assumes that packetbuf starts at zero offset, anytime
            // we write out part of the packetbuf we either need to write it all and clear it
            // or we need to start with a new buffer with data starting at offset 0
            if (bbremaining == 0) {
                pbb.clear();
            } else {
                ByteBuf npb = Hub.instance.getBufferAllocator().heapBuffer(MAX_PACKET_SIZE);
                npb.writeBytes(pbb, bbremaining);
                this.packetbuf = npb;

                pbb.release();
                pbb = npb;
            }
        }
    }

    // only write if there is space available in current packet or if we have a total
    // amount of data larger than max packet size
    while ((remaining > 0) && ((avail > 0) || (remaining >= MAX_PACKET_SIZE))) {
        // out of current packet space? create more packets
        if (avail == 0) {
            this.packetsize = MAX_PACKET_SIZE;
            this.packetpos = 0;

            this.writeDataInternal((byte) MAX_PARTIAL_LEN); // partial packet length

            avail = this.packetsize;
        }

        // figure out how much we can write to the current packet, write it, update indexes
        int alen = Math.min(avail, remaining);

        this.writeDataInternal(bytes, offset, alen);

        remaining -= alen;
        offset += alen;
        this.packetpos += alen;
        avail = this.packetsize - this.packetpos;
    }

    // buffer remaining to build larger packet later
    if (remaining > 0) {
        if (this.packetbuf == null)
            this.packetbuf = Hub.instance.getBufferAllocator().heapBuffer(MAX_PACKET_SIZE);

        // add to new buffer or add to existing buffer, either way it should be less than max here
        this.packetbuf.writeBytes(bytes, offset, remaining);
    }
}

From source file:divconq.pgp.EncryptedFileStream.java

License:Open Source License

public void writeData(ByteBuf buf) {
    this.writeData(buf.array(), buf.arrayOffset() + buf.readerIndex(), buf.readableBytes());
}