List of usage examples for io.netty.buffer ByteBuf readableBytes
public abstract int readableBytes();
From source file:alluxio.client.block.stream.NettyPacketWriterTest.java
License:Apache License
/** * Verifies the packets written. After receiving the last packet, it will also send an EOF to * the channel./*from w w w.jav a 2 s.c o m*/ * * @param checksumStart the start position to calculate the checksum * @param checksumEnd the end position to calculate the checksum * @return the checksum of the data read starting from checksumStart */ private Future<Long> verifyWriteRequests(final EmbeddedChannel channel, final long checksumStart, final long checksumEnd) { return EXECUTOR.submit(new Callable<Long>() { @Override public Long call() { try { long checksum = 0; long pos = 0; while (true) { RPCProtoMessage request = (RPCProtoMessage) CommonUtils.waitForResult("wrtie request", new Function<Void, Object>() { @Override public Object apply(Void v) { return channel.readOutbound(); } }, WaitForOptions.defaults().setTimeout(Constants.MINUTE_MS)); validateWriteRequest(request.getMessage().<Protocol.WriteRequest>getMessage(), pos); DataBuffer buffer = request.getPayloadDataBuffer(); // Last packet. if (buffer == null) { channel.writeInbound(RPCProtoMessage.createOkResponse(null)); return checksum; } try { Assert.assertTrue(buffer instanceof DataNettyBufferV2); ByteBuf buf = (ByteBuf) buffer.getNettyOutput(); while (buf.readableBytes() > 0) { if (pos >= checksumStart && pos <= checksumEnd) { checksum += BufferUtils.byteToInt(buf.readByte()); } else { buf.readByte(); } pos++; } } finally { buffer.release(); } } } catch (Throwable throwable) { LOG.error("Failed to verify write requests.", throwable); Assert.fail(); throw throwable; } } }); }
From source file:alluxio.client.block.stream.PacketInStream.java
License:Apache License
@Override public int positionedRead(long pos, byte[] b, int off, int len) throws IOException { if (len == 0) { return 0; }//from w w w . j av a 2s . c o m if (pos < 0 || pos >= mLength) { return -1; } int lenCopy = len; try (PacketReader reader = mPacketReaderFactory.create(pos, len)) { // We try to read len bytes instead of returning after reading one packet because // it is not free to create/close a PacketReader. while (len > 0) { ByteBuf buf = null; try { buf = reader.readPacket(); if (buf == null) { break; } Preconditions.checkState(buf.readableBytes() <= len); int toRead = buf.readableBytes(); buf.readBytes(b, off, toRead); len -= toRead; off += toRead; } finally { if (buf != null) { buf.release(); } } } } if (lenCopy == len) { return -1; } return lenCopy - len; }
From source file:alluxio.client.block.stream.TestDataWriter.java
License:Apache License
@Override public void writeChunk(ByteBuf chunk) throws IOException { mBuffer.limit(mBuffer.position() + chunk.readableBytes()); chunk.readBytes(mBuffer);//from ww w . j ava2 s . c om }
From source file:alluxio.client.block.stream.TestPacketWriter.java
License:Apache License
@Override public void writePacket(ByteBuf packet) throws IOException { mBuffer.limit(mBuffer.position() + packet.readableBytes()); packet.readBytes(mBuffer);/*from w ww.j a v a2 s . com*/ }
From source file:alluxio.client.block.stream.UfsFallbackLocalFilePacketWriterTest.java
License:Apache License
/** * Verifies the packets written. After receiving the last packet, it will also send an EOF to * the channel.//from w ww . j a v a2 s. co m * * @return the checksum of the data read starting from checksumStart */ private Future<WriteSummary> getUfsWrite(final EmbeddedChannel channel) { return EXECUTOR.submit(new Callable<WriteSummary>() { @Override public WriteSummary call() throws TimeoutException, InterruptedException { try { long checksum = 0; long pos = 0; long len = 0; while (true) { RPCProtoMessage request = (RPCProtoMessage) CommonUtils.waitForResult("write request", () -> channel.readOutbound(), WaitForOptions.defaults().setTimeoutMs(Constants.MINUTE_MS)); Protocol.WriteRequest writeRequest = request.getMessage().asWriteRequest(); validateWriteRequest(writeRequest, pos); DataBuffer buffer = request.getPayloadDataBuffer(); // Last packet. if (writeRequest.hasEof() && writeRequest.getEof()) { assertTrue(buffer == null); channel.writeInbound(RPCProtoMessage.createOkResponse(null)); return new WriteSummary(len, checksum); } // UFS block init if (writeRequest.getCreateUfsBlockOptions().hasBytesInBlockStore()) { assertTrue(buffer == null); pos += writeRequest.getCreateUfsBlockOptions().getBytesInBlockStore(); continue; } try { Assert.assertTrue(buffer instanceof DataNettyBufferV2); ByteBuf buf = (ByteBuf) buffer.getNettyOutput(); while (buf.readableBytes() > 0) { checksum += BufferUtils.byteToInt(buf.readByte()); pos++; len++; } } finally { buffer.release(); } } } catch (Throwable throwable) { fail("Failed to verify write requests." + throwable.getMessage()); throw throwable; } } }); }
From source file:alluxio.worker.block.io.LocalFileBlockWriter.java
License:Apache License
@Override public void transferFrom(ByteBuf buf) throws IOException { mPosition += buf.readBytes(mLocalFileChannel, buf.readableBytes()); }
From source file:alluxio.worker.block.io.MockBlockReader.java
License:Apache License
@Override public int transferTo(ByteBuf buf) throws IOException { int remaining = buf.readableBytes(); return buf.writeBytes(mBytes).readableBytes() - remaining; }
From source file:alluxio.worker.block.io.MockBlockWriter.java
License:Apache License
@Override public void transferFrom(ByteBuf buf) throws IOException { mPosition += buf.readBytes(getChannel(), buf.readableBytes()); }
From source file:alluxio.worker.block.UnderFileSystemBlockReader.java
License:Apache License
/** * This interface is supposed to be used for sequence block reads. * * @param buf the byte buffer/* w w w. j a v a2 s .com*/ * @return the number of bytes read, -1 if it reaches EOF and none was read * @throws IOException if any I/O errors occur when reading the block */ @Override public int transferTo(ByteBuf buf) throws IOException { Preconditions.checkState(!mClosed); if (mUnderFileSystemInputStream == null) { return -1; } if (mBlockMeta.getBlockSize() <= mInStreamPos) { return -1; } // Make a copy of the state to keep track of what we have read in this transferTo call. ByteBuf bufCopy = null; if (mBlockWriter != null) { bufCopy = buf.duplicate(); bufCopy.readerIndex(bufCopy.writerIndex()); } int bytesToRead = (int) Math.min((long) buf.writableBytes(), mBlockMeta.getBlockSize() - mInStreamPos); int bytesRead = buf.writeBytes(mUnderFileSystemInputStream, bytesToRead); if (bytesRead <= 0) { return bytesRead; } mInStreamPos += bytesRead; if (mBlockWriter != null) { bufCopy.writerIndex(buf.writerIndex()); while (bufCopy.readableBytes() > 0) { mBlockWriter.transferFrom(bufCopy); } } return bytesRead; }
From source file:alluxio.worker.netty.AbstractWriteHandler.java
License:Apache License
@Override public void channelRead(ChannelHandlerContext ctx, Object object) throws Exception { if (!acceptMessage(object)) { ctx.fireChannelRead(object);//from ww w .j a v a 2s.c o m return; } RPCProtoMessage msg = (RPCProtoMessage) object; Protocol.WriteRequest writeRequest = msg.getMessage().asWriteRequest(); try (LockResource lr = new LockResource(mLock)) { boolean isNewContextCreated = false; if (mContext == null || mContext.isDoneUnsafe()) { // We create a new context if the previous request completes (done flag is true) or the // context is still null (an empty channel so far). And in this case, we create a new one as // catching exceptions and replying errors // leverages data structures in context, regardless of the request is valid or not. // TODO(binfan): remove the dependency on an instantiated request context which is required // to reply errors to client side. mContext = createRequestContext(writeRequest); isNewContextCreated = true; } // Only initialize (open the writers) if this is the first packet in the block/file. if (writeRequest.getOffset() == 0) { // Expected state: context equals null as this handler is new for request, or the previous // context is not active (done / cancel / abort). Otherwise, notify the client an illegal // state. Note that, we reset the context before validation msg as validation may require to // update error in context. Preconditions.checkState(isNewContextCreated); initRequestContext(mContext); } // If we have seen an error, return early and release the data. This can // happen for (1) those mis-behaving clients who first sends some invalid requests, then // then some random data, or (2) asynchronous requests arrive after the previous request fails // and triggers abortion. It can leak memory if we do not release buffers here. if (mContext.getError() != null) { if (msg.getPayloadDataBuffer() != null) { msg.getPayloadDataBuffer().release(); } LOG.warn("Ignore the request {} due to the error {} on context", mContext.getRequest(), mContext.getError()); return; } else { // Validate the write request. The validation is performed only when no error is in the // context in order to prevent excessive logging on the subsequent arrived asynchronous // requests after a previous request fails and triggers the abortion validateWriteRequest(writeRequest, msg.getPayloadDataBuffer()); } ByteBuf buf; if (writeRequest.getEof()) { buf = EOF; } else if (writeRequest.getCancel()) { buf = CANCEL; } else { DataBuffer dataBuffer = msg.getPayloadDataBuffer(); Preconditions.checkState(dataBuffer != null && dataBuffer.getLength() > 0); Preconditions.checkState(dataBuffer.getNettyOutput() instanceof ByteBuf); buf = (ByteBuf) dataBuffer.getNettyOutput(); mContext.setPosToQueue(mContext.getPosToQueue() + buf.readableBytes()); } if (!mContext.isPacketWriterActive()) { mContext.setPacketWriterActive(true); mPacketWriterExecutor.submit(createPacketWriter(mContext, ctx.channel())); } mContext.getPackets().offer(buf); if (tooManyPacketsInFlight()) { NettyUtils.disableAutoRead(ctx.channel()); } } }