List of usage examples for java.nio ByteBuffer limit
public final int limit()
From source file:com.newatlanta.appengine.nio.channels.GaeFileChannel.java
@Override public synchronized int write(ByteBuffer src) throws IOException { checkWriteOptions();/*from w w w. j av a 2 s .c o m*/ int bytesWritten = 0; while (src.hasRemaining()) { int r = src.remaining(); initBuffer(r); if (calcBlockIndex(position + r - 1) == index) { // writing entirely within current block bytesWritten += writeBuffer(src); } else { // fill the current block then repeat loop int limit = src.limit(); src.limit(src.position() + (blockSize - buffer.position())); bytesWritten += writeBuffer(src); src.limit(limit); } } //flush(); return bytesWritten; }
From source file:org.apache.hadoop.hbase.ipc.BlockingRpcConnection.java
/** * Initiates a call by sending the parameter to the remote server. Note: this is not called from * the Connection thread, but by other threads. * @see #readResponse()//from w w w . j a v a 2 s .c o m */ private void writeRequest(Call call) throws IOException { ByteBuffer cellBlock = this.rpcClient.cellBlockBuilder.buildCellBlock(this.codec, this.compressor, call.cells); CellBlockMeta cellBlockMeta; if (cellBlock != null) { cellBlockMeta = CellBlockMeta.newBuilder().setLength(cellBlock.limit()).build(); } else { cellBlockMeta = null; } RequestHeader requestHeader = buildRequestHeader(call, cellBlockMeta); setupIOstreams(); // Now we're going to write the call. We take the lock, then check that the connection // is still valid, and, if so we do the write to the socket. If the write fails, we don't // know where we stand, we have to close the connection. if (Thread.interrupted()) { throw new InterruptedIOException(); } calls.put(call.id, call); // We put first as we don't want the connection to become idle. // from here, we do not throw any exception to upper layer as the call has been tracked in the // pending calls map. try { call.callStats.setRequestSizeBytes(write(this.out, requestHeader, call.param, cellBlock)); } catch (IOException e) { closeConn(e); return; } notifyAll(); }
From source file:de.rub.nds.burp.espresso.gui.attacker.saml.UIDTDAttack.java
/** * Notify all registered listeners with the new code. * @param evt The new source code.//from w w w . j ava 2 s.c om */ @Override public void notifyAllTabs(AbstractCodeEvent evt) { if (evt instanceof SamlCodeEvent) { String code = evt.getCode(); if (listeners != null) { // Encode dtd vector if needed switch (EncodingType.fromString(encodingButtonGroup.getSelection().getActionCommand())) { case UTF_7: Charset charset = new CharsetProvider().charsetForName("UTF-7"); ByteBuffer byteBuffer = charset.encode(code); code = new String(byteBuffer.array()).substring(0, byteBuffer.limit()); break; case UTF_8: try { code = new String(code.getBytes("UTF-8"), "UTF-8"); } catch (UnsupportedEncodingException ex) { Logging.getInstance().log(getClass(), ex); } break; case UTF_16: try { code = new String(code.getBytes("UTF-8"), "UTF-16"); } catch (UnsupportedEncodingException ex) { Logging.getInstance().log(getClass(), ex); } break; default: break; } listeners.notifyAll(new SamlCodeEvent(this, code)); } } else { listeners.notifyAll(evt); } }
From source file:com.github.hrpc.rpc.Server.java
/** * Helper for {@link #channelRead(ReadableByteChannel, ByteBuffer)} * and {@link #channelWrite(WritableByteChannel, ByteBuffer)}. Only * one of readCh or writeCh should be non-null. * * @see #channelRead(ReadableByteChannel, ByteBuffer) * @see #channelWrite(WritableByteChannel, ByteBuffer) *//*from w w w . j av a 2 s. c om*/ private static int channelIO(ReadableByteChannel readCh, WritableByteChannel writeCh, ByteBuffer buf) throws IOException { int originalLimit = buf.limit(); int initialRemaining = buf.remaining(); int ret = 0; while (buf.remaining() > 0) { try { int ioSize = Math.min(buf.remaining(), NIO_BUFFER_LIMIT); buf.limit(buf.position() + ioSize); ret = (readCh == null) ? writeCh.write(buf) : readCh.read(buf); if (ret < ioSize) { break; } } finally { buf.limit(originalLimit); } } int nBytes = initialRemaining - buf.remaining(); return (nBytes > 0) ? nBytes : ret; }
From source file:org.apache.hadoop.hbase.ipc.AsyncRpcChannel.java
/** * Write request to channel/*from w ww . j a v a2 s . com*/ * * @param call to write */ private void writeRequest(final AsyncCall call) { try { final RPCProtos.RequestHeader.Builder requestHeaderBuilder = RPCProtos.RequestHeader.newBuilder(); requestHeaderBuilder.setCallId(call.id).setMethodName(call.method.getName()) .setRequestParam(call.param != null); if (Trace.isTracing()) { Span s = Trace.currentSpan(); requestHeaderBuilder.setTraceInfo( TracingProtos.RPCTInfo.newBuilder().setParentId(s.getSpanId()).setTraceId(s.getTraceId())); } ByteBuffer cellBlock = client.buildCellBlock(call.controller.cellScanner()); if (cellBlock != null) { final RPCProtos.CellBlockMeta.Builder cellBlockBuilder = RPCProtos.CellBlockMeta.newBuilder(); cellBlockBuilder.setLength(cellBlock.limit()); requestHeaderBuilder.setCellBlockMeta(cellBlockBuilder.build()); } // Only pass priority if there one. Let zero be same as no priority. if (call.controller.getPriority() != 0) { requestHeaderBuilder.setPriority(call.controller.getPriority()); } RPCProtos.RequestHeader rh = requestHeaderBuilder.build(); int totalSize = IPCUtil.getTotalSizeWhenWrittenDelimited(rh, call.param); if (cellBlock != null) { totalSize += cellBlock.remaining(); } ByteBuf b = channel.alloc().directBuffer(4 + totalSize); try (ByteBufOutputStream out = new ByteBufOutputStream(b)) { IPCUtil.write(out, rh, call.param, cellBlock); } channel.writeAndFlush(b).addListener(new CallWriteListener(this, call.id)); } catch (IOException e) { close(e); } }
From source file:org.apache.hadoop.hbase.ipc.AsyncRpcChannelImpl.java
/** * Write request to channel// w ww. j a v a 2 s . c om * @param call to write */ private void writeRequest(final AsyncCall call) { try { final RPCProtos.RequestHeader.Builder requestHeaderBuilder = RPCProtos.RequestHeader.newBuilder(); requestHeaderBuilder.setCallId(call.id).setMethodName(call.method.getName()) .setRequestParam(call.param != null); if (Trace.isTracing()) { Span s = Trace.currentSpan(); requestHeaderBuilder.setTraceInfo( TracingProtos.RPCTInfo.newBuilder().setParentId(s.getSpanId()).setTraceId(s.getTraceId())); } ByteBuffer cellBlock = client.buildCellBlock(call.cellScanner()); if (cellBlock != null) { final RPCProtos.CellBlockMeta.Builder cellBlockBuilder = RPCProtos.CellBlockMeta.newBuilder(); cellBlockBuilder.setLength(cellBlock.limit()); requestHeaderBuilder.setCellBlockMeta(cellBlockBuilder.build()); } // Only pass priority if there one. Let zero be same as no priority. if (call.getPriority() != PayloadCarryingRpcController.PRIORITY_UNSET) { requestHeaderBuilder.setPriority(call.getPriority()); } requestHeaderBuilder .setTimeout(call.rpcTimeout > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) call.rpcTimeout); RPCProtos.RequestHeader rh = requestHeaderBuilder.build(); int totalSize = IPCUtil.getTotalSizeWhenWrittenDelimited(rh, call.param); if (cellBlock != null) { totalSize += cellBlock.remaining(); } ByteBuf b = channel.alloc().directBuffer(4 + totalSize); try (ByteBufOutputStream out = new ByteBufOutputStream(b)) { call.callStats.setRequestSizeBytes(IPCUtil.write(out, rh, call.param, cellBlock)); } channel.writeAndFlush(b).addListener(new CallWriteListener(this, call.id)); } catch (IOException e) { close(e); } }
From source file:fuse.okuyamafs.OkuyamaFilesystem.java
public int write_bk(String path, Object fh, boolean isWritepage, ByteBuffer buf, long offset) throws FuseException { byte[] tmpBuf = new byte[buf.limit()]; buf.get(tmpBuf);/* w w w .j av a 2s .c o m*/ return 0; }
From source file:fuse.okuyamafs.OkuyamaFilesystem.java
public int realWrite(String path, Object fh, boolean isWritepage, ByteBuffer buf, long offset) throws FuseException { byte[] writeData = new byte[buf.limit()]; buf.get(writeData);//from w w w. j a v a 2 s .co m return realWrite(path, fh, isWritepage, writeData, offset); }
From source file:org.alfresco.contentstore.patch.PatchServiceImpl.java
private void updatePatchDocument(PatchDocument patchDocument, NodeChecksums checksums, Reader reader) throws IOException { ByteBuffer data = ByteBuffer.allocate(blockSize * 20); int blockSize = checksums.getBlockSize(); int i = 0;//w w w . ja v a 2s.co m Adler32 adlerInfo = new Adler32(hasher); int lastMatchIndex = 1; // starts at 1 ByteBuffer currentPatch = ByteBuffer.allocate(5000000); // TODO int x = 0; for (;;) { if (x == 0 || i >= data.limit()) { data.clear(); i = 0; int numRead = reader.read(data); if (numRead <= 0) { break; } data.flip(); x += numRead; } int chunkSize = 0; // determine the size of the next data chuck to evaluate. Default to // blockSize, but clamp to end of data if ((i + blockSize) > data.limit()) { chunkSize = data.limit() - i; adlerInfo.reset(); // need to reset this because the rolling // checksum doesn't work correctly on a final // non-aligned block } else { chunkSize = blockSize; } int end = i + chunkSize - 1; int matchedBlockIndex = adlerInfo.checkMatch(lastMatchIndex, checksums, data, i, end); if (matchedBlockIndex != -1) { // try // { // String y = hasher.md5(data, i, end); // System.out.println("y = " + y + ", x = " + x + ", i = " + i + ", end = " + end); // } // catch (NoSuchAlgorithmException e) // { // // TODO Auto-generated catch block // e.printStackTrace(); // } // if we have a match, do the following: // 1) add the matched block index to our tracking buffer // 2) check to see if there's a current patch. If so, add it to // the patch document. // 3) jump forward blockSize bytes and continue patchDocument.addMatchedBlock(matchedBlockIndex); if (currentPatch.position() > 0) { // there are outstanding patches, add them to the list // create the patch and append it to the patches buffer currentPatch.flip(); int size = currentPatch.limit(); byte[] dst = new byte[size]; currentPatch.get(dst, 0, size); Patch patch = new Patch(lastMatchIndex, size, dst); patchDocument.addPatch(patch); currentPatch.clear(); } lastMatchIndex = matchedBlockIndex; i += chunkSize; adlerInfo.reset(); } else { // while we don't have a block match, append bytes to the // current patch if (currentPatch.position() >= currentPatch.limit()) { // System.out.println("count=" + (x + i)); // System.out.println("count1=" + currentPatch.position() + ", " + currentPatch.limit()); // System.out.println(matchedBlockIndexes); // System.out.println(patches); } currentPatch.put(data.get(i)); i++; } } // end for each byte in the data if (currentPatch.position() > 0) { currentPatch.flip(); int size = currentPatch.limit(); byte[] dst = new byte[size]; currentPatch.get(dst, 0, size); Patch patch = new Patch(lastMatchIndex, size, dst); patchDocument.addPatch(patch); } }
From source file:org.apache.kylin.storage.hbase.cube.v2.CubeHBaseEndpointRPC.java
@SuppressWarnings("checkstyle:methodlength") @Override// w w w . ja v a 2s . co m public IGTScanner getGTScanner(final GTScanRequest scanRequest) throws IOException { final String toggle = BackdoorToggles.getCoprocessorBehavior() == null ? CoprocessorBehavior.SCAN_FILTER_AGGR_CHECKMEM.toString() : BackdoorToggles.getCoprocessorBehavior(); logger.debug("New scanner for current segment {} will use {} as endpoint's behavior", cubeSeg, toggle); Pair<Short, Short> shardNumAndBaseShard = getShardNumAndBaseShard(); short shardNum = shardNumAndBaseShard.getFirst(); short cuboidBaseShard = shardNumAndBaseShard.getSecond(); int totalShards = cubeSeg.getTotalShards(); ByteString scanRequestByteString = null; ByteString rawScanByteString = null; // primary key (also the 0th column block) is always selected final ImmutableBitSet selectedColBlocks = scanRequest.getSelectedColBlocks().set(0); // globally shared connection, does not require close final HConnection conn = HBaseConnection.get(cubeSeg.getCubeInstance().getConfig().getStorageUrl()); final List<IntList> hbaseColumnsToGTIntList = Lists.newArrayList(); List<List<Integer>> hbaseColumnsToGT = getHBaseColumnsGTMapping(selectedColBlocks); for (List<Integer> list : hbaseColumnsToGT) { hbaseColumnsToGTIntList.add(IntList.newBuilder().addAllInts(list).build()); } //TODO: raw scan can be constructed at region side to reduce traffic List<RawScan> rawScans = preparedHBaseScans(scanRequest.getGTScanRanges(), selectedColBlocks); int rawScanBufferSize = BytesSerializer.SERIALIZE_BUFFER_SIZE; while (true) { try { ByteBuffer rawScanBuffer = ByteBuffer.allocate(rawScanBufferSize); BytesUtil.writeVInt(rawScans.size(), rawScanBuffer); for (RawScan rs : rawScans) { RawScan.serializer.serialize(rs, rawScanBuffer); } rawScanBuffer.flip(); rawScanByteString = HBaseZeroCopyByteString.wrap(rawScanBuffer.array(), rawScanBuffer.position(), rawScanBuffer.limit()); break; } catch (BufferOverflowException boe) { logger.info("Buffer size {} cannot hold the raw scans, resizing to 4 times", rawScanBufferSize); rawScanBufferSize *= 4; } } scanRequest.setGTScanRanges(Lists.<GTScanRange>newArrayList());//since raw scans are sent to coprocessor, we don't need to duplicate sending it int scanRequestBufferSize = BytesSerializer.SERIALIZE_BUFFER_SIZE; while (true) { try { ByteBuffer buffer = ByteBuffer.allocate(scanRequestBufferSize); GTScanRequest.serializer.serialize(scanRequest, buffer); buffer.flip(); scanRequestByteString = HBaseZeroCopyByteString.wrap(buffer.array(), buffer.position(), buffer.limit()); break; } catch (BufferOverflowException boe) { logger.info("Buffer size {} cannot hold the scan request, resizing to 4 times", scanRequestBufferSize); scanRequestBufferSize *= 4; } } logger.debug("Serialized scanRequestBytes {} bytes, rawScanBytesString {} bytes", scanRequestByteString.size(), rawScanByteString.size()); logger.info( "The scan {} for segment {} is as below with {} separate raw scans, shard part of start/end key is set to 0", Integer.toHexString(System.identityHashCode(scanRequest)), cubeSeg, rawScans.size()); for (RawScan rs : rawScans) { logScan(rs, cubeSeg.getStorageLocationIdentifier()); } logger.debug("Submitting rpc to {} shards starting from shard {}, scan range count {}", shardNum, cuboidBaseShard, rawScans.size()); final AtomicInteger totalScannedCount = new AtomicInteger(0); final ExpectedSizeIterator epResultItr = new ExpectedSizeIterator(shardNum); // KylinConfig: use env instance instead of CubeSegment, because KylinConfig will share among queries // for different cubes until redeployment of coprocessor jar. final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); final boolean compressionResult = kylinConfig.getCompressionResult(); final CubeVisitProtos.CubeVisitRequest.Builder builder = CubeVisitProtos.CubeVisitRequest.newBuilder(); builder.setGtScanRequest(scanRequestByteString).setHbaseRawScan(rawScanByteString); for (IntList intList : hbaseColumnsToGTIntList) { builder.addHbaseColumnsToGT(intList); } builder.setRowkeyPreambleSize(cubeSeg.getRowKeyPreambleSize()); builder.setBehavior(toggle); builder.setStartTime(System.currentTimeMillis()); builder.setTimeout(epResultItr.getTimeout()); builder.setKylinProperties(kylinConfig.getConfigAsString()); for (final Pair<byte[], byte[]> epRange : getEPKeyRanges(cuboidBaseShard, shardNum, totalShards)) { executorService.submit(new Runnable() { @Override public void run() { final String logHeader = "<sub-thread for GTScanRequest " + Integer.toHexString(System.identityHashCode(scanRequest)) + "> "; final boolean[] abnormalFinish = new boolean[1]; try { HTableInterface table = conn.getTable(cubeSeg.getStorageLocationIdentifier(), HBaseConnection.getCoprocessorPool()); final CubeVisitRequest request = builder.build(); final byte[] startKey = epRange.getFirst(); final byte[] endKey = epRange.getSecond(); table.coprocessorService(CubeVisitService.class, startKey, endKey, // new Batch.Call<CubeVisitService, CubeVisitResponse>() { public CubeVisitResponse call(CubeVisitService rowsService) throws IOException { ServerRpcController controller = new ServerRpcController(); BlockingRpcCallback<CubeVisitResponse> rpcCallback = new BlockingRpcCallback<>(); rowsService.visitCube(controller, request, rpcCallback); CubeVisitResponse response = rpcCallback.get(); if (controller.failedOnException()) { throw controller.getFailedOn(); } return response; } }, new Batch.Callback<CubeVisitResponse>() { @Override public void update(byte[] region, byte[] row, CubeVisitResponse result) { if (region == null) return; totalScannedCount.addAndGet(result.getStats().getScannedRowCount()); logger.info(logHeader + getStatsString(region, result)); if (result.getStats().getNormalComplete() != 1) { abnormalFinish[0] = true; return; } try { if (compressionResult) { epResultItr .append(CompressionUtils.decompress(HBaseZeroCopyByteString .zeroCopyGetBytes(result.getCompressedRows()))); } else { epResultItr.append(HBaseZeroCopyByteString .zeroCopyGetBytes(result.getCompressedRows())); } } catch (IOException | DataFormatException e) { throw new RuntimeException(logHeader + "Error when decompressing", e); } } }); } catch (Throwable ex) { logger.error(logHeader + "Error when visiting cubes by endpoint", ex); // double log coz the query thread may already timeout epResultItr.notifyCoprocException(ex); return; } if (abnormalFinish[0]) { Throwable ex = new RuntimeException(logHeader + "The coprocessor thread stopped itself due to scan timeout, failing current query..."); logger.error(logHeader + "Error when visiting cubes by endpoint", ex); // double log coz the query thread may already timeout epResultItr.notifyCoprocException(ex); return; } } }); } return new EndpointResultsAsGTScanner(fullGTInfo, epResultItr, scanRequest.getColumns(), totalScannedCount.get()); }