List of usage examples for io.netty.buffer ByteBufOutputStream ByteBufOutputStream
public ByteBufOutputStream(ByteBuf buffer)
From source file:co.rsk.rpc.netty.JsonRpcWeb3ServerHandler.java
License:Open Source License
@Override protected void channelRead0(ChannelHandlerContext ctx, ByteBufHolder request) throws Exception { ByteBuf responseContent = Unpooled.buffer(); int responseCode; try (ByteBufOutputStream os = new ByteBufOutputStream(responseContent); ByteBufInputStream is = new ByteBufInputStream(request.content().retain())) { responseCode = jsonRpcServer.handleRequest(is, os); } catch (Exception e) { String unexpectedErrorMsg = "Unexpected error"; LOGGER.error(unexpectedErrorMsg, e); int errorCode = ErrorResolver.JsonError.CUSTOM_SERVER_ERROR_LOWER; responseContent = buildErrorContent(errorCode, unexpectedErrorMsg); responseCode = errorCode;//www . j ava 2 s . c o m } ctx.fireChannelRead(new Web3Result(responseContent, responseCode)); }
From source file:com.addthis.hydra.store.skiplist.SkipListCache.java
License:Apache License
/** * If the value of the {@link Page#nextFirstKey} field of a Page * is detected to be incorrect, then this method will correct that * field.// w ww . j a v a 2s. c om */ private void updateNextFirstKey(Page<K, V> prevPage, K newNextFirstKey, K targetKey, byte[] encodedTargetKey) { assert (prevPage.isWriteLockedByCurrentThread()); Map.Entry<byte[], byte[]> entry = externalStore.floorEntry(encodedTargetKey); K floorKey = keyCoder.keyDecode(entry.getKey()); if (floorKey.equals(prevPage.firstKey)) { if (prevPage.keys == null) { pullPageHelper(prevPage, entry.getValue()); } assert (prevPage.nextFirstKey.equals(targetKey)); prevPage.nextFirstKey = newNextFirstKey; if (prevPage.state == ExternalMode.DISK_MEMORY_IDENTICAL) { prevPage.state = ExternalMode.DISK_MEMORY_DIRTY; } } else { Page<K, V> diskPage = pageFactory.generateEmptyPage(SkipListCache.this, floorKey, KeyCoder.EncodeType.SPARSE); diskPage.decode(entry.getValue()); assert (diskPage.nextFirstKey.equals(targetKey)); assert (compareKeys(prevPage.firstKey, diskPage.firstKey) <= 0); diskPage.nextFirstKey = newNextFirstKey; ByteBufOutputStream byteBufOutputStream = new ByteBufOutputStream( PooledByteBufAllocator.DEFAULT.buffer()); try { externalStore.put(entry.getKey(), diskPage.encode(byteBufOutputStream)); } finally { byteBufOutputStream.buffer().release(); } } }
From source file:com.addthis.hydra.store.skiplist.SkipListCache.java
License:Apache License
/** * Splits a page in half. The input page must be write locked, * hold enough keys to satisfy the split condition, and cannot * be in a transient state. The skip-list cache uses the invariant * that each page in the cache must have some copy of the page * in external storage. We currently use Berkeley DB as the external * storage system, which is an append-only database. To conserve * disk space we do not store a full page to the database but instead * insert (new key, empty page) as a stub into the database. * * @param target page to split//from w w w. j a va2 s .c om */ private Page<K, V> splitOnePage(Page<K, V> target) { assert (target.isWriteLockedByCurrentThread()); assert (target.splitCondition()); assert (!target.inTransientState()); if (target.keys == null) { pullPageFromDisk(target, LockMode.WRITEMODE); } int newSize = target.size / 2; int sibSize = target.size - newSize; List<K> keyRange = target.keys.subList(newSize, target.size); List<V> valueRange = target.values.subList(newSize, target.size); List<byte[]> rawValueRange = target.rawValues.subList(newSize, target.size); ArrayList<K> sibKeys = new ArrayList<>(keyRange); ArrayList<V> sibValues = new ArrayList<>(valueRange); ArrayList<byte[]> sibRawValues = new ArrayList<>(rawValueRange); K sibMinKey = sibKeys.get(0); Page<K, V> sibling = pageFactory.generateSiblingPage(SkipListCache.this, sibMinKey, target.nextFirstKey, sibSize, sibKeys, sibValues, sibRawValues, target.getEncodeType()); sibling.writeLock(); byte[] encodeKey; byte[] placeHolder; sibling.state = ExternalMode.DISK_MEMORY_DIRTY; target.state = ExternalMode.DISK_MEMORY_DIRTY; Page<K, V> prev = cache.putIfAbsent(sibMinKey, sibling); if (prev != null) { throw new IllegalStateException("Page split " + target.firstKey.toString() + " resulted in a new page " + sibMinKey.toString() + " that already exists in cache."); } cacheSize.getAndIncrement(); numPagesInMemory.getAndIncrement(); sibling.avgEntrySize = target.avgEntrySize; sibling.estimates = target.estimates; sibling.estimateTotal = target.estimateTotal; target.nextFirstKey = sibMinKey; target.size = newSize; int prevMem = target.getMemoryEstimate(); target.updateMemoryEstimate(); sibling.updateMemoryEstimate(); int updatedMem = target.getMemoryEstimate() + sibling.getMemoryEstimate(); updateMemoryEstimate(updatedMem - prevMem); encodeKey = keyCoder.keyEncode(sibMinKey); ByteBufOutputStream byteBufOutputStream = new ByteBufOutputStream(ByteBufAllocator.DEFAULT.buffer()); try { placeHolder = pageFactory.generateEmptyPage(SkipListCache.this, sibling.firstKey, sibling.nextFirstKey, sibling.getEncodeType()).encode(byteBufOutputStream, false); } finally { byteBufOutputStream.buffer().release(); } externalStore.put(encodeKey, placeHolder); evictionQueue.offer(sibling); numPagesSplit.getAndIncrement(); keyRange.clear(); valueRange.clear(); rawValueRange.clear(); return sibling; }
From source file:com.addthis.hydra.store.skiplist.SkipListCache.java
License:Apache License
/** * Invoked by the constructor. If the left sentinel page is not * found in the external storage, then create the left sentinel * page.// w ww .ja va 2s . c om */ private void loadFromExternalStore() { Page<K, V> leftSentinel = pageFactory.generateEmptyPage(this, negInf, KeyCoder.EncodeType.SPARSE); ByteBufOutputStream byteBufOutputStream = null; try { if (externalStore.count() == 0) { byteBufOutputStream = new ByteBufOutputStream(PooledByteBufAllocator.DEFAULT.buffer()); leftSentinel.initialize(); byte[] encodeKey = keyCoder.keyEncode(negInf); byte[] encodePage = leftSentinel.encode(byteBufOutputStream); externalStore.put(encodeKey, encodePage); } else { byte[] encodedFirstKey = externalStore.firstKey(); K firstKey = keyCoder.keyDecode(encodedFirstKey); byte[] page = externalStore.get(encodedFirstKey); if (firstKey.equals(negInf)) { leftSentinel.decode(page); updateMemoryEstimate(leftSentinel.getMemoryEstimate()); } else { byteBufOutputStream = new ByteBufOutputStream(PooledByteBufAllocator.DEFAULT.buffer()); leftSentinel.initialize(); leftSentinel.nextFirstKey = firstKey; byte[] encodeKey = keyCoder.keyEncode(negInf); byte[] encodePage = leftSentinel.encode(byteBufOutputStream); externalStore.put(encodeKey, encodePage); Page<K, V> minPage = pageFactory.generateEmptyPage(this, firstKey, leftSentinel.getEncodeType()); minPage.decode(page); cache.put(firstKey, minPage); updateMemoryEstimate(minPage.getMemoryEstimate()); cacheSize.getAndIncrement(); numPagesInMemory.getAndIncrement(); evictionQueue.offer(minPage); } } } finally { if (byteBufOutputStream != null) { byteBufOutputStream.buffer().release(); } } cache.put(negInf, leftSentinel); cacheSize.getAndIncrement(); numPagesInMemory.getAndIncrement(); evictionQueue.offer(leftSentinel); }
From source file:com.addthis.hydra.store.skiplist.SkipListCache.java
License:Apache License
private void pushAllPagesToDisk() { final ByteBufOutputStream byteStream = new ByteBufOutputStream(PooledByteBufAllocator.DEFAULT.buffer()); try {/*from ww w . j a v a 2s . c om*/ for (Page<K, V> page : evictionQueue) { page.writeLock(); if (!page.inTransientState() && page.keys != null) { pushPageToDisk(page, byteStream); } page.writeUnlock(); } } finally { byteStream.buffer().release(); } assert (pushAllPagesToDiskAssertion()); }
From source file:com.addthis.hydra.store.skiplist.SkipListCache.java
License:Apache License
/** * Emit a log message that a page has been detected with a null nextFirstKey * and the page is not the largest page in the database. * * @param repair if true then repair the page * @param counter page number.//from ww w . j a v a 2 s .c o m * @param page contents of the page. * @param key key associated with the page. * @param nextKey key associated with the next page. */ private void missingNextFirstKey(final boolean repair, final int counter, Page<K, V> page, final K key, final K nextKey) { log.warn( "On page {} the firstKey is {} " + " the length is {} " + " the nextFirstKey is null and the next page" + " is associated with key {}.", counter, page.firstKey, page.size, nextKey); if (repair) { log.info("Repairing nextFirstKey on page {}.", counter); page.nextFirstKey = nextKey; ByteBufOutputStream byteBufOutputStream = new ByteBufOutputStream( PooledByteBufAllocator.DEFAULT.buffer()); try { byte[] pageEncoded = page.encode(byteBufOutputStream); externalStore.put(keyCoder.keyEncode(key), pageEncoded); } finally { byteBufOutputStream.buffer().release(); } } }
From source file:com.addthis.hydra.store.skiplist.SkipListCache.java
License:Apache License
private void repairInvalidKeys(final int counter, Page<K, V> page, final K key, final K nextKey) { boolean pageTransfer = false; Page<K, V> nextPage = pageFactory.generateEmptyPage(this, nextKey, page.getEncodeType()); byte[] encodedNextPage = externalStore.get(keyCoder.keyEncode(nextKey)); nextPage.decode(encodedNextPage);/*ww w . ja v a 2 s . c o m*/ for (int i = 0, pos = 0; i < page.size; i++, pos++) { K testKey = page.keys.get(i); // if testKey >= nextKey then we need to move the testKey off the current page if (compareKeys(testKey, nextKey) >= 0) { // non-negative value from binary search indicates the key was found on the next page if (binarySearch(nextPage.keys, testKey, comparator) >= 0) { log.info("Key {} was detected on next page. Deleting from page {}.", pos, counter); } else { log.info("Moving key {} on page {}.", pos, counter); page.fetchValue(i); V value = page.values.get(i); putIntoPage(nextPage, testKey, value); pageTransfer = true; } page.keys.remove(i); page.rawValues.remove(i); page.values.remove(i); page.size--; i--; } } ByteBufOutputStream byteBufOutputStream = new ByteBufOutputStream(PooledByteBufAllocator.DEFAULT.buffer()); try { byte[] pageEncoded = page.encode(byteBufOutputStream); externalStore.put(keyCoder.keyEncode(key), pageEncoded); if (pageTransfer) { encodedNextPage = nextPage.encode(byteBufOutputStream); externalStore.put(keyCoder.keyEncode(nextKey), encodedNextPage); } } finally { byteBufOutputStream.buffer().release(); } }
From source file:com.allanbank.mongodb.netty.ByteToMessageDecoderTest.java
License:Apache License
/** * Test method for/* www .j ava 2 s. co m*/ * {@link ByteToMessageDecoder#decode(ChannelHandlerContext, ByteBuf)}. * * @throws Exception * On a test failure. */ @Test public void testDecodeBadOpCode() throws Exception { final Random rand = new Random(System.currentTimeMillis()); final Message msg = new KillCursors(new long[] { rand.nextLong() }, ReadPreference.PRIMARY); final int msgId = rand.nextInt() & 0xFFFFFF; final ByteBuf buffer = ourAllocator.buffer(); final ByteBufOutputStream out = new ByteBufOutputStream(buffer); final BsonOutputStream bout = new BsonOutputStream(out); msg.write(msgId, bout); // OpCode is bytes 12-16. buffer.setByte(12, (byte) 0xAA); buffer.setByte(13, (byte) 0xBB); buffer.setByte(14, (byte) 0xCC); buffer.setByte(15, (byte) 0xDD); final ChannelHandlerContext mockContext = createMock(ChannelHandlerContext.class); replay(mockContext); final ByteToMessageDecoder decoder = new ByteToMessageDecoder(new StringDecoderCache()); try { decoder.decode(mockContext, out.buffer()); fail("Should have thrown a MongoDBException."); } catch (final MongoDbException good) { assertThat(good.getMessage(), is("Unexpected operation read '" + 0xDDCCBBAA + "'.")); } verify(mockContext); }
From source file:com.allanbank.mongodb.netty.ByteToMessageDecoderTest.java
License:Apache License
/** * Test method for/*from w ww . j a va2s . c o m*/ * {@link ByteToMessageDecoder#decode(ChannelHandlerContext, ByteBuf)}. * * @throws Exception * On a test failure. */ @Test public void testDecodeInCompleteFrame() throws Exception { final Random rand = new Random(System.currentTimeMillis()); final Message msg = new KillCursors(new long[] { rand.nextLong() }, ReadPreference.PRIMARY); final int msgId = rand.nextInt() & 0xFFFFFF; final ByteBuf buffer = ourAllocator.buffer(); final ByteBufOutputStream out = new ByteBufOutputStream(buffer); final BsonOutputStream bout = new BsonOutputStream(out); msg.write(msgId, bout); final ChannelHandlerContext mockContext = createMock(ChannelHandlerContext.class); replay(mockContext); final ByteToMessageDecoder decoder = new ByteToMessageDecoder(new StringDecoderCache()); final Object result = decoder.decode(mockContext, buffer.slice(0, buffer.writerIndex() - 1)); assertThat(result, nullValue()); verify(mockContext); }
From source file:com.allanbank.mongodb.netty.ByteToMessageDecoderTest.java
License:Apache License
/** * Performs the basic receive test with the message. * * @param message// w w w . j a v a 2 s. c o m * The message to send. * @param random * The source of random for the test. * @throws Exception * On a test failure. */ private void runDecode(final Message message, final Random random) throws Exception { final ByteBuf buffer = ourAllocator.buffer(); final ByteBufOutputStream out = new ByteBufOutputStream(buffer); final BsonOutputStream bout = new BsonOutputStream(out); final int msgId = random.nextInt() & 0xFFFFFF; message.write(msgId, bout); final ChannelHandlerContext mockContext = createMock(ChannelHandlerContext.class); replay(mockContext); final ByteToMessageDecoder decoder = new ByteToMessageDecoder(new StringDecoderCache()); final Object result = decoder.decode(mockContext, out.buffer()); assertThat(result, is((Object) message)); verify(mockContext); }