Example usage for io.netty.buffer ByteBufOutputStream buffer

Introduction

In this page you can find the example usage for io.netty.buffer ByteBufOutputStream buffer.

Prototype

ByteBuf buffer

To view the source code for io.netty.buffer ByteBufOutputStream buffer.

Click Source Link

Usage

From source file:com.addthis.hydra.store.skiplist.Page.java

License:Apache License

public byte[] encode(ByteBufOutputStream out, boolean record) {
    SkipListCacheMetrics metrics = parent.metrics;
    parent.numPagesEncoded.getAndIncrement();
    try {//from  w ww  .  j  a va2 s . c om
        OutputStream os = out;
        out.write(gztype | FLAGS_HAS_ESTIMATES | FLAGS_IS_SPARSE);
        switch (gztype) {
        case 0:
            break;
        case 1:
            os = new DeflaterOutputStream(out, new Deflater(gzlevel));
            break;
        case 2:
            os = new GZOut(out, gzbuf, gzlevel);
            break;
        case 3:
            os = new LZFOutputStream(out);
            break;
        case 4:
            os = new SnappyOutputStream(out);
            break;
        default:
            throw new RuntimeException("invalid gztype: " + gztype);
        }

        DataOutputStream dos = new DataOutputStream(os);
        byte[] firstKeyEncoded = keyCoder.keyEncode(firstKey);
        byte[] nextFirstKeyEncoded = keyCoder.keyEncode(nextFirstKey);

        updateHistogram(metrics.encodeNextFirstKeySize, nextFirstKeyEncoded.length, record);

        Varint.writeUnsignedVarInt(size, dos);
        Varint.writeUnsignedVarInt(firstKeyEncoded.length, dos);
        dos.write(firstKeyEncoded);
        Varint.writeUnsignedVarInt(nextFirstKeyEncoded.length, dos);
        if (nextFirstKeyEncoded.length > 0) {
            dos.write(nextFirstKeyEncoded);
        }
        for (int i = 0; i < size; i++) {
            byte[] keyEncoded = keyCoder.keyEncode(keys.get(i));
            byte[] rawVal = rawValues.get(i);

            if (rawVal == null || encodeType != KeyCoder.EncodeType.SPARSE) {
                fetchValue(i);
                rawVal = keyCoder.valueEncode(values.get(i), KeyCoder.EncodeType.SPARSE);
            }

            updateHistogram(metrics.encodeKeySize, keyEncoded.length, record);
            updateHistogram(metrics.encodeValueSize, rawVal.length, record);

            Varint.writeUnsignedVarInt(keyEncoded.length, dos);
            dos.write(keyEncoded);
            Varint.writeUnsignedVarInt(rawVal.length, dos);
            dos.write(rawVal);
        }

        Varint.writeUnsignedVarInt((estimateTotal > 0 ? estimateTotal : 1), dos);
        Varint.writeUnsignedVarInt((estimates > 0 ? estimates : 1), dos);
        switch (gztype) {
        case 1:
            ((DeflaterOutputStream) os).finish();
            break;
        case 2:
            ((GZOut) os).finish();
            break;
        case 4:
            os.flush();
            break;
        }
        os.flush();
        os.close();
        dos.close();

        ByteBuf buffer = out.buffer();

        byte[] returnValue = new byte[out.writtenBytes()];

        buffer.readBytes(returnValue);
        buffer.clear();
        updateHistogram(metrics.numberKeysPerPage, size, record);
        updateHistogram(metrics.encodePageSize, returnValue.length, record);
        return returnValue;
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    }
}

---

From source file:com.addthis.hydra.store.skiplist.SkipListCache.java

License:Apache License

/**
 * If the value of the {@link Page#nextFirstKey} field of a Page
 * is detected to be incorrect, then this method will correct that
 * field./*from  ww  w . j  a v a 2  s  .  c  om*/
 */
private void updateNextFirstKey(Page<K, V> prevPage, K newNextFirstKey, K targetKey, byte[] encodedTargetKey) {
    assert (prevPage.isWriteLockedByCurrentThread());

    Map.Entry<byte[], byte[]> entry = externalStore.floorEntry(encodedTargetKey);
    K floorKey = keyCoder.keyDecode(entry.getKey());
    if (floorKey.equals(prevPage.firstKey)) {
        if (prevPage.keys == null) {
            pullPageHelper(prevPage, entry.getValue());
        }
        assert (prevPage.nextFirstKey.equals(targetKey));
        prevPage.nextFirstKey = newNextFirstKey;
        if (prevPage.state == ExternalMode.DISK_MEMORY_IDENTICAL) {
            prevPage.state = ExternalMode.DISK_MEMORY_DIRTY;
        }
    } else {
        Page<K, V> diskPage = pageFactory.generateEmptyPage(SkipListCache.this, floorKey,
                KeyCoder.EncodeType.SPARSE);
        diskPage.decode(entry.getValue());
        assert (diskPage.nextFirstKey.equals(targetKey));
        assert (compareKeys(prevPage.firstKey, diskPage.firstKey) <= 0);
        diskPage.nextFirstKey = newNextFirstKey;
        ByteBufOutputStream byteBufOutputStream = new ByteBufOutputStream(
                PooledByteBufAllocator.DEFAULT.buffer());
        try {
            externalStore.put(entry.getKey(), diskPage.encode(byteBufOutputStream));
        } finally {
            byteBufOutputStream.buffer().release();
        }
    }
}

---

From source file:com.addthis.hydra.store.skiplist.SkipListCache.java

License:Apache License

/**
 * Splits a page in half. The input page must be write locked,
 * hold enough keys to satisfy the split condition, and cannot
 * be in a transient state. The skip-list cache uses the invariant
 * that each page in the cache must have some copy of the page
 * in external storage. We currently use Berkeley DB as the external
 * storage system, which is an append-only database. To conserve
 * disk space we do not store a full page to the database but instead
 * insert (new key, empty page) as a stub into the database.
 *
 * @param target page to split/*  w  ww .  jav  a2  s . c  om*/
 */
private Page<K, V> splitOnePage(Page<K, V> target) {
    assert (target.isWriteLockedByCurrentThread());
    assert (target.splitCondition());
    assert (!target.inTransientState());

    if (target.keys == null) {
        pullPageFromDisk(target, LockMode.WRITEMODE);
    }

    int newSize = target.size / 2;
    int sibSize = target.size - newSize;

    List<K> keyRange = target.keys.subList(newSize, target.size);
    List<V> valueRange = target.values.subList(newSize, target.size);
    List<byte[]> rawValueRange = target.rawValues.subList(newSize, target.size);

    ArrayList<K> sibKeys = new ArrayList<>(keyRange);
    ArrayList<V> sibValues = new ArrayList<>(valueRange);
    ArrayList<byte[]> sibRawValues = new ArrayList<>(rawValueRange);
    K sibMinKey = sibKeys.get(0);

    Page<K, V> sibling = pageFactory.generateSiblingPage(SkipListCache.this, sibMinKey, target.nextFirstKey,
            sibSize, sibKeys, sibValues, sibRawValues, target.getEncodeType());
    sibling.writeLock();

    byte[] encodeKey;
    byte[] placeHolder;

    sibling.state = ExternalMode.DISK_MEMORY_DIRTY;
    target.state = ExternalMode.DISK_MEMORY_DIRTY;

    Page<K, V> prev = cache.putIfAbsent(sibMinKey, sibling);
    if (prev != null) {
        throw new IllegalStateException("Page split " + target.firstKey.toString() + " resulted in a new page "
                + sibMinKey.toString() + " that already exists in cache.");
    }

    cacheSize.getAndIncrement();
    numPagesInMemory.getAndIncrement();

    sibling.avgEntrySize = target.avgEntrySize;
    sibling.estimates = target.estimates;
    sibling.estimateTotal = target.estimateTotal;

    target.nextFirstKey = sibMinKey;
    target.size = newSize;

    int prevMem = target.getMemoryEstimate();

    target.updateMemoryEstimate();
    sibling.updateMemoryEstimate();

    int updatedMem = target.getMemoryEstimate() + sibling.getMemoryEstimate();
    updateMemoryEstimate(updatedMem - prevMem);

    encodeKey = keyCoder.keyEncode(sibMinKey);

    ByteBufOutputStream byteBufOutputStream = new ByteBufOutputStream(ByteBufAllocator.DEFAULT.buffer());
    try {
        placeHolder = pageFactory.generateEmptyPage(SkipListCache.this, sibling.firstKey, sibling.nextFirstKey,
                sibling.getEncodeType()).encode(byteBufOutputStream, false);
    } finally {
        byteBufOutputStream.buffer().release();
    }
    externalStore.put(encodeKey, placeHolder);

    evictionQueue.offer(sibling);
    numPagesSplit.getAndIncrement();

    keyRange.clear();
    valueRange.clear();
    rawValueRange.clear();

    return sibling;
}

---

From source file:com.addthis.hydra.store.skiplist.SkipListCache.java

License:Apache License

/**
 * Invoked by the constructor. If the left sentinel page is not
 * found in the external storage, then create the left sentinel
 * page.//  ww w  . j  a v a2s. c  o m
 */
private void loadFromExternalStore() {
    Page<K, V> leftSentinel = pageFactory.generateEmptyPage(this, negInf, KeyCoder.EncodeType.SPARSE);
    ByteBufOutputStream byteBufOutputStream = null;
    try {
        if (externalStore.count() == 0) {
            byteBufOutputStream = new ByteBufOutputStream(PooledByteBufAllocator.DEFAULT.buffer());
            leftSentinel.initialize();
            byte[] encodeKey = keyCoder.keyEncode(negInf);
            byte[] encodePage = leftSentinel.encode(byteBufOutputStream);
            externalStore.put(encodeKey, encodePage);
        } else {
            byte[] encodedFirstKey = externalStore.firstKey();
            K firstKey = keyCoder.keyDecode(encodedFirstKey);
            byte[] page = externalStore.get(encodedFirstKey);

            if (firstKey.equals(negInf)) {
                leftSentinel.decode(page);
                updateMemoryEstimate(leftSentinel.getMemoryEstimate());
            } else {
                byteBufOutputStream = new ByteBufOutputStream(PooledByteBufAllocator.DEFAULT.buffer());
                leftSentinel.initialize();
                leftSentinel.nextFirstKey = firstKey;

                byte[] encodeKey = keyCoder.keyEncode(negInf);
                byte[] encodePage = leftSentinel.encode(byteBufOutputStream);
                externalStore.put(encodeKey, encodePage);

                Page<K, V> minPage = pageFactory.generateEmptyPage(this, firstKey,
                        leftSentinel.getEncodeType());
                minPage.decode(page);

                cache.put(firstKey, minPage);
                updateMemoryEstimate(minPage.getMemoryEstimate());
                cacheSize.getAndIncrement();
                numPagesInMemory.getAndIncrement();
                evictionQueue.offer(minPage);
            }
        }
    } finally {
        if (byteBufOutputStream != null) {
            byteBufOutputStream.buffer().release();
        }
    }
    cache.put(negInf, leftSentinel);
    cacheSize.getAndIncrement();
    numPagesInMemory.getAndIncrement();
    evictionQueue.offer(leftSentinel);
}

---

From source file:com.addthis.hydra.store.skiplist.SkipListCache.java

License:Apache License

private void pushAllPagesToDisk() {
    final ByteBufOutputStream byteStream = new ByteBufOutputStream(PooledByteBufAllocator.DEFAULT.buffer());
    try {//from  w ww .  ja va 2s  . co  m

        for (Page<K, V> page : evictionQueue) {

            page.writeLock();

            if (!page.inTransientState() && page.keys != null) {
                pushPageToDisk(page, byteStream);
            }

            page.writeUnlock();
        }
    } finally {
        byteStream.buffer().release();
    }

    assert (pushAllPagesToDiskAssertion());
}

---

From source file:com.addthis.hydra.store.skiplist.SkipListCache.java

License:Apache License

/**
 * Emit a log message that a page has been detected with a null nextFirstKey
 * and the page is not the largest page in the database.
 *
 * @param repair   if true then repair the page
 * @param counter  page number.//from ww  w.j a va  2 s  . c om
 * @param page     contents of the page.
 * @param key      key associated with the page.
 * @param nextKey  key associated with the next page.
 */
private void missingNextFirstKey(final boolean repair, final int counter, Page<K, V> page, final K key,
        final K nextKey) {
    log.warn(
            "On page {} the firstKey is {} " + " the length is {} "
                    + " the nextFirstKey is null and the next page" + " is associated with key {}.",
            counter, page.firstKey, page.size, nextKey);
    if (repair) {
        log.info("Repairing nextFirstKey on page {}.", counter);
        page.nextFirstKey = nextKey;
        ByteBufOutputStream byteBufOutputStream = new ByteBufOutputStream(
                PooledByteBufAllocator.DEFAULT.buffer());
        try {
            byte[] pageEncoded = page.encode(byteBufOutputStream);
            externalStore.put(keyCoder.keyEncode(key), pageEncoded);
        } finally {
            byteBufOutputStream.buffer().release();
        }
    }
}

---

From source file:com.addthis.hydra.store.skiplist.SkipListCache.java

License:Apache License

private void repairInvalidKeys(final int counter, Page<K, V> page, final K key, final K nextKey) {
    boolean pageTransfer = false;
    Page<K, V> nextPage = pageFactory.generateEmptyPage(this, nextKey, page.getEncodeType());
    byte[] encodedNextPage = externalStore.get(keyCoder.keyEncode(nextKey));
    nextPage.decode(encodedNextPage);//from   w ww .  j  av  a2  s  .c  o m
    for (int i = 0, pos = 0; i < page.size; i++, pos++) {
        K testKey = page.keys.get(i);
        // if testKey >= nextKey then we need to move the testKey off the current page
        if (compareKeys(testKey, nextKey) >= 0) {
            // non-negative value from binary search indicates the key was found on the next page
            if (binarySearch(nextPage.keys, testKey, comparator) >= 0) {
                log.info("Key {} was detected on next page. Deleting from page {}.", pos, counter);
            } else {
                log.info("Moving key {} on page {}.", pos, counter);
                page.fetchValue(i);
                V value = page.values.get(i);
                putIntoPage(nextPage, testKey, value);
                pageTransfer = true;
            }
            page.keys.remove(i);
            page.rawValues.remove(i);
            page.values.remove(i);
            page.size--;
            i--;
        }
    }
    ByteBufOutputStream byteBufOutputStream = new ByteBufOutputStream(PooledByteBufAllocator.DEFAULT.buffer());
    try {
        byte[] pageEncoded = page.encode(byteBufOutputStream);
        externalStore.put(keyCoder.keyEncode(key), pageEncoded);
        if (pageTransfer) {
            encodedNextPage = nextPage.encode(byteBufOutputStream);
            externalStore.put(keyCoder.keyEncode(nextKey), encodedNextPage);
        }
    } finally {
        byteBufOutputStream.buffer().release();
    }

}

---

From source file:com.addthis.hydra.store.skiplist.SparsePage.java

License:Apache License

public byte[] encode(ByteBufOutputStream out, boolean record) {
    SkipListCacheMetrics metrics = parent.metrics;
    parent.numPagesEncoded.getAndIncrement();
    try {/*from w w w. ja  v  a2 s .  c o m*/
        OutputStream os = out;
        out.write(gztype | FLAGS_HAS_ESTIMATES | FLAGS_IS_SPARSE);
        switch (gztype) {
        case 0:
            break;
        case 1:
            os = new DeflaterOutputStream(out, new Deflater(gzlevel));
            break;
        case 2:
            os = new GZOut(out, gzbuf, gzlevel);
            break;
        case 3:
            os = new LZFOutputStream(out);
            break;
        case 4:
            os = new SnappyOutputStream(out);
            break;
        default:
            throw new RuntimeException("invalid gztype: " + gztype);
        }

        DataOutputStream dos = new DataOutputStream(os);
        byte[] firstKeyEncoded = keyCoder.keyEncode(firstKey);
        byte[] nextFirstKeyEncoded = keyCoder.keyEncode(nextFirstKey);

        updateHistogram(metrics.encodeNextFirstKeySize, nextFirstKeyEncoded.length, record);

        Varint.writeUnsignedVarInt(size, dos);
        Varint.writeUnsignedVarInt(firstKeyEncoded.length, dos);
        dos.write(firstKeyEncoded);
        Varint.writeUnsignedVarInt(nextFirstKeyEncoded.length, dos);
        if (nextFirstKeyEncoded.length > 0) {
            dos.write(nextFirstKeyEncoded);
        }
        for (int i = 0; i < size; i++) {
            byte[] keyEncoded = keyCoder.keyEncode(keys.get(i));
            byte[] rawVal = rawValues.get(i);

            if (rawVal == null || encodeType != PageEncodeType.SPARSE) {
                fetchValue(i);
                rawVal = keyCoder.valueEncode(values.get(i), PageEncodeType.SPARSE);
            }

            updateHistogram(metrics.encodeKeySize, keyEncoded.length, record);
            updateHistogram(metrics.encodeValueSize, rawVal.length, record);

            Varint.writeUnsignedVarInt(keyEncoded.length, dos);
            dos.write(keyEncoded);
            Varint.writeUnsignedVarInt(rawVal.length, dos);
            dos.write(rawVal);
        }

        Varint.writeUnsignedVarInt((estimateTotal > 0 ? estimateTotal : 1), dos);
        Varint.writeUnsignedVarInt((estimates > 0 ? estimates : 1), dos);
        switch (gztype) {
        case 1:
            ((DeflaterOutputStream) os).finish();
            break;
        case 2:
            ((GZOut) os).finish();
            break;
        }
        os.flush(); // flush should be called by dos.close(), but better safe than sorry
        dos.close();

        ByteBuf buffer = out.buffer();

        byte[] returnValue = new byte[out.writtenBytes()];

        buffer.readBytes(returnValue);
        buffer.clear();
        updateHistogram(metrics.numberKeysPerPage, size, record);
        updateHistogram(metrics.encodePageSize, returnValue.length, record);
        return returnValue;
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    }
}

---

From source file:com.allanbank.mongodb.netty.ByteToMessageDecoderTest.java

License:Apache License

/**
 * Test method for/*from   www . ja v  a 2s  .  c  om*/
 * {@link ByteToMessageDecoder#decode(ChannelHandlerContext, ByteBuf)}.
 *
 * @throws Exception
 *             On a test failure.
 */
@Test
public void testDecodeBadOpCode() throws Exception {
    final Random rand = new Random(System.currentTimeMillis());
    final Message msg = new KillCursors(new long[] { rand.nextLong() }, ReadPreference.PRIMARY);
    final int msgId = rand.nextInt() & 0xFFFFFF;

    final ByteBuf buffer = ourAllocator.buffer();

    final ByteBufOutputStream out = new ByteBufOutputStream(buffer);
    final BsonOutputStream bout = new BsonOutputStream(out);

    msg.write(msgId, bout);

    // OpCode is bytes 12-16.
    buffer.setByte(12, (byte) 0xAA);
    buffer.setByte(13, (byte) 0xBB);
    buffer.setByte(14, (byte) 0xCC);
    buffer.setByte(15, (byte) 0xDD);

    final ChannelHandlerContext mockContext = createMock(ChannelHandlerContext.class);

    replay(mockContext);

    final ByteToMessageDecoder decoder = new ByteToMessageDecoder(new StringDecoderCache());

    try {
        decoder.decode(mockContext, out.buffer());
        fail("Should have thrown a MongoDBException.");
    } catch (final MongoDbException good) {
        assertThat(good.getMessage(), is("Unexpected operation read '" + 0xDDCCBBAA + "'."));
    }

    verify(mockContext);
}

---

From source file:com.allanbank.mongodb.netty.ByteToMessageDecoderTest.java

License:Apache License

/**
 * Performs the basic receive test with the message.
 *
 * @param message/*w  ww .j a  v a 2s .c o  m*/
 *            The message to send.
 * @param random
 *            The source of random for the test.
 * @throws Exception
 *             On a test failure.
 */
private void runDecode(final Message message, final Random random) throws Exception {
    final ByteBuf buffer = ourAllocator.buffer();

    final ByteBufOutputStream out = new ByteBufOutputStream(buffer);
    final BsonOutputStream bout = new BsonOutputStream(out);

    final int msgId = random.nextInt() & 0xFFFFFF;
    message.write(msgId, bout);

    final ChannelHandlerContext mockContext = createMock(ChannelHandlerContext.class);

    replay(mockContext);

    final ByteToMessageDecoder decoder = new ByteToMessageDecoder(new StringDecoderCache());

    final Object result = decoder.decode(mockContext, out.buffer());
    assertThat(result, is((Object) message));

    verify(mockContext);
}

---

• «
• 1
• 2
• 3
• »