org.apache.bookkeeper.mledger.offload.jcloud.impl.BlockAwareSegmentInputStreamTest.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.bookkeeper.mledger.offload.jcloud.impl.BlockAwareSegmentInputStreamTest.java

Source

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.bookkeeper.mledger.offload.jcloud.impl;

import static org.testng.Assert.assertEquals;
import static org.testng.Assert.fail;
import static org.testng.internal.junit.ArrayAsserts.assertArrayEquals;

import com.google.common.io.ByteStreams;
import com.google.common.primitives.Ints;
import com.google.common.primitives.Longs;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
import java.io.ByteArrayInputStream;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.Random;
import java.util.concurrent.CompletableFuture;
import java.util.function.Supplier;
import java.util.stream.IntStream;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.apache.bookkeeper.client.api.LastConfirmedAndEntry;
import org.apache.bookkeeper.client.api.LedgerEntries;
import org.apache.bookkeeper.client.api.LedgerEntry;
import org.apache.bookkeeper.client.api.LedgerMetadata;
import org.apache.bookkeeper.client.api.ReadHandle;
import org.apache.bookkeeper.mledger.offload.jcloud.DataBlockHeader;
import org.testng.annotations.Test;
import org.testng.collections.Lists;

@Slf4j
public class BlockAwareSegmentInputStreamTest {
    private static final byte DEFAULT_ENTRY_BYTE = 0xB;

    @Data
    class MockLedgerEntry implements LedgerEntry {
        long ledgerId;
        long entryId;
        long length;
        byte entryBytes[];
        ByteBuf entryBuffer;

        MockLedgerEntry(long ledgerId, long entryId, long length, Supplier<Byte> dataSupplier) {
            this.ledgerId = ledgerId;
            this.entryId = entryId;
            this.length = length;
            this.entryBytes = new byte[(int) length];
            entryBuffer = Unpooled.wrappedBuffer(entryBytes);
            entryBuffer.writerIndex(0);
            IntStream.range(0, (int) length).forEach(i -> entryBuffer.writeByte(dataSupplier.get()));
        }

        @Override
        public ByteBuffer getEntryNioBuffer() {
            return null;
        }

        @Override
        public LedgerEntry duplicate() {
            return null;
        }

        @Override
        public void close() {
            entryBuffer.release();
        }
    }

    @Data
    class MockLedgerEntries implements LedgerEntries {
        int ledgerId;
        int startEntryId;
        int count;
        int entrySize;
        List<LedgerEntry> entries;

        MockLedgerEntries(int ledgerId, int startEntryId, int count, int entrySize, Supplier<Byte> dataSupplier) {
            this.ledgerId = ledgerId;
            this.startEntryId = startEntryId;
            this.count = count;
            this.entrySize = entrySize;
            this.entries = Lists.newArrayList(count);

            IntStream.range(startEntryId, startEntryId + count)
                    .forEach(i -> entries.add(new MockLedgerEntry(ledgerId, i, entrySize, dataSupplier)));
        }

        @Override
        public void close() {
            entries.clear();
        }

        @Override
        public LedgerEntry getEntry(long entryId) {
            if (entryId < startEntryId || entryId >= startEntryId + count) {
                return null;
            }

            return entries.get(((int) entryId - startEntryId));
        }

        @Override
        public Iterator<LedgerEntry> iterator() {
            return entries.iterator();
        }
    }

    class MockReadHandle implements ReadHandle {
        int ledgerId;
        int entrySize;
        int lac;
        Supplier<Byte> dataSupplier;

        MockReadHandle(int ledgerId, int entrySize, int lac, Supplier<Byte> dataSupplier) {
            this.ledgerId = ledgerId;
            this.entrySize = entrySize;
            this.lac = lac;
            this.dataSupplier = dataSupplier;
        }

        MockReadHandle(int ledgerId, int entrySize, int lac) {
            this(ledgerId, entrySize, lac, () -> DEFAULT_ENTRY_BYTE);
        }

        @Override
        public CompletableFuture<LedgerEntries> readAsync(long firstEntry, long lastEntry) {
            CompletableFuture<LedgerEntries> future = new CompletableFuture<>();
            LedgerEntries entries = new MockLedgerEntries(ledgerId, (int) firstEntry,
                    (int) (lastEntry - firstEntry + 1), entrySize, dataSupplier);

            future.complete(entries);
            return future;
        }

        @Override
        public CompletableFuture<LedgerEntries> readUnconfirmedAsync(long firstEntry, long lastEntry) {
            return readAsync(firstEntry, lastEntry);
        }

        @Override
        public CompletableFuture<Long> readLastAddConfirmedAsync() {
            return null;
        }

        @Override
        public CompletableFuture<Long> tryReadLastAddConfirmedAsync() {
            return null;
        }

        @Override
        public long getLastAddConfirmed() {
            return lac;
        }

        @Override
        public long getLength() {
            return (lac + 1) * entrySize;
        }

        @Override
        public boolean isClosed() {
            return true;
        }

        @Override
        public CompletableFuture<LastConfirmedAndEntry> readLastAddConfirmedAndEntryAsync(long entryId,
                long timeOutInMillis, boolean parallel) {
            return null;
        }

        @Override
        public LedgerMetadata getLedgerMetadata() {
            return null;
        }

        @Override
        public long getId() {
            return ledgerId;
        }

        @Override
        public CompletableFuture<Void> closeAsync() {
            return null;
        }
    }

    @Test
    public void testHaveEndPadding() throws Exception {
        int ledgerId = 1;
        int entrySize = 8;
        int lac = 160;
        ReadHandle readHandle = new MockReadHandle(ledgerId, entrySize, lac);

        // set block size bigger than to (header + entry) size.
        int blockSize = 3148 + 5;
        BlockAwareSegmentInputStreamImpl inputStream = new BlockAwareSegmentInputStreamImpl(readHandle, 0,
                blockSize);
        int expectedEntryCount = (blockSize - DataBlockHeaderImpl.getDataStartOffset()) / (entrySize + 4 + 8);

        // verify get methods
        assertEquals(inputStream.getLedger(), readHandle);
        assertEquals(inputStream.getStartEntryId(), 0);
        assertEquals(inputStream.getBlockSize(), blockSize);

        // verify read inputStream
        // 1. read header. 128
        byte headerB[] = new byte[DataBlockHeaderImpl.getDataStartOffset()];
        ByteStreams.readFully(inputStream, headerB);
        DataBlockHeader headerRead = DataBlockHeaderImpl.fromStream(new ByteArrayInputStream(headerB));
        assertEquals(headerRead.getBlockLength(), blockSize);
        assertEquals(headerRead.getFirstEntryId(), 0);

        byte[] entryData = new byte[entrySize];
        Arrays.fill(entryData, (byte) 0xB); // 0xB is MockLedgerEntry.blockPadding

        // 2. read Ledger entries. 201 * 20
        IntStream.range(0, expectedEntryCount).forEach(i -> {
            try {
                byte lengthBuf[] = new byte[4];
                byte entryIdBuf[] = new byte[8];
                byte content[] = new byte[entrySize];
                inputStream.read(lengthBuf);
                inputStream.read(entryIdBuf);
                inputStream.read(content);

                assertEquals(entrySize, Ints.fromByteArray(lengthBuf));
                assertEquals(i, Longs.fromByteArray(entryIdBuf));
                assertArrayEquals(entryData, content);
            } catch (Exception e) {
                fail("meet exception", e);
            }
        });

        // 3. read padding
        int left = blockSize - DataBlockHeaderImpl.getDataStartOffset() - expectedEntryCount * (entrySize + 4 + 8);
        assertEquals(left, 5);
        byte padding[] = new byte[left];
        inputStream.read(padding);
        ByteBuf paddingBuf = Unpooled.wrappedBuffer(padding);
        IntStream.range(0, paddingBuf.capacity() / 4).forEach(
                i -> assertEquals(Integer.toHexString(paddingBuf.readInt()), Integer.toHexString(0xFEDCDEAD)));

        // 4. reach end.
        assertEquals(inputStream.read(), -1);

        assertEquals(inputStream.getBlockEntryCount(), expectedEntryCount);
        assertEquals(inputStream.getBlockEntryBytesCount(), entrySize * expectedEntryCount);
        assertEquals(inputStream.getEndEntryId(), expectedEntryCount - 1);

        inputStream.close();
    }

    @Test
    public void testNoEndPadding() throws Exception {
        int ledgerId = 1;
        int entrySize = 8;
        int lac = 120;
        ReadHandle readHandle = new MockReadHandle(ledgerId, entrySize, lac);

        // set block size equals to (header + entry) size.
        int blockSize = 2148;
        BlockAwareSegmentInputStreamImpl inputStream = new BlockAwareSegmentInputStreamImpl(readHandle, 0,
                blockSize);
        int expectedEntryCount = (blockSize - DataBlockHeaderImpl.getDataStartOffset())
                / (entrySize + BlockAwareSegmentInputStreamImpl.ENTRY_HEADER_SIZE);

        // verify get methods
        assertEquals(inputStream.getLedger(), readHandle);
        assertEquals(inputStream.getStartEntryId(), 0);
        assertEquals(inputStream.getBlockSize(), blockSize);

        // verify read inputStream
        // 1. read header. 128
        byte headerB[] = new byte[DataBlockHeaderImpl.getDataStartOffset()];
        ByteStreams.readFully(inputStream, headerB);
        DataBlockHeader headerRead = DataBlockHeaderImpl.fromStream(new ByteArrayInputStream(headerB));
        assertEquals(headerRead.getBlockLength(), blockSize);
        assertEquals(headerRead.getFirstEntryId(), 0);

        byte[] entryData = new byte[entrySize];
        Arrays.fill(entryData, (byte) 0xB); // 0xB is MockLedgerEntry.blockPadding

        // 2. read Ledger entries. 201 * 20
        IntStream.range(0, expectedEntryCount).forEach(i -> {
            try {
                byte lengthBuf[] = new byte[4];
                byte entryIdBuf[] = new byte[8];
                byte content[] = new byte[entrySize];
                inputStream.read(lengthBuf);
                inputStream.read(entryIdBuf);
                inputStream.read(content);

                assertEquals(entrySize, Ints.fromByteArray(lengthBuf));
                assertEquals(i, Longs.fromByteArray(entryIdBuf));
                assertArrayEquals(entryData, content);
            } catch (Exception e) {
                fail("meet exception", e);
            }
        });

        // 3. should be no padding
        int left = blockSize - DataBlockHeaderImpl.getDataStartOffset() - expectedEntryCount * (entrySize + 4 + 8);
        assertEquals(left, 0);

        // 4. reach end.
        assertEquals(inputStream.read(), -1);

        assertEquals(inputStream.getBlockEntryCount(), expectedEntryCount);
        assertEquals(inputStream.getBlockEntryBytesCount(), entrySize * expectedEntryCount);
        assertEquals(inputStream.getEndEntryId(), expectedEntryCount - 1);

        inputStream.close();
    }

    @Test
    public void testReadTillLac() throws Exception {
        // simulate last data block read.
        int ledgerId = 1;
        int entrySize = 8;
        int lac = 89;
        ReadHandle readHandle = new MockReadHandle(ledgerId, entrySize, lac);

        // set block size equals to (header + lac_entry) size.
        int blockSize = DataBlockHeaderImpl.getDataStartOffset() + (1 + lac) * (entrySize + 4 + 8);
        BlockAwareSegmentInputStreamImpl inputStream = new BlockAwareSegmentInputStreamImpl(readHandle, 0,
                blockSize);
        int expectedEntryCount = (blockSize - DataBlockHeaderImpl.getDataStartOffset()) / (entrySize + 4 + 8);

        // verify get methods
        assertEquals(inputStream.getLedger(), readHandle);
        assertEquals(inputStream.getStartEntryId(), 0);
        assertEquals(inputStream.getBlockSize(), blockSize);

        // verify read inputStream
        // 1. read header. 128
        byte headerB[] = new byte[DataBlockHeaderImpl.getDataStartOffset()];
        ByteStreams.readFully(inputStream, headerB);
        DataBlockHeader headerRead = DataBlockHeaderImpl.fromStream(new ByteArrayInputStream(headerB));
        assertEquals(headerRead.getBlockLength(), blockSize);
        assertEquals(headerRead.getFirstEntryId(), 0);

        byte[] entryData = new byte[entrySize];
        Arrays.fill(entryData, (byte) 0xB); // 0xB is MockLedgerEntry.blockPadding

        // 2. read Ledger entries. 96 * 20
        IntStream.range(0, expectedEntryCount).forEach(i -> {
            try {
                byte lengthBuf[] = new byte[4];
                byte entryIdBuf[] = new byte[8];
                byte content[] = new byte[entrySize];
                inputStream.read(lengthBuf);
                inputStream.read(entryIdBuf);
                inputStream.read(content);

                assertEquals(entrySize, Ints.fromByteArray(lengthBuf));
                assertEquals(i, Longs.fromByteArray(entryIdBuf));
                assertArrayEquals(entryData, content);
            } catch (Exception e) {
                fail("meet exception", e);
            }
        });

        // 3. should have no padding
        int left = blockSize - DataBlockHeaderImpl.getDataStartOffset() - expectedEntryCount * (entrySize + 4 + 8);
        assertEquals(left, 0);

        // 4. reach end.
        assertEquals(inputStream.read(), -1);

        assertEquals(inputStream.getBlockEntryCount(), expectedEntryCount);
        assertEquals(inputStream.getBlockEntryBytesCount(), entrySize * expectedEntryCount);
        assertEquals(inputStream.getEndEntryId(), expectedEntryCount - 1);

        inputStream.close();
    }

    @Test
    public void testNoEntryPutIn() throws Exception {
        // simulate first entry size over the block size budget, it shouldn't be added.
        // 2 entries, each with bigger size than block size, so there should no entry added into block.
        int ledgerId = 1;
        int entrySize = 1000;
        int lac = 1;
        ReadHandle readHandle = new MockReadHandle(ledgerId, entrySize, lac);

        // set block size not able to hold one entry
        int blockSize = DataBlockHeaderImpl.getDataStartOffset() + entrySize;
        BlockAwareSegmentInputStreamImpl inputStream = new BlockAwareSegmentInputStreamImpl(readHandle, 0,
                blockSize);
        int expectedEntryCount = 0;

        // verify get methods
        assertEquals(inputStream.getLedger(), readHandle);
        assertEquals(inputStream.getStartEntryId(), 0);
        assertEquals(inputStream.getBlockSize(), blockSize);

        // verify read inputStream
        // 1. read header. 128
        byte headerB[] = new byte[DataBlockHeaderImpl.getDataStartOffset()];
        ByteStreams.readFully(inputStream, headerB);
        DataBlockHeader headerRead = DataBlockHeaderImpl.fromStream(new ByteArrayInputStream(headerB));
        assertEquals(headerRead.getBlockLength(), blockSize);
        assertEquals(headerRead.getFirstEntryId(), 0);

        // 2. since no entry put in, it should only get padding after header.
        byte padding[] = new byte[blockSize - DataBlockHeaderImpl.getDataStartOffset()];
        inputStream.read(padding);
        ByteBuf paddingBuf = Unpooled.wrappedBuffer(padding);
        IntStream.range(0, paddingBuf.capacity() / 4).forEach(
                i -> assertEquals(Integer.toHexString(paddingBuf.readInt()), Integer.toHexString(0xFEDCDEAD)));

        // 3. reach end.
        assertEquals(inputStream.read(), -1);

        assertEquals(inputStream.getBlockEntryCount(), 0);
        assertEquals(inputStream.getBlockEntryBytesCount(), 0);
        assertEquals(inputStream.getEndEntryId(), -1);

        inputStream.close();
    }

    @Test
    public void testPaddingOnLastBlock() throws Exception {
        int ledgerId = 1;
        int entrySize = 1000;
        int lac = 0;
        ReadHandle readHandle = new MockReadHandle(ledgerId, entrySize, lac);

        // set block size not able to hold one entry
        int blockSize = DataBlockHeaderImpl.getDataStartOffset() + entrySize * 2;
        BlockAwareSegmentInputStreamImpl inputStream = new BlockAwareSegmentInputStreamImpl(readHandle, 0,
                blockSize);
        int expectedEntryCount = 1;

        // verify get methods
        assertEquals(inputStream.getLedger(), readHandle);
        assertEquals(inputStream.getStartEntryId(), 0);
        assertEquals(inputStream.getBlockSize(), blockSize);

        // verify read inputStream
        // 1. read header. 128
        byte headerB[] = new byte[DataBlockHeaderImpl.getDataStartOffset()];
        ByteStreams.readFully(inputStream, headerB);
        DataBlockHeader headerRead = DataBlockHeaderImpl.fromStream(new ByteArrayInputStream(headerB));
        assertEquals(headerRead.getBlockLength(), blockSize);
        assertEquals(headerRead.getFirstEntryId(), 0);

        // 2. There should be a single entry
        byte[] entryData = new byte[entrySize];
        Arrays.fill(entryData, (byte) 0xB); // 0xB is MockLedgerEntry.blockPadding

        IntStream.range(0, expectedEntryCount).forEach(i -> {
            try {
                byte lengthBuf[] = new byte[4];
                byte entryIdBuf[] = new byte[8];
                byte content[] = new byte[entrySize];
                inputStream.read(lengthBuf);
                inputStream.read(entryIdBuf);
                inputStream.read(content);

                assertEquals(entrySize, Ints.fromByteArray(lengthBuf));
                assertEquals(i, Longs.fromByteArray(entryIdBuf));
                assertArrayEquals(entryData, content);
            } catch (Exception e) {
                fail("meet exception", e);
            }
        });

        // 3. Then padding
        int consumedBytes = DataBlockHeaderImpl.getDataStartOffset()
                + expectedEntryCount * (entrySize + BlockAwareSegmentInputStreamImpl.ENTRY_HEADER_SIZE);
        byte padding[] = new byte[blockSize - consumedBytes];
        inputStream.read(padding);
        ByteBuf paddingBuf = Unpooled.wrappedBuffer(padding);
        IntStream.range(0, paddingBuf.capacity() / 4).forEach(
                i -> assertEquals(Integer.toHexString(paddingBuf.readInt()), Integer.toHexString(0xFEDCDEAD)));

        // 3. reach end.
        assertEquals(inputStream.read(), -1);

        assertEquals(inputStream.getBlockEntryCount(), 1);
        assertEquals(inputStream.getBlockEntryBytesCount(), entrySize);
        assertEquals(inputStream.getEndEntryId(), 0);

        inputStream.close();
    }

    @Test
    public void testOnlyNegativeOnEOF() throws Exception {
        int ledgerId = 1;
        int entrySize = 10000;
        int lac = 0;

        Random r = new Random(0);
        ReadHandle readHandle = new MockReadHandle(ledgerId, entrySize, lac, () -> (byte) r.nextInt());

        int blockSize = DataBlockHeaderImpl.getDataStartOffset() + entrySize * 2;
        BlockAwareSegmentInputStreamImpl inputStream = new BlockAwareSegmentInputStreamImpl(readHandle, 0,
                blockSize);

        int bytesRead = 0;
        for (int i = 0; i < blockSize * 2; i++) {
            int ret = inputStream.read();
            if (ret < 0) { // should only be EOF
                assertEquals(bytesRead, blockSize);
                break;
            } else {
                bytesRead++;
            }
        }
    }

}