org.apache.asterix.external.feed.test.InputHandlerTest.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.asterix.external.feed.test.InputHandlerTest.java

Source

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.asterix.external.feed.test;

import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.util.Collection;
import java.util.Collections;
import java.util.Random;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;

import org.apache.asterix.active.ActiveRuntimeId;
import org.apache.asterix.active.EntityId;
import org.apache.asterix.common.memory.ConcurrentFramePool;
import org.apache.asterix.external.feed.dataflow.FeedRuntimeInputHandler;
import org.apache.asterix.external.feed.management.FeedConnectionId;
import org.apache.asterix.external.feed.policy.FeedPolicyAccessor;
import org.apache.asterix.external.util.FeedUtils;
import org.apache.asterix.external.util.FeedUtils.FeedRuntimeType;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.filefilter.WildcardFileFilter;
import org.apache.hyracks.api.comm.IFrameWriter;
import org.apache.hyracks.api.comm.VSizeFrame;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.test.FrameWriterTestUtils;
import org.apache.hyracks.api.test.TestControlledFrameWriter;
import org.apache.hyracks.api.test.TestFrameWriter;
import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
import org.apache.hyracks.test.support.TestUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;

public class InputHandlerTest {

    private static final int DEFAULT_FRAME_SIZE = 32768;
    private static final int NUM_FRAMES = 128;
    private static final long FEED_MEM_BUDGET = DEFAULT_FRAME_SIZE * NUM_FRAMES;
    private static final String DATAVERSE = "dataverse";
    private static final String DATASET = "dataset";
    private static final String FEED = "feed";
    private static final String NODE_ID = "NodeId";
    private static final float DISCARD_ALLOWANCE = 0.15f;
    private static final ExecutorService EXECUTOR = Executors.newFixedThreadPool(1);
    private volatile static HyracksDataException cause = null;

    private FeedRuntimeInputHandler createInputHandler(IHyracksTaskContext ctx, IFrameWriter writer,
            FeedPolicyAccessor fpa, ConcurrentFramePool framePool) throws HyracksDataException {
        FrameTupleAccessor fta = Mockito.mock(FrameTupleAccessor.class);
        EntityId feedId = new EntityId(FeedUtils.FEED_EXTENSION_NAME, DATAVERSE, FEED);
        FeedConnectionId connectionId = new FeedConnectionId(feedId, DATASET);
        ActiveRuntimeId runtimeId = new ActiveRuntimeId(feedId, FeedRuntimeType.COLLECT.toString(), 0);
        return new FeedRuntimeInputHandler(ctx, connectionId, runtimeId, writer, fpa, fta, framePool);
    }

    /*
     * Testing the following scenarios
     * 01. Positive Frames memory budget with fixed size frames, no spill, no discard.
     * 02. Positive Frames memory budget with variable size frames, no spill, no discard.
     * 03. Positive Frames memory budget with fixed size frames, with spill, no discard.
     * 04. Positive Frames memory budget with variable size frames, with spill, no discard.
     * 05. Positive Frames memory budget with fixed size frames, no spill, with discard.
     * 06. Positive Frames memory budget with variable size frames, no spill, with discard.
     * 07. Positive Frames memory budget with fixed size frames, with spill, with discard.
     * 08. Positive Frames memory budget with variable size frames, with spill, with discard.
     * 09. 0 Frames memory budget with fixed size frames, with spill, no discard.
     * 10. 0 Frames memory budget with variable size frames, with spill, no discard.
     * 11. TODO 0 Frames memory budget with fixed size frames, with spill, with discard.
     * 12. TODO 0 Frames memory budget with variable size frames, with spill, with discard.
     * 13. TODO Test exception handling with Open, NextFrame,Flush,Close,Fail exception throwing FrameWriter
     * 14. TODO Test exception while waiting for subscription
     */

    private static FeedPolicyAccessor createFeedPolicyAccessor(boolean spill, boolean discard, long spillBudget,
            float discardFraction) {
        FeedPolicyAccessor fpa = Mockito.mock(FeedPolicyAccessor.class);
        Mockito.when(fpa.flowControlEnabled()).thenReturn(true);
        Mockito.when(fpa.spillToDiskOnCongestion()).thenReturn(spill);
        Mockito.when(fpa.getMaxSpillOnDisk()).thenReturn(spillBudget);
        Mockito.when(fpa.discardOnCongestion()).thenReturn(discard);
        Mockito.when(fpa.getMaxFractionDiscard()).thenReturn(discardFraction);
        return fpa;
    }

    private void cleanDiskFiles() throws IOException {
        String filePrefix = "dataverse.feed(Feed)_dataset*";
        Collection<File> files = FileUtils.listFiles(new File("."), new WildcardFileFilter(filePrefix), null);
        for (File ifile : files) {
            Files.deleteIfExists(ifile.toPath());
        }
    }

    @Before
    public void testCleanBefore() throws IOException {
        cleanDiskFiles();
    }

    @After
    public void testCleanAfter() throws IOException {
        cleanDiskFiles();
    }

    @org.junit.Test
    public void testZeroMemoryVarSizeFrameWithDiskNoDiscard() {
        try {
            int numRounds = 5;
            Random random = new Random();
            IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
            // No spill, No discard
            FeedPolicyAccessor fpa = createFeedPolicyAccessor(true, false, NUM_FRAMES * DEFAULT_FRAME_SIZE,
                    DISCARD_ALLOWANCE);
            // Non-Active Writer
            TestFrameWriter writer = FrameWriterTestUtils.create(Collections.emptyList(), Collections.emptyList(),
                    false);
            // FramePool
            ConcurrentFramePool framePool = new ConcurrentFramePool(NODE_ID, 0, DEFAULT_FRAME_SIZE);
            FeedRuntimeInputHandler handler = createInputHandler(ctx, writer, fpa, framePool);
            handler.open();
            ByteBuffer buffer = ByteBuffer.allocate(DEFAULT_FRAME_SIZE);
            handler.nextFrame(buffer);
            Assert.assertEquals(0, handler.getNumProcessedInMemory());
            Assert.assertEquals(1, handler.getNumSpilled());
            // add NUM_FRAMES times
            for (int i = 0; i < NUM_FRAMES * numRounds; i++) {
                int multiplier = random.nextInt(10) + 1;
                buffer = ByteBuffer.allocate(DEFAULT_FRAME_SIZE * multiplier);
                handler.nextFrame(buffer);
            }
            // Check that no records were discarded
            Assert.assertEquals(handler.getNumDiscarded(), 0);
            // Check that no records were spilled
            Assert.assertEquals(NUM_FRAMES * numRounds + 1, handler.getNumSpilled());
            writer.validate(false);
            handler.close();
            // Check that nextFrame was called
            Assert.assertEquals(NUM_FRAMES * numRounds + 1, writer.nextFrameCount());
            writer.validate(true);
        } catch (Throwable th) {
            th.printStackTrace();
            Assert.fail();
        } finally {
            Assert.assertNull(cause);
        }
    }

    @Test
    public void testZeroMemoryFixedSizeFrameWithDiskNoDiscard() {
        try {
            int numRounds = 10;
            IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
            // No spill, No discard
            FeedPolicyAccessor fpa = createFeedPolicyAccessor(true, false, NUM_FRAMES * DEFAULT_FRAME_SIZE,
                    DISCARD_ALLOWANCE);
            // Non-Active Writer
            TestFrameWriter writer = FrameWriterTestUtils.create(Collections.emptyList(), Collections.emptyList(),
                    false);
            // FramePool
            ConcurrentFramePool framePool = new ConcurrentFramePool(NODE_ID, 0, DEFAULT_FRAME_SIZE);
            FeedRuntimeInputHandler handler = createInputHandler(ctx, writer, fpa, framePool);
            handler.open();
            VSizeFrame frame = new VSizeFrame(ctx);
            handler.nextFrame(frame.getBuffer());
            Assert.assertEquals(0, handler.getNumProcessedInMemory());
            Assert.assertEquals(1, handler.getNumSpilled());
            // add NUM_FRAMES times
            for (int i = 0; i < NUM_FRAMES * numRounds; i++) {
                handler.nextFrame(frame.getBuffer());
            }
            // Check that no records were discarded
            Assert.assertEquals(handler.getNumDiscarded(), 0);
            // Check that no records were spilled
            Assert.assertEquals(NUM_FRAMES * numRounds + 1, handler.getNumSpilled());
            writer.validate(false);
            handler.close();
            // Check that nextFrame was called
            Assert.assertEquals(NUM_FRAMES * numRounds + 1, writer.nextFrameCount());
            writer.validate(true);
        } catch (Throwable th) {
            th.printStackTrace();
            Assert.fail();
        } finally {
            Assert.assertNull(cause);
        }
    }

    /*
     * Spill = false;
     * Discard = true; discard only 5%
     * Fixed size frames
     */
    @Test
    public void testMemoryVarSizeFrameWithSpillWithDiscard() {
        try {
            int numberOfMemoryFrames = 50;
            int numberOfSpillFrames = 50;
            int notDiscarded = 0;
            int totalMinFrames = 0;
            IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
            // Spill budget = Memory budget, No discard
            FeedPolicyAccessor fpa = createFeedPolicyAccessor(true, true, DEFAULT_FRAME_SIZE * numberOfSpillFrames,
                    DISCARD_ALLOWANCE);
            // Non-Active Writer
            TestControlledFrameWriter writer = FrameWriterTestUtils.create(DEFAULT_FRAME_SIZE, false);
            writer.freeze();
            // FramePool
            ConcurrentFramePool framePool = new ConcurrentFramePool(NODE_ID,
                    numberOfMemoryFrames * DEFAULT_FRAME_SIZE, DEFAULT_FRAME_SIZE);
            FeedRuntimeInputHandler handler = createInputHandler(ctx, writer, fpa, framePool);
            handler.open();
            ByteBuffer buffer1 = ByteBuffer.allocate(DEFAULT_FRAME_SIZE);
            ByteBuffer buffer2 = ByteBuffer.allocate(DEFAULT_FRAME_SIZE * 2);
            ByteBuffer buffer3 = ByteBuffer.allocate(DEFAULT_FRAME_SIZE * 3);
            ByteBuffer buffer4 = ByteBuffer.allocate(DEFAULT_FRAME_SIZE * 4);
            ByteBuffer buffer5 = ByteBuffer.allocate(DEFAULT_FRAME_SIZE * 5);
            while (true) {
                if (totalMinFrames + 1 < numberOfMemoryFrames) {
                    handler.nextFrame(buffer1);
                    notDiscarded++;
                    totalMinFrames++;
                } else {
                    break;
                }
                if (totalMinFrames + 2 < numberOfMemoryFrames) {
                    notDiscarded++;
                    totalMinFrames += 2;
                    handler.nextFrame(buffer2);
                } else {
                    break;
                }
                if (totalMinFrames + 3 < numberOfMemoryFrames) {
                    notDiscarded++;
                    totalMinFrames += 3;
                    handler.nextFrame(buffer3);
                } else {
                    break;
                }
            }
            // Now we need to verify that the frame pool memory has been consumed!
            Assert.assertTrue(framePool.remaining() < 3);
            Assert.assertEquals(0, handler.getNumSpilled());
            Assert.assertEquals(0, handler.getNumStalled());
            Assert.assertEquals(0, handler.getNumDiscarded());
            while (true) {
                if (handler.getNumSpilled() < numberOfSpillFrames) {
                    notDiscarded++;
                    handler.nextFrame(buffer3);
                } else {
                    break;
                }
                if (handler.getNumSpilled() < numberOfSpillFrames) {
                    notDiscarded++;
                    handler.nextFrame(buffer4);
                } else {
                    break;
                }
                if (handler.getNumSpilled() < numberOfSpillFrames) {
                    notDiscarded++;
                    handler.nextFrame(buffer5);
                } else {
                    break;
                }
            }
            Assert.assertTrue(framePool.remaining() < 3);
            Assert.assertEquals(handler.framesOnDisk(), handler.getNumSpilled());
            Assert.assertEquals(handler.framesOnDisk(), numberOfSpillFrames);
            Assert.assertEquals(0, handler.getNumStalled());
            Assert.assertEquals(0, handler.getNumDiscarded());
            // We can only discard one frame
            double numDiscarded = 0;
            boolean nextShouldDiscard = ((numDiscarded + 1.0) / (handler.getTotal() + 1.0)) <= fpa
                    .getMaxFractionDiscard();
            while (nextShouldDiscard) {
                handler.nextFrame(buffer5);
                numDiscarded++;
                nextShouldDiscard = ((numDiscarded + 1.0) / (handler.getTotal() + 1.0)) <= fpa
                        .getMaxFractionDiscard();
            }
            Assert.assertTrue(framePool.remaining() < 3);
            Assert.assertEquals(handler.framesOnDisk(), handler.getNumSpilled());
            Assert.assertEquals(0, handler.getNumStalled());
            Assert.assertEquals((int) numDiscarded, handler.getNumDiscarded());
            // Next Call should block since we're exceeding the discard allowance
            Future<?> result = EXECUTOR.submit(new Pusher(buffer5, handler));
            if (result.isDone()) {
                Assert.fail("The producer should switch to stall mode since it is exceeding the discard allowance");
            }
            // consume memory frames
            writer.unfreeze();
            result.get();
            handler.close();
            Assert.assertEquals(writer.nextFrameCount(), notDiscarded + 1);
        } catch (Throwable th) {
            th.printStackTrace();
            Assert.fail();
        }
        Assert.assertNull(cause);
    }

    /*
     * Spill = true;
     * Discard = true
     * Fixed size frames
     */
    @Test
    public void testMemoryFixedSizeFrameWithSpillWithDiscard() {
        try {
            int numberOfMemoryFrames = 50;
            int numberOfSpillFrames = 50;
            IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
            // Spill budget = Memory budget, No discard
            FeedPolicyAccessor fpa = createFeedPolicyAccessor(true, true, DEFAULT_FRAME_SIZE * numberOfSpillFrames,
                    DISCARD_ALLOWANCE);
            // Non-Active Writer
            TestControlledFrameWriter writer = FrameWriterTestUtils.create(DEFAULT_FRAME_SIZE, false);
            writer.freeze();
            // FramePool
            ConcurrentFramePool framePool = new ConcurrentFramePool(NODE_ID,
                    numberOfMemoryFrames * DEFAULT_FRAME_SIZE, DEFAULT_FRAME_SIZE);
            FeedRuntimeInputHandler handler = createInputHandler(ctx, writer, fpa, framePool);
            handler.open();
            VSizeFrame frame = new VSizeFrame(ctx);
            for (int i = 0; i < numberOfMemoryFrames; i++) {
                handler.nextFrame(frame.getBuffer());
            }
            // Now we need to verify that the frame pool memory has been consumed!
            Assert.assertEquals(0, framePool.remaining());
            Assert.assertEquals(numberOfMemoryFrames, handler.getTotal());
            Assert.assertEquals(0, handler.getNumSpilled());
            Assert.assertEquals(0, handler.getNumStalled());
            Assert.assertEquals(0, handler.getNumDiscarded());
            for (int i = 0; i < numberOfSpillFrames; i++) {
                handler.nextFrame(frame.getBuffer());
            }
            Assert.assertEquals(0, framePool.remaining());
            Assert.assertEquals(numberOfMemoryFrames + numberOfSpillFrames, handler.getTotal());
            Assert.assertEquals(numberOfSpillFrames, handler.getNumSpilled());
            Assert.assertEquals(0, handler.getNumStalled());
            Assert.assertEquals(0, handler.getNumDiscarded());
            // We can only discard one frame
            double numDiscarded = 0;
            boolean nextShouldDiscard = ((numDiscarded + 1.0) / (handler.getTotal() + 1.0)) <= fpa
                    .getMaxFractionDiscard();
            while (nextShouldDiscard) {
                handler.nextFrame(frame.getBuffer());
                numDiscarded++;
                nextShouldDiscard = (numDiscarded + 1.0) / (handler.getTotal() + 1.0) <= fpa
                        .getMaxFractionDiscard();
            }
            Assert.assertEquals(0, framePool.remaining());
            Assert.assertEquals((int) (numberOfMemoryFrames + numberOfSpillFrames + numDiscarded),
                    handler.getTotal());
            Assert.assertEquals(numberOfSpillFrames, handler.getNumSpilled());
            Assert.assertEquals(0, handler.getNumStalled());
            Assert.assertEquals((int) numDiscarded, handler.getNumDiscarded());
            // Next Call should block since we're exceeding the discard allowance
            Future<?> result = EXECUTOR.submit(new Pusher(frame.getBuffer(), handler));
            if (result.isDone()) {
                Assert.fail("The producer should switch to stall mode since it is exceeding the discard allowance");
            } else {
                Assert.assertEquals((int) numDiscarded, handler.getNumDiscarded());
            }
            // consume memory frames
            writer.unfreeze();
            result.get();
            handler.close();
            Assert.assertTrue(result.isDone());
            Assert.assertEquals(writer.nextFrameCount(), numberOfMemoryFrames + numberOfSpillFrames + 1);
        } catch (Throwable th) {
            th.printStackTrace();
            Assert.fail();
        }
        Assert.assertNull(cause);
    }

    /*
     * Spill = false;
     * Discard = true; discard only 5%
     * Fixed size frames
     */
    @Test
    public void testMemoryVariableSizeFrameNoSpillWithDiscard() {
        try {
            int discardTestFrames = 100;
            Random random = new Random();
            IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
            // Spill budget = Memory budget, No discard
            FeedPolicyAccessor fpa = createFeedPolicyAccessor(false, true, DEFAULT_FRAME_SIZE, DISCARD_ALLOWANCE);
            // Non-Active Writer
            TestControlledFrameWriter writer = FrameWriterTestUtils.create(DEFAULT_FRAME_SIZE, false);
            writer.freeze();
            // FramePool
            ConcurrentFramePool framePool = new ConcurrentFramePool(NODE_ID, discardTestFrames * DEFAULT_FRAME_SIZE,
                    DEFAULT_FRAME_SIZE);
            FeedRuntimeInputHandler handler = createInputHandler(ctx, writer, fpa, framePool);
            handler.open();
            // add NUM_FRAMES times
            ByteBuffer buffer = ByteBuffer.allocate(DEFAULT_FRAME_SIZE);
            int multiplier = 1;
            int numFrames = 0;
            // add NUM_FRAMES times
            while ((multiplier <= framePool.remaining())) {
                numFrames++;
                handler.nextFrame(buffer);
                multiplier = random.nextInt(10) + 1;
                buffer = ByteBuffer.allocate(DEFAULT_FRAME_SIZE * multiplier);
            }
            // Next call should NOT block but should discard.
            double numDiscarded = 0.0;
            boolean nextShouldDiscard = ((numDiscarded + 1.0) / (handler.getTotal() + 1.0)) <= fpa
                    .getMaxFractionDiscard();
            while (nextShouldDiscard) {
                handler.nextFrame(buffer);
                numDiscarded++;
                nextShouldDiscard = ((numDiscarded + 1.0) / (handler.getTotal() + 1.0)) <= fpa
                        .getMaxFractionDiscard();
            }
            Future<?> result = EXECUTOR.submit(new Pusher(buffer, handler));
            if (result.isDone()) {
                Assert.fail("The producer should switch to stall mode since it is exceeding the discard allowance");
            } else {
                // Check that no records were discarded
                Assert.assertEquals((int) numDiscarded, handler.getNumDiscarded());
                // Check that one frame is spilled
                Assert.assertEquals(handler.getNumSpilled(), 0);
            }
            // consume memory frames
            writer.unfreeze();
            result.get();
            handler.close();
            Assert.assertEquals(writer.nextFrameCount(), numFrames + 1);
            // exit
        } catch (Throwable th) {
            th.printStackTrace();
            Assert.fail();
        }
        Assert.assertNull(cause);
    }

    /*
     * Spill = false;
     * Discard = true; discard only 5%
     * Fixed size frames
     */
    @Test
    public void testMemoryFixedSizeFrameNoSpillWithDiscard() {
        try {
            int discardTestFrames = 100;
            IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
            // Spill budget = Memory budget, No discard
            FeedPolicyAccessor fpa = createFeedPolicyAccessor(false, true, DEFAULT_FRAME_SIZE, DISCARD_ALLOWANCE);
            // Non-Active Writer
            TestControlledFrameWriter writer = FrameWriterTestUtils.create(DEFAULT_FRAME_SIZE, false);
            writer.freeze();
            // FramePool
            ConcurrentFramePool framePool = new ConcurrentFramePool(NODE_ID, discardTestFrames * DEFAULT_FRAME_SIZE,
                    DEFAULT_FRAME_SIZE);
            FeedRuntimeInputHandler handler = createInputHandler(ctx, writer, fpa, framePool);
            handler.open();
            VSizeFrame frame = new VSizeFrame(ctx);
            // add NUM_FRAMES times
            for (int i = 0; i < discardTestFrames; i++) {
                handler.nextFrame(frame.getBuffer());
            }
            // Next 5 calls call should NOT block but should discard.
            double numDiscarded = 0.0;
            boolean nextShouldDiscard = ((numDiscarded + 1.0) / (handler.getTotal() + 1.0)) <= fpa
                    .getMaxFractionDiscard();
            while (nextShouldDiscard) {
                handler.nextFrame(frame.getBuffer());
                numDiscarded++;
                nextShouldDiscard = ((numDiscarded + 1.0) / (handler.getTotal() + 1.0)) <= fpa
                        .getMaxFractionDiscard();
            }
            // Next Call should block since we're exceeding the discard allowance
            Future<?> result = EXECUTOR.submit(new Pusher(frame.getBuffer(), handler));
            if (result.isDone()) {
                Assert.fail("The producer should switch to stall mode since it is exceeding the discard allowance");
            } else {
                // Check that no records were discarded
                Assert.assertEquals((int) numDiscarded, handler.getNumDiscarded());
                // Check that one frame is spilled
                Assert.assertEquals(handler.getNumSpilled(), 0);
            }
            // consume memory frames
            writer.unfreeze();
            result.get();
            handler.close();
            Assert.assertEquals(writer.nextFrameCount(), discardTestFrames + 1);
            // exit
        } catch (Throwable th) {
            th.printStackTrace();
            Assert.fail();
        }
        Assert.assertNull(cause);
    }

    /*
     * Spill = true;
     * Discard = false;
     * Fixed size frames
     */
    @Test
    public void testMemoryFixedSizeFrameWithSpillNoDiscard() {
        try {
            IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
            // Spill budget = Memory budget, No discard
            FeedPolicyAccessor fpa = createFeedPolicyAccessor(true, false, DEFAULT_FRAME_SIZE * NUM_FRAMES,
                    DISCARD_ALLOWANCE);
            // Non-Active Writer
            TestControlledFrameWriter writer = FrameWriterTestUtils.create(DEFAULT_FRAME_SIZE, false);
            writer.freeze();
            // FramePool
            ConcurrentFramePool framePool = new ConcurrentFramePool(NODE_ID, FEED_MEM_BUDGET, DEFAULT_FRAME_SIZE);
            FeedRuntimeInputHandler handler = createInputHandler(ctx, writer, fpa, framePool);
            handler.open();
            VSizeFrame frame = new VSizeFrame(ctx);
            // add NUM_FRAMES times
            for (int i = 0; i < NUM_FRAMES; i++) {
                handler.nextFrame(frame.getBuffer());
            }
            // Next call should NOT block. we will do it in a different thread
            Future<?> result = EXECUTOR.submit(new Pusher(frame.getBuffer(), handler));
            result.get();
            // Check that no records were discarded
            Assert.assertEquals(handler.getNumDiscarded(), 0);
            // Check that one frame is spilled
            Assert.assertEquals(handler.getNumSpilled(), 1);
            // consume memory frames
            writer.unfreeze();
            handler.close();
            Assert.assertEquals(handler.framesOnDisk(), 0);
            // exit
        } catch (Throwable th) {
            th.printStackTrace();
            Assert.fail();
        }
        Assert.assertNull(cause);
    }

    /*
     * Spill = false;
     * Discard = false;
     * Fixed size frames
     * Very fast next operator
     */
    @Test
    public void testMemoryFixedSizeFrameNoDiskNoDiscardFastConsumer() {
        try {
            int numRounds = 10;
            IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
            // No spill, No discard
            FeedPolicyAccessor fpa = createFeedPolicyAccessor(false, false, 0L, DISCARD_ALLOWANCE);
            // Non-Active Writer
            TestFrameWriter writer = FrameWriterTestUtils.create(Collections.emptyList(), Collections.emptyList(),
                    false);
            // FramePool
            ConcurrentFramePool framePool = new ConcurrentFramePool(NODE_ID, FEED_MEM_BUDGET, DEFAULT_FRAME_SIZE);
            FeedRuntimeInputHandler handler = createInputHandler(ctx, writer, fpa, framePool);
            handler.open();
            VSizeFrame frame = new VSizeFrame(ctx);
            // add NUM_FRAMES times
            for (int i = 0; i < NUM_FRAMES * numRounds; i++) {
                handler.nextFrame(frame.getBuffer());
            }
            // Check that no records were discarded
            Assert.assertEquals(handler.getNumDiscarded(), 0);
            // Check that no records were spilled
            Assert.assertEquals(handler.getNumSpilled(), 0);
            writer.validate(false);
            handler.close();
            // Check that nextFrame was called
            Assert.assertEquals(NUM_FRAMES * numRounds, writer.nextFrameCount());
            writer.validate(true);
        } catch (Throwable th) {
            th.printStackTrace();
            Assert.fail();
        }
        Assert.assertNull(cause);
    }

    /*
     * Spill = false;
     * Discard = false;
     * Fixed size frames
     * Slow next operator
     */
    @Test
    public void testMemoryFixedSizeFrameNoDiskNoDiscardSlowConsumer() {
        try {
            int numRounds = 10;
            IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
            // No spill, No discard
            FeedPolicyAccessor fpa = createFeedPolicyAccessor(false, false, 0L, DISCARD_ALLOWANCE);
            // Non-Active Writer
            TestFrameWriter writer = FrameWriterTestUtils.create(Collections.emptyList(), Collections.emptyList(),
                    false);
            // FramePool
            ConcurrentFramePool framePool = new ConcurrentFramePool(NODE_ID, FEED_MEM_BUDGET, DEFAULT_FRAME_SIZE);
            FeedRuntimeInputHandler handler = createInputHandler(ctx, writer, fpa, framePool);
            handler.open();
            VSizeFrame frame = new VSizeFrame(ctx);
            writer.setNextDuration(1);
            // add NUM_FRAMES times
            for (int i = 0; i < NUM_FRAMES * numRounds; i++) {
                handler.nextFrame(frame.getBuffer());
            }
            // Check that no records were discarded
            Assert.assertEquals(handler.getNumDiscarded(), 0);
            // Check that no records were spilled
            Assert.assertEquals(handler.getNumSpilled(), 0);
            // Check that nextFrame was called
            writer.validate(false);
            handler.close();
            Assert.assertEquals(writer.nextFrameCount(), (NUM_FRAMES * numRounds));
            writer.validate(true);
        } catch (Throwable th) {
            th.printStackTrace();
            Assert.fail();
        }
        Assert.assertNull(cause);
    }

    /*
     * Spill = false
     * Discard = false
     * VarSizeFrame
     */
    @Test
    public void testMemoryVarSizeFrameNoDiskNoDiscard() {
        try {
            Random random = new Random();
            IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
            // No spill, No discard
            FeedPolicyAccessor fpa = createFeedPolicyAccessor(false, false, 0L, DISCARD_ALLOWANCE);
            // Non-Active Writer
            TestControlledFrameWriter writer = FrameWriterTestUtils.create(DEFAULT_FRAME_SIZE, false);
            writer.freeze();
            // FramePool
            ConcurrentFramePool framePool = new ConcurrentFramePool(NODE_ID, FEED_MEM_BUDGET, DEFAULT_FRAME_SIZE);
            FeedRuntimeInputHandler handler = createInputHandler(ctx, writer, fpa, framePool);
            handler.open();
            ByteBuffer buffer = ByteBuffer.allocate(DEFAULT_FRAME_SIZE);
            int multiplier = 1;
            // add NUM_FRAMES times
            while ((multiplier <= framePool.remaining())) {
                handler.nextFrame(buffer);
                multiplier = random.nextInt(10) + 1;
                buffer = ByteBuffer.allocate(DEFAULT_FRAME_SIZE * multiplier);
            }
            // we can't satisfy the next request
            // Next call should block we will do it in a different thread
            Future<?> result = EXECUTOR.submit(new Pusher(buffer, handler));
            // Check that the nextFrame didn't return
            if (result.isDone()) {
                Assert.fail();
            }
            // Check that no records were discarded
            Assert.assertEquals(handler.getNumDiscarded(), 0);
            // Check that no records were spilled
            Assert.assertEquals(handler.getNumSpilled(), 0);
            // Check that number of stalled is not greater than 1
            Assert.assertTrue(handler.getNumStalled() <= 1);
            writer.unfreeze();
            handler.close();
            result.get();
        } catch (Throwable th) {
            th.printStackTrace();
            Assert.fail();
        }
        Assert.assertNull(cause);
    }

    /*
     * Spill = true;
     * Discard = false;
     * Variable size frames
     */
    @Test
    public void testMemoryVarSizeFrameWithSpillNoDiscard() {
        for (int k = 0; k < 1000; k++) {
            try {
                Random random = new Random();
                IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
                // Spill budget = Memory budget, No discard
                FeedPolicyAccessor fpa = createFeedPolicyAccessor(true, false, DEFAULT_FRAME_SIZE * NUM_FRAMES,
                        DISCARD_ALLOWANCE);
                // Non-Active Writer
                TestControlledFrameWriter writer = FrameWriterTestUtils.create(DEFAULT_FRAME_SIZE, false);
                writer.freeze();
                // FramePool
                ConcurrentFramePool framePool = new ConcurrentFramePool(NODE_ID, FEED_MEM_BUDGET,
                        DEFAULT_FRAME_SIZE);
                FeedRuntimeInputHandler handler = createInputHandler(ctx, writer, fpa, framePool);
                handler.open();
                ByteBuffer buffer = ByteBuffer.allocate(DEFAULT_FRAME_SIZE);
                int multiplier = 1;
                int numOfBuffersInMemory = 0;
                // add NUM_FRAMES times
                while ((multiplier <= framePool.remaining())) {
                    numOfBuffersInMemory++;
                    handler.nextFrame(buffer);
                    multiplier = random.nextInt(10) + 1;
                    buffer = ByteBuffer.allocate(DEFAULT_FRAME_SIZE * multiplier);
                }
                // Next call should Not block. we will do it in a different thread
                Future<?> result = EXECUTOR.submit(new Pusher(buffer, handler));
                result.get();
                // Check that no records were discarded
                Assert.assertEquals(handler.getNumDiscarded(), 0);
                // Check that one frame is spilled
                Assert.assertEquals(handler.getNumSpilled(), 1);
                // consume memory frames
                while (numOfBuffersInMemory > 1) {
                    writer.kick();
                    numOfBuffersInMemory--;
                }
                // There should be 1 frame on disk
                Assert.assertEquals(1, handler.framesOnDisk());
                writer.unfreeze();
                handler.close();
                Assert.assertEquals(0, handler.framesOnDisk());
            } catch (Throwable th) {
                th.printStackTrace();
                Assert.fail();
            }
        }
        Assert.assertNull(cause);
    }

    /*
     * Spill = false;
     * Discard = false;
     * Fixed size frames
     */
    @Test
    public void testMemoryFixedSizeFrameNoDiskNoDiscard() {
        try {
            IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE);
            // No spill, No discard
            FeedPolicyAccessor fpa = createFeedPolicyAccessor(false, false, 0L, DISCARD_ALLOWANCE);
            // Non-Active Writer
            TestControlledFrameWriter writer = FrameWriterTestUtils.create(DEFAULT_FRAME_SIZE, false);
            writer.freeze();
            // FramePool
            ConcurrentFramePool framePool = new ConcurrentFramePool(NODE_ID, FEED_MEM_BUDGET, DEFAULT_FRAME_SIZE);

            FeedRuntimeInputHandler handler = createInputHandler(ctx, writer, fpa, framePool);
            handler.open();
            VSizeFrame frame = new VSizeFrame(ctx);
            // add NUM_FRAMES times
            for (int i = 0; i < NUM_FRAMES; i++) {
                handler.nextFrame(frame.getBuffer());
            }
            // Next call should block we will do it in a different thread
            Future<?> result = EXECUTOR.submit(new Pusher(frame.getBuffer(), handler));
            // Check that the nextFrame didn't return
            if (result.isDone()) {
                Assert.fail();
            } else {
                // Check that no records were discarded
                Assert.assertEquals(handler.getNumDiscarded(), 0);
                // Check that no records were spilled
                Assert.assertEquals(handler.getNumSpilled(), 0);
                // Check that no records were discarded
                // Check that the inputHandler subscribed to the framePool
                // Check that number of stalled is not greater than 1
                Assert.assertTrue(handler.getNumStalled() <= 1);
                writer.kick();
            }
            result.get();
            writer.unfreeze();
            handler.close();
        } catch (Throwable th) {
            th.printStackTrace();
            Assert.fail();
        }
        Assert.assertNull(cause);
    }

    private class Pusher implements Runnable {
        private final ByteBuffer buffer;
        private final IFrameWriter writer;

        public Pusher(ByteBuffer buffer, IFrameWriter writer) {
            this.buffer = buffer;
            this.writer = writer;
        }

        @Override
        public void run() {
            try {
                writer.nextFrame(buffer);
            } catch (HyracksDataException e) {
                e.printStackTrace();
                cause = e;
            }
        }
    }
}