com.datatorrent.contrib.hdht.WALTest.java Source code

Java tutorial

Introduction

Here is the source code for com.datatorrent.contrib.hdht.WALTest.java

Source

/**
 * Copyright (c) 2016 DataTorrent, Inc. ALL Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package com.datatorrent.contrib.hdht;

import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Random;

import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.apache.commons.io.FileUtils;

import com.esotericsoftware.kryo.Kryo;
import com.google.common.util.concurrent.MoreExecutors;

import com.datatorrent.api.Attribute.AttributeMap.DefaultAttributeMap;
import com.datatorrent.contrib.hdht.wal.FSWALReader;
import com.datatorrent.contrib.hdht.wal.FSWALWriter;
import com.datatorrent.lib.fileaccess.FileAccessFSImpl;
import com.datatorrent.lib.helper.OperatorContextTestHelper;
import com.datatorrent.lib.util.KryoCloneUtils;
import com.datatorrent.netlet.util.Slice;

public class WALTest {
    static final Random rand = new Random();

    File file = new File("target/hds");

    static byte[] genRandomByteArray(int len) {
        byte[] val = new byte[len];
        rand.nextBytes(val);
        return val;
    }

    static Slice genRandomKey(int len) {
        byte[] val = new byte[len];
        rand.nextBytes(val);
        return new Slice(val);
    }

    /**
     * - Write some data to WAL
     * - Read the data back. The amount of data read should be
     *   same as amount of data written.
     * @throws IOException
     */
    @Test
    public void testWalWriteAndRead() throws IOException {
        FileUtils.deleteDirectory(file);
        FileAccessFSImpl bfs = new MockFileAccess();
        bfs.setBasePath(file.getAbsolutePath());
        bfs.init();

        int keySize = 100;
        int valSize = 110;
        int delKeySize = 50;
        int purgeKeySize = 60;
        int numTuples = 100;
        int numPuts = 0;
        int numDeletes = 0;
        int numPurges = 0;

        FSWALWriter wWriter = new FSWALWriter(bfs, new HDHTLogEntry.HDHTLogSerializer(), 1, "WAL-0");
        for (int i = 0; i < numTuples; i++) {
            int type = rand.nextInt(3);
            switch (type) {
            case 0:
                wWriter.append(new HDHTLogEntry.PutEntry(0, genRandomKey(keySize), genRandomByteArray(valSize)));
                numPuts++;
                break;
            case 1:
                wWriter.append(
                        new HDHTLogEntry.PurgeEntry(0, genRandomKey(purgeKeySize), genRandomKey(purgeKeySize)));
                numPurges++;
                break;
            case 2:
                wWriter.append(new HDHTLogEntry.DeleteEntry(0, genRandomKey(delKeySize)));
                numDeletes++;
                break;
            default:
            }
        }
        wWriter.close();
        File wal0 = new File(file.getAbsoluteFile().toString() + "/1/WAL-0");
        Assert.assertEquals("WAL file created ", true, wal0.exists());

        FSWALReader wReader = new FSWALReader(bfs, new HDHTLogEntry.HDHTLogSerializer(), 1, "WAL-0");
        int tuples = 0;
        int puts = 0;
        int purges = 0;
        int deletes = 0;
        while (wReader.advance()) {
            HDHTLogEntry.HDHTWalEntry entry = (HDHTLogEntry.HDHTWalEntry) wReader.get();
            logger.debug("entry read {}", entry);
            if (entry instanceof HDHTLogEntry.PutEntry) {
                HDHTLogEntry.PutEntry keyVal = (HDHTLogEntry.PutEntry) entry;
                Assert.assertEquals("Key size ", keySize, keyVal.key.length);
                Assert.assertEquals("Value size ", valSize, keyVal.val.length);
                puts++;
            } else if (entry instanceof HDHTLogEntry.PurgeEntry) {
                HDHTLogEntry.PurgeEntry purge = (HDHTLogEntry.PurgeEntry) entry;
                Assert.assertEquals("Purge start key size", purgeKeySize, purge.startKey.length);
                Assert.assertEquals("Purge end key size", purgeKeySize, purge.endKey.length);
                purges++;
            } else if (entry instanceof HDHTLogEntry.DeleteEntry) {
                HDHTLogEntry.DeleteEntry del = (HDHTLogEntry.DeleteEntry) entry;
                Assert.assertEquals("Del key size ", delKeySize, del.key.length);
                deletes++;
            }
            tuples++;
        }
        wReader.close();
        Assert.assertEquals("Write and read same number of tuples ", numTuples, tuples);
        Assert.assertEquals("Number of puts ", numPuts, puts);
        Assert.assertEquals("Number of purges", numPurges, purges);
        Assert.assertEquals("Number of deletes ", numDeletes, deletes);

    }

    /**
     * Read WAL from middle of the file by seeking to known valid
     * offset and start reading from that point till the end.
     */
    @Test
    public void testWalSkip() throws IOException {
        FileUtils.deleteDirectory(file);
        FileAccessFSImpl bfs = new MockFileAccess();
        bfs.setBasePath(file.getAbsolutePath());
        bfs.init();

        long offset = 0;

        FSWALWriter wWriter = new FSWALWriter(bfs, new HDHTLogEntry.HDHTLogSerializer(), 1, "WAL-0");
        int totalTuples = 100;
        int recoveryTuples = 30;
        for (int i = 0; i < totalTuples; i++) {
            wWriter.append(new HDHTLogEntry.PutEntry(0, genRandomKey(100), genRandomByteArray(100)));
            if (i == recoveryTuples) {
                offset = wWriter.getSize();
            }
        }
        logger.info("total file size is " + wWriter.getSize() + " recovery offset is " + offset);
        wWriter.close();

        FSWALReader wReader = new FSWALReader(bfs, new HDHTLogEntry.HDHTLogSerializer(), 1, "WAL-0");
        wReader.seek(offset);
        int read = 0;
        while (wReader.advance()) {
            read++;
            wReader.get();
        }
        wReader.close();

        Assert.assertEquals("Number of tuples read after skipping", read, (totalTuples - recoveryTuples - 1));
    }

    /**
     * Test WAL rolling functionality, set maximumWal size to 1024.
     * Write some data which will go over WAL size.
     * call endWindow
     * Write some more data.
     * Two files should be created.
     * @throws IOException
     */
    @Test
    public void testWalRolling() throws IOException {
        File file = new File("target/hds");
        FileUtils.deleteDirectory(file);
        final long BUCKET1 = 1L;
        final int OPERATOR_ID = 1;

        FileAccessFSImpl bfs = new MockFileAccess();
        bfs.setBasePath(file.getAbsolutePath());
        bfs.init();

        HDHTWriter hds = new HDHTWriter();
        hds.setFileStore(bfs);
        hds.setKeyComparator(new HDHTWriterTest.SequenceComparator());
        hds.setFlushIntervalCount(5);
        hds.setFlushSize(1000);
        hds.setMaxWalFileSize(1024);

        hds.setup(new OperatorContextTestHelper.TestIdOperatorContext(OPERATOR_ID, new DefaultAttributeMap()));
        hds.writeExecutor = MoreExecutors.sameThreadExecutor();

        hds.beginWindow(0);
        hds.put(BUCKET1, genRandomKey(500), genRandomByteArray(500));
        hds.put(BUCKET1, genRandomKey(500), genRandomByteArray(500));
        hds.endWindow();

        hds.beginWindow(1);
        hds.put(BUCKET1, genRandomKey(500), genRandomByteArray(500));
        hds.put(BUCKET1, genRandomKey(500), genRandomByteArray(500));
        hds.endWindow();
        hds.forceWal();

        File wal0 = new File(file.getAbsoluteFile().toString() + "/WAL/1/_WAL-0");
        Assert.assertEquals("New Wal-0 created ", wal0.exists(), true);

        File wal1 = new File(file.getAbsoluteFile().toString() + "/WAL/1/_WAL-1");
        Assert.assertEquals("New Wal-1 created ", wal1.exists(), true);
    }

    /**
     * Rest recovery of operator cache. Steps
     * - Add some tuples
     * - Flush data to disk.
     * - Add some more tuples, which are not flushed to data, but flushed to WAL.
     * - Save WAL state (operator checkpoint)
     * - Add a tuple to start recovery from tuples.
     * @throws IOException
     */
    @Test
    public void testWalRecovery() throws IOException {
        File file = new File("target/hds");
        FileUtils.deleteDirectory(file);

        FileUtils.deleteDirectory(file);
        FileAccessFSImpl bfs = new MockFileAccess();
        bfs.setBasePath(file.getAbsolutePath());
        bfs.init();

        HDHTWriter hds = new HDHTWriter();
        hds.setFileStore(bfs);
        hds.setKeyComparator(new HDHTWriterTest.SequenceComparator());
        hds.setFlushSize(1);

        hds.setup(new OperatorContextTestHelper.TestIdOperatorContext(1, new DefaultAttributeMap()));
        hds.writeExecutor = MoreExecutors.sameThreadExecutor();

        hds.beginWindow(1);
        hds.put(1, genRandomKey(500), genRandomByteArray(500));
        hds.put(1, genRandomKey(500), genRandomByteArray(500));
        hds.endWindow();
        hds.checkpointed(1);

        hds.beginWindow(2);
        hds.put(1, genRandomKey(500), genRandomByteArray(500));
        hds.put(1, genRandomKey(500), genRandomByteArray(500));
        hds.endWindow();
        hds.checkpointed(2);
        hds.committed(2);

        // Tuples added till this point is written to data files,
        // Tuples being added in this window, will not be written to data files
        // but will be saved in WAL. These should get recovered when bucket
        // is initialized for use next time.
        hds.beginWindow(3);
        hds.put(1, genRandomKey(500), genRandomByteArray(500));
        hds.put(1, genRandomKey(500), genRandomByteArray(500));
        hds.endWindow();
        hds.checkpointed(3);

        hds.forceWal();
        hds.teardown();

        /* Get a check-pointed state of the WAL */
        HDHTWriter newOperator = KryoCloneUtils.cloneObject(new Kryo(), hds);

        newOperator.setKeyComparator(new HDHTWriterTest.SequenceComparator());
        newOperator.setFlushIntervalCount(1);
        newOperator.setFlushSize(3);
        newOperator.writeExecutor = MoreExecutors.sameThreadExecutor();

        newOperator.setFileStore(bfs);
        newOperator.setup(new OperatorContextTestHelper.TestIdOperatorContext(1, new DefaultAttributeMap()));

        // This should run recovery, as first tuple is added in bucket
        newOperator.beginWindow(4);
        newOperator.put(1, genRandomKey(500), genRandomByteArray(500));

        // current tuple, being added is put into write cache.
        Assert.assertEquals("Number of tuples in write cache ", 1, newOperator.unflushedDataSize(1));

        // two tuples are put in to committed write cache.
        Assert.assertEquals("Number of tuples in committed cache ", 2, newOperator.committedDataSize(1));

        newOperator.put(1, genRandomKey(500), genRandomByteArray(500));
        newOperator.put(1, genRandomKey(500), genRandomByteArray(500));
        newOperator.put(1, genRandomKey(500), genRandomByteArray(500));
        newOperator.endWindow();
        newOperator.forceWal();

        File wal1 = new File(file.getAbsoluteFile().toString() + "/WAL/1/_WAL-1");
        Assert.assertEquals("New Wal-1 created ", wal1.exists(), true);
    }

    /**
     * Test WAL cleanup functionality, WAL file is deleted, once data
     * from it is written to data files.
     * @throws IOException
     */
    @Test
    public void testOldWalCleanup() throws IOException {
        File file = new File("target/hds");
        FileUtils.deleteDirectory(file);
        final long BUCKET1 = 1L;
        final int OPERATOR_ID = 1;

        FileAccessFSImpl bfs = new MockFileAccess();
        bfs.setBasePath(file.getAbsolutePath());
        bfs.init();

        HDHTWriter hds = new HDHTWriter();
        hds.setFileStore(bfs);
        hds.setKeyComparator(new HDHTWriterTest.SequenceComparator());
        // Flush at every window.
        hds.setFlushIntervalCount(2);
        hds.setFlushSize(1000);
        hds.setMaxWalFileSize(4000);
        hds.setup(new OperatorContextTestHelper.TestIdOperatorContext(1, new DefaultAttributeMap()));
        hds.writeExecutor = MoreExecutors.sameThreadExecutor();

        hds.beginWindow(1);
        hds.put(BUCKET1, genRandomKey(500), genRandomByteArray(500));
        hds.put(BUCKET1, genRandomKey(500), genRandomByteArray(500));
        hds.endWindow();

        hds.beginWindow(2);
        hds.put(BUCKET1, genRandomKey(500), genRandomByteArray(500));
        hds.put(BUCKET1, genRandomKey(500), genRandomByteArray(500));
        // log file will roll at this point because of limit on WAL file size,
        hds.endWindow();

        File wal0 = new File(file.getAbsoluteFile().toString() + "/WAL/1/_WAL-0");
        Assert.assertEquals("New Wal-0 created ", wal0.exists(), true);

        hds.beginWindow(3);
        hds.put(BUCKET1, genRandomKey(500), genRandomByteArray(500));
        hds.put(BUCKET1, genRandomKey(500), genRandomByteArray(500));
        hds.endWindow();
        hds.checkpointed(3);
        hds.committed(3);
        // Data till this point is committed to disk, and old WAL file WAL-0
        // is deleted, as all data from that file is committed.
        hds.forceWal();

        wal0 = new File(file.getAbsoluteFile().toString() + "/WAL/1/_WAL-0");
        Assert.assertEquals("New Wal-0 deleted ", wal0.exists(), false);

        File wal1 = new File(file.getAbsoluteFile().toString() + "/WAL/1/_WAL-1");
        Assert.assertEquals("New Wal-1 created ", wal1.exists(), true);
    }

    static Slice getLongByteArray(long key) {
        ByteBuffer bb = ByteBuffer.allocate(8);
        bb.putLong(key);
        return new Slice(bb.array());
    }

    /**
    * checkpointed(1)  1 -> 10
    * checkpointed(2)  1 -> 20
    * checkpointed(3)  1 -> 30
    * checkpointed(4)  1 -> 40
    * committed(2)
    * checkpointed(5)
    *
    * restore from 3rd checkpoint.
    * do a get and value should be 30.
    */
    @Test
    public void testWalRecoveryValues() throws IOException {
        File file = new File("target/hds");
        FileUtils.deleteDirectory(file);

        FileAccessFSImpl bfs = new MockFileAccess();
        bfs.setBasePath(file.getAbsolutePath());
        bfs.init();
        ((MockFileAccess) bfs).disableChecksum();

        FileAccessFSImpl walfs = new MockFileAccess();
        walfs.setBasePath(file.getAbsolutePath());
        walfs.init();
        ((MockFileAccess) walfs).disableChecksum();

        HDHTWriter hds = new HDHTWriter();
        hds.setFileStore(bfs);
        hds.setWalStore(walfs);
        hds.setFlushSize(1);
        hds.setFlushIntervalCount(1);
        hds.setup(new OperatorContextTestHelper.TestIdOperatorContext(1, new DefaultAttributeMap()));
        hds.writeExecutor = MoreExecutors.sameThreadExecutor();

        hds.beginWindow(1);
        hds.put(1, getLongByteArray(1), getLongByteArray(10).toByteArray());
        hds.endWindow();
        hds.checkpointed(1);

        hds.beginWindow(2);
        hds.put(1, getLongByteArray(1), getLongByteArray(20).toByteArray());
        hds.endWindow();
        hds.checkpointed(2);

        hds.beginWindow(3);
        hds.put(1, getLongByteArray(1), getLongByteArray(30).toByteArray());
        hds.endWindow();
        hds.checkpointed(3);

        // Commit window id 2
        hds.committed(2);
        // use checkpoint after window 3 for recovery.
        HDHTWriter newOperator = KryoCloneUtils.cloneObject(new Kryo(), hds);

        hds.beginWindow(4);
        hds.put(1, getLongByteArray(1), getLongByteArray(40).toByteArray());
        hds.put(1, getLongByteArray(2), getLongByteArray(200).toByteArray());
        hds.endWindow();
        hds.checkpointed(4);

        hds.beginWindow(5);
        hds.put(1, getLongByteArray(1), getLongByteArray(50).toByteArray());
        hds.put(1, getLongByteArray(2), getLongByteArray(210).toByteArray());
        hds.endWindow();
        hds.checkpointed(5);
        hds.forceWal();

        /* Simulate recovery after failure, checkpoint is restored to after
           processing of window 3.
         */
        newOperator.setFlushIntervalCount(1);
        newOperator.setFileStore(bfs);
        newOperator.setWalStore(bfs);
        newOperator.setFlushSize(1);
        newOperator.setup(new OperatorContextTestHelper.TestIdOperatorContext(1, new DefaultAttributeMap()));
        newOperator.writeExecutor = MoreExecutors.sameThreadExecutor();

        // This should run recovery, as first tuple is added in bucket
        newOperator.beginWindow(4);
        newOperator.put(1, getLongByteArray(1), getLongByteArray(40).toByteArray());
        newOperator.put(1, getLongByteArray(2), getLongByteArray(200).toByteArray());
        // current tuple, being added is put into write cache.
        Assert.assertEquals("Number of tuples in write cache ", 2, newOperator.unflushedDataSize(1));
        // one tuples are put in to committed write cache.
        Assert.assertEquals("Number of tuples in committed cache ", 1, newOperator.committedDataSize(1));
        newOperator.endWindow();
        newOperator.checkpointed(4);
        newOperator.forceWal();
        /* The latest value is recovered from WAL */
        ByteBuffer bb = ByteBuffer.wrap(newOperator.getUncommitted(1, getLongByteArray(1)));
        long l = bb.getLong();
        Assert.assertEquals("Value of 1 is recovered from WAL", 40, l);

        newOperator.committed(3);

        bb = ByteBuffer.wrap(newOperator.get(1, getLongByteArray(1)));
        l = bb.getLong();
        Assert.assertEquals("Value is persisted ", 30, l);

        newOperator.committed(4);
        bb = ByteBuffer.wrap(newOperator.get(1, getLongByteArray(1)));
        l = bb.getLong();
        Assert.assertEquals("Value is persisted ", 40, l);
    }

    /**
     * checkpointed(1)  1 -> 10
     * checkpointed(2)  1 -> 20
     * committed(2)
     * checkpointed(3)  1 -> 30
     * committed(3)
     * checkpointed(4)
     * committed(4)
     *
     * no null pointer exception should occure.
     */
    @Test
    public void testIssue4008() throws IOException {
        File file = new File("target/hds");
        FileUtils.deleteDirectory(file);

        FileAccessFSImpl bfs = new MockFileAccess();
        bfs.setBasePath(file.getAbsolutePath());
        bfs.init();
        ((MockFileAccess) bfs).disableChecksum();

        HDHTWriter hds = new HDHTWriter();
        hds.setFileStore(bfs);
        hds.setFlushSize(2);
        hds.setFlushIntervalCount(1);
        hds.setup(new OperatorContextTestHelper.TestIdOperatorContext(1, new DefaultAttributeMap()));
        hds.writeExecutor = MoreExecutors.sameThreadExecutor();

        hds.beginWindow(1);
        hds.put(1, getLongByteArray(1), getLongByteArray(10).toByteArray());
        hds.endWindow();
        hds.checkpointed(1);

        hds.beginWindow(2);
        hds.put(1, getLongByteArray(1), getLongByteArray(20).toByteArray());
        hds.endWindow();
        hds.checkpointed(2);
        hds.committed(2);

        hds.beginWindow(3);
        hds.put(1, getLongByteArray(1), getLongByteArray(30).toByteArray());
        hds.endWindow();
        hds.checkpointed(3);
        hds.committed(3);

        hds.beginWindow(4);
        hds.endWindow();
        hds.checkpointed(4);
        hds.committed(4);

        /* The latest value is recovered from WAL */
        ByteBuffer bb = ByteBuffer.wrap(hds.get(1, getLongByteArray(1)));
        long l = bb.getLong();
        Assert.assertEquals("Value of 1 is recovered from WAL", 30, l);
    }

    @Test
    public void testMultipleBucketsPerWal() throws IOException {
        File file = new File("target/hds");
        FileUtils.deleteDirectory(file);

        FileAccessFSImpl bfs = new MockFileAccess();
        bfs.setBasePath(file.getAbsolutePath());
        bfs.init();
        ((MockFileAccess) bfs).disableChecksum();

        HDHTWriter hds = new HDHTWriter();
        hds.setFileStore(bfs);
        hds.setFlushSize(1);
        hds.setFlushIntervalCount(1);
        hds.setup(new OperatorContextTestHelper.TestIdOperatorContext(1, new DefaultAttributeMap()));
        hds.writeExecutor = MoreExecutors.sameThreadExecutor();

        hds.beginWindow(1);
        hds.put(1, getLongByteArray(1), getLongByteArray(10).toByteArray());
        hds.put(2, getLongByteArray(2), getLongByteArray(100).toByteArray());
        hds.endWindow();
        hds.checkpointed(1);

        hds.beginWindow(2);
        hds.put(1, getLongByteArray(1), getLongByteArray(20).toByteArray());
        hds.put(2, getLongByteArray(2), getLongByteArray(200).toByteArray());
        hds.endWindow();
        hds.checkpointed(2);

        // Commit window id 3
        hds.committed(3);

        // Check Buckets 1 and 2 are created
        File meta1 = new File(file.getAbsoluteFile().toString() + "/1/_META");
        Assert.assertEquals("New _META created for bucket 1", true, meta1.exists());

        File meta2 = new File(file.getAbsoluteFile().toString() + "/2/_META");
        Assert.assertEquals("New _META created for bucket 2", true, meta2.exists());

        File file1 = new File(file.getAbsoluteFile().toString() + "/1/1-0");
        Assert.assertEquals("New _META created for bucket 1", true, file1.exists());

        File file2 = new File(file.getAbsoluteFile().toString() + "/2/2-0");
        Assert.assertEquals("New _META created for bucket 2", true, file2.exists());

        // Check WAL file is created under /WAL/
        File walFile = new File(file.getAbsoluteFile().toString() + "/WAL/1/_WAL-0");
        Assert.assertEquals("Single WAL file created for buckets 1 & 2", true, walFile.exists());

        File secondWalFile = new File(file.getAbsoluteFile().toString() + "/WAL/2/_WAL-0");
        Assert.assertEquals("No separate WAL file created for bucket 2", false, secondWalFile.exists());
    }

    /**
     * checkpointed(1) bucket 1, key 1 -> 10 bucket 2, key 1 -> 100
     * checkpointed(2) bucket 1, key 1 -> 20 bucket 2, key 1 -> 200
     * checkpointed(3) bucket 1, key 1 -> 30 bucket 2, key 1 -> 300 committed(2)
     * restore from 3rd checkpoint. do a get for bucket 1, key 1 and value should
     * be 20. do a get for bucket 2, key 1 and value should be 200.
     */
    @Test
    public void testMultipleBucketsPerWalRecovery() throws IOException {
        File file = new File("target/hds");
        FileUtils.deleteDirectory(file);

        FileAccessFSImpl bfs = new MockFileAccess();
        bfs.setBasePath(file.getAbsolutePath());
        bfs.init();
        ((MockFileAccess) bfs).disableChecksum();

        FileAccessFSImpl walFs = new MockFileAccess();
        walFs.setBasePath(file.getAbsolutePath() + "/WAL/");
        walFs.init();
        ((MockFileAccess) walFs).disableChecksum();

        HDHTWriter hds = new HDHTWriter();
        hds.setFileStore(bfs);
        hds.setFlushSize(1);
        hds.setFlushIntervalCount(1);
        hds.setup(new OperatorContextTestHelper.TestIdOperatorContext(1, new DefaultAttributeMap()));

        hds.writeExecutor = MoreExecutors.sameThreadExecutor();

        hds.beginWindow(1);
        hds.put(1, getLongByteArray(1), getLongByteArray(10).toByteArray());
        hds.put(2, getLongByteArray(1), getLongByteArray(100).toByteArray());
        hds.endWindow();
        hds.checkpointed(1);

        hds.beginWindow(2);
        hds.put(1, getLongByteArray(1), getLongByteArray(20).toByteArray());
        hds.put(2, getLongByteArray(1), getLongByteArray(200).toByteArray());
        hds.endWindow();
        hds.checkpointed(2);

        hds.beginWindow(3);
        hds.put(1, getLongByteArray(1), getLongByteArray(30).toByteArray());
        hds.put(2, getLongByteArray(1), getLongByteArray(300).toByteArray());
        hds.endWindow();
        hds.checkpointed(3);

        // Commit window id 2
        hds.committed(2);
        HDHTWriter newOperator = KryoCloneUtils.cloneObject(new Kryo(), hds);

        hds.beginWindow(4);
        hds.put(1, getLongByteArray(1), getLongByteArray(40).toByteArray());
        hds.put(1, getLongByteArray(2), getLongByteArray(200).toByteArray());
        hds.put(2, getLongByteArray(1), getLongByteArray(400).toByteArray());
        hds.endWindow();
        hds.checkpointed(4);

        hds.beginWindow(5);
        hds.put(1, getLongByteArray(1), getLongByteArray(50).toByteArray());
        hds.put(1, getLongByteArray(2), getLongByteArray(210).toByteArray());
        hds.put(2, getLongByteArray(1), getLongByteArray(500).toByteArray());
        hds.put(2, getLongByteArray(2), getLongByteArray(100).toByteArray());
        hds.endWindow();
        hds.checkpointed(5);
        hds.forceWal();

        /* Simulate recovery after failure, checkpoint is restored to after
           processing of window 3.
         */

        newOperator.setFlushIntervalCount(1);
        newOperator.setFileStore(bfs);
        newOperator.setWalStore(walFs);
        newOperator.setFlushSize(1);
        newOperator.setup(new OperatorContextTestHelper.TestIdOperatorContext(1, new DefaultAttributeMap()));

        newOperator.writeExecutor = MoreExecutors.sameThreadExecutor();

        // This should run recovery, as first tuple is added in bucket
        newOperator.beginWindow(4);
        newOperator.put(1, getLongByteArray(0), getLongByteArray(60).toByteArray());
        newOperator.endWindow();
        newOperator.checkpointed(4);

        newOperator.beginWindow(5);
        newOperator.put(1, getLongByteArray(2), getLongByteArray(700).toByteArray());
        newOperator.put(2, getLongByteArray(1), getLongByteArray(800).toByteArray());
        newOperator.put(2, getLongByteArray(2), getLongByteArray(1000).toByteArray());
        // current tuple, being added is put into write cache.
        Assert.assertEquals("Number of tuples in write cache ", 1, newOperator.unflushedDataSize(1));
        // one tuples are put in to committed write cache.
        Assert.assertEquals("Number of tuples in committed cache ", 1, newOperator.committedDataSize(1));
        newOperator.endWindow();
        newOperator.checkpointed(5);

        newOperator.beginWindow(6);
        newOperator.put(1, getLongByteArray(3), getLongByteArray(300).toByteArray());
        newOperator.put(1, getLongByteArray(4), getLongByteArray(100).toByteArray());
        newOperator.endWindow();
        Assert.assertEquals("Number of tuples in write cache ", 2, newOperator.unflushedDataSize(1));
        newOperator.checkpointed(6);

        /* The latest value is recovered from WAL */
        Assert.assertEquals("Value of key=1, bucket=1 is recovered from WAL", 30,
                getLong(newOperator.getUncommitted(1, getLongByteArray(1))));
        Assert.assertEquals("Value of key=1, bucket=1 is recovered from WAL", 60,
                getLong(newOperator.getUncommitted(1, getLongByteArray(0))));
        Assert.assertEquals("Value of key=2, bucket=1 is recovered from WAL", 700,
                getLong(newOperator.getUncommitted(1, getLongByteArray(2))));
        Assert.assertEquals("Value of key=1, bucket=2 is recovered from WAL", 800,
                getLong(newOperator.getUncommitted(2, getLongByteArray(1))));
        Assert.assertEquals("Value of  key=2, bucket=2 is recovered from WAL", 1000,
                getLong(newOperator.getUncommitted(2, getLongByteArray(2))));

        /*Committed value check*/
        Assert.assertEquals("Bucket 1, Key 1 value should be 20", 20,
                getLong(newOperator.get(1, getLongByteArray(1))));
        Assert.assertNull("Key 2 should not be present in bucket 1", newOperator.get(1, getLongByteArray(2)));

        Assert.assertEquals("Bucket 2, Key 1 value should be 200", 200,
                getLong(newOperator.get(2, getLongByteArray(1))));
        Assert.assertNull("Key 2 should not be present in bucket 2", newOperator.get(2, getLongByteArray(2)));

        newOperator.committed(3);
        Assert.assertEquals("Value is persisted ", 30, getLong(newOperator.get(1, getLongByteArray(1))));
        Assert.assertEquals("Value is persisted ", 300, getLong(newOperator.get(2, getLongByteArray(1))));

        newOperator.committed(4);
        Assert.assertEquals("Value is persisted ", 60, getLong(newOperator.get(1, getLongByteArray(0))));
        newOperator.committed(5);

        Assert.assertEquals("Value is persisted ", 700, getLong(newOperator.get(1, getLongByteArray(2))));
        Assert.assertEquals("Value is persisted ", 800, getLong(newOperator.get(2, getLongByteArray(1))));
        Assert.assertEquals("Value is persisted ", 1000, getLong(newOperator.get(2, getLongByteArray(2))));

        newOperator.checkpointed(6);
        newOperator.forceWal();

        newOperator.committed(6);
        Assert.assertEquals("Value is persisted ", 300, getLong(newOperator.get(1, getLongByteArray(3))));
        Assert.assertEquals("Value is persisted ", 100, getLong(newOperator.get(1, getLongByteArray(4))));
    }

    public long getLong(byte[] value) throws IOException {
        ByteBuffer bb = ByteBuffer.wrap(value);
        return bb.getLong();
    }

    private static final Logger logger = LoggerFactory.getLogger(WALTest.class);

}